#!/usr/bin/env python
# encoding: utf-8
"""hgsite
Create and/or upload a static copy of the repository.
The main goal is sharing Mercurial on servers with only FTP access and
statically served files, while providing the same information as hg
serve and full solutions like bitbucket and gitorious (naturally
without the interactivity).
"""
__copyright__ = """Copyright 2011 Arne Babenhauserheide
This software may be used and distributed according to the terms of the
GNU General Public License version 2 or any later version.
"""
import os
from os.path import join, isdir, isfile, basename, dirname
import shutil
import re
import mercurial
import ftplib
import socket
import datetime
from mercurial import cmdutil
from mercurial import commands
from mercurial.i18n import _
from mercurial import hg, discovery
_staticidentifier = ".statichgrepo"
templates = {
"head": """<!DOCTYPE html>
<html><head>
<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <!--duplicate for older browsers-->
<link rel="stylesheet" href="style.css" type="text/css" media="screen" />
<link rel="stylesheet" href="print.css" type="text/css" media="print" />
<title>{title}</title>
</head>
<body>
<h1>{reponame}</h1>
""",
"srchead": """<!DOCTYPE html>
<html><head>
<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <!--duplicate for older browsers-->
<link rel="stylesheet" href="style.css" type="text/css" media="screen" />
<link rel="stylesheet" href="print.css" type="text/css" media="print" />
<title>{filetitle}</title>
</head>
<body>
""",
"foot": "</body></html>\n",
"screenstyle": """ """,
"printstyle": """ """,
"manifesthead": """<h2>""" + _("Commit (click to see the diff)")+""": <a href='../../commit/{hex}.html'>{hex}</a></h2>
<p>{desc}</p><p>{user}</p>
<h2>""" + _("Files in this revision") + "</h2>",
"commitlog": """\n<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='{relativepath}src/{node}/index.html'>{desc|strip|fill68|firstline}</a></strong> <span style='font-size: xx-small'>{branches} {tags} {bookmarks}</span><p>{desc|escape}</p>\n"""
}
_indexregexp = re.compile("^\\.*index.html$")
def samefilecontent(filepath1, filepath2):
"""Check if the content of the two referenced files is equal."""
try:
with open(filepath1) as f1:
with open(filepath2) as f2:
return f1.read() == f2.read()
except OSError: return False
def contentequals(filepath, content):
"""Check if the files content is content."""
try:
with open(filepath) as f:
return f.read() == content
except OSError: return not content
def parsereadme(filepath):
"""Parse the readme file"""
with open(filepath) as r:
return "<pre>" + r.read() + "</pre>"
def overviewlogstring(ui, repo, revs, template=templates["commitlog"]):
"""Get the string for a log of the given revisions for the overview page."""
ui.pushbuffer()
t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
t.use_template(template.replace("{relativepath}", ""))
for c in revs:
ctx = repo.changectx(c)
t.show(ctx)
return ui.popbuffer()
def writeoverview(ui, repo, target, name):
"""Create the overview page"""
overview = ""
overview += templates["head"].replace("{reponame}", name).replace("{title}", name)
# add a short identifier from the first line of the readme, if it
# exists # TODO: Parse different types of readme files
readme = name
for f in os.listdir(repo.root):
if f.lower().startswith("readme"):
readme = parsereadme(f)
overview += "\n".join(readme.splitlines()[:5])
break
# now the links to the log and the files.
overview += "</pre>\n<p><a href='commits'>changelog</a> | <a href='src/" + repo["tip"].hex() + "/'>files</a></p>"
# now add the 5 most recent log entries
# divert all following ui output to a string, so we can just use standard functions
overview += "\n<h2>Changes (<a href='commits'>full changelog</a>)</h2>\n"
ui.pushbuffer()
t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
t.use_template(templates["commitlog"].replace("{relativepath}", ""))
for c in range(1, min(len(repo.changelog), 5)):
ctx = repo.changectx(str(-c))
t.show(ctx)
overview += ui.popbuffer()
# Add branch, bookmark and tag information, if they exist.
branches = []
for branch, heads in repo.branchmap().items():
if branch and branch != "default": # not default
branches.extend(heads)
tags = repo._tags
bookmarks = repo._bookmarks
if branches: # add branches
overview += "\n<h2>Branches</h2>\n"
overview += overviewlogstring(ui, repo, branches,
template=templates["commitlog"].replace(
"{branches}", "XXXXX").replace(
"{date|shortdate}", "{branches}").replace(
"XXXXX", "{date|shortdate}").replace(
"{tags}", "XXXXX").replace(
"{date|shortdate}", "{tags}").replace(
"XXXXX", "{date|shortdate}"))
if len(tags) > 1:
overview += "\n<h2>Tags</h2>\n"
overview += overviewlogstring(ui, repo, [tags[t] for t in tags if t != "tip"],
template=templates["commitlog"].replace(
"{tags}", "XXXXX").replace(
"{date|shortdate}", "{tags}").replace(
"XXXXX", "{date|shortdate}"))
if len(bookmarks):
overview += "\n<h2>Bookmarks</h2>\n"
overview += overviewlogstring(ui, repo, bookmarks.values(),
template=templates["commitlog"].replace(
"{bookmarks}", "XXXXX").replace(
"{date|shortdate}", "{bookmarks}").replace(
"XXXXX", "{date|shortdate}"))
# add the full readme
overview += "<h2>"+_("Readme")+"</h2>\n"
overview += readme
# finish the overview
overview += templates["foot"]
if not contentequals(join(target, "index.html"), overview):
with open(join(target, "index.html"), "w") as f:
f.write(overview)
def writelog(ui, repo, target, name):
"""Write the full changelog, in steps of 100."""
commits = join(target, "commits")
# create the folders
if not isdir(commits):
os.makedirs(commits)
for i in range(len(repo.changelog)/100):
d = commits+"-"+str(i+1)+"00"
if not isdir(d):
os.makedirs(d)
# create the log files
t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
t.use_template(templates["commitlog"].replace("{relativepath}", "../"))
logs = []
for ck in range(len(repo.changelog)/100+1):
ui.pushbuffer()
if ck:
dd = d
di = str(ck)+"00"
d = commits+"-"+di
logs[-1][-1] += "<p><a href=\"../commits-"+di+"\">earlier</a></p>"
if ck>2:
# the older log gets a reference to the newer one
logs[-1][-1] += "<p><a href=\"../commits-"+str(ck-2)+"00"+"\">later</a></p>"
elif ck>1:
logs[-1][-1] += "<p><a href=\"../commits\">later</a></p>"
logs.append([join(d, "index.html"), ""])
else:
d = commits
logs.append([join(d, "index.html"), ""])
logs[-1][-1] += templates["head"].replace("{reponame}", "<a href='../'>"+name+"</a>").replace("{title}", name)
for c in range(ck*100+1, min(len(repo.changelog)+1, (ck+1)*100)):
ctx = repo.changectx(str(-c))
t.show(ctx)
logs[-1][-1] += ui.popbuffer()
for filepath,data in logs:
data += templates["foot"].replace("{reponame}", "<a href='../'>"+name+"</a>")
if not contentequals(filepath,data):
with open(filepath, "w") as f:
f.write(data)
def writecommits(ui, repo, target, name, force=False):
"""Write all not yet existing commit files."""
commit = join(target, "commit")
# create the folders
if not isdir(commit):
os.makedirs(commit)
t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
t.use_template(templates["commitlog"].replace("{relativepath}", "../"))
for c in range(len(repo.changelog)):
ctx = repo.changectx(str(c))
cpath = join(commit, ctx.hex() + ".html")
if not force and isfile(cpath):
continue
with open(cpath, "w") as cf:
cf.write(templates["head"].replace("{reponame}", "<a href='../'>"+name+"</a>").replace("{title}", name))
ui.pushbuffer()
t.show(ctx)
cf.write(ui.popbuffer())
ui.pushbuffer()
commands.diff(ui, repo, change=str(c), git=True)
cf.write("<pre>"+ui.popbuffer().replace("<", "<")+"</pre>")
cf.write(templates["foot"].replace("{reponame}", "<a href='../'>"+name+"</a>"))
def escapename(filename):
"""escape index.html as .index.html and .ind… as ..ind… and so fort."""
if _indexregexp.match(filename) is not None:
return "." + filename
else: return filename
def parsesrcdata(data):
"""Parse a src file into a html file."""
return "<pre>"+data.replace("<", "<")+"</pre>"
def srcpath(target, ctx, filename):
"""Get the relative path to the static sourcefile for an already escaped filename."""
return join(target,"src",ctx.hex(),filename+".html")
def rawpath(target, ctx, filename):
"""Get the relative path to the static sourcefile for an already escaped filename."""
return join(target,"raw",ctx.hex(),filename)
def createindex(target, ctx):
"""Create an index page for the changecontext: the commit message + the user + all files in the changecontext."""
# first the head
index = templates["manifesthead"].replace(
"{hex}", ctx.hex()).replace(
"{desc}", ctx.description()).replace(
"{user}", ctx.user())
# then the files
index += "<ul>"
for filename in ctx:
filectx = ctx[filename]
lasteditctx = filectx.filectx(filectx.filerev())
index += "<li><a href='../../"+ join("src",lasteditctx.hex(), escapename(filename)+".html") + "'>" + filename + "</a>"# (<a href='../../" + join("raw",lasteditctx.hex(), filename) + "'>raw</a>)</li>"
index += "</ul>"
return index
def writesourcetree(ui, repo, target, name, force, rawfiles=False):
"""Write manifests for all commits and websites for all files.
* For each file, write sites for all revisions where the file was changed: under src/<hex>/path as html site (with linenumbers and maybe colored source), under raw/<hex>/<path> as plain files. If there is an index.html file, write it as .index.html. If there also is .index.html, turn it to ..index.html, …
* For each commit write an index with links to the included files at their latest revisions before/at the commit.
"""
# first write all files in all commits.
for c in range(len(repo.changelog)):
ctx = repo.changectx(str(c))
for filename in ctx.files():
try:
filectx = ctx.filectx(filename)
except LookupError, e:
ui.warn("File not found, likely moved ", e, "\n")
if rawfiles:
# first write the raw data
filepath = rawpath(target,ctx,filectx.path())
# skip already existing files
if not force and isfile(filepath):
continue
try:
os.makedirs(dirname(filepath))
except OSError: pass # exists
with open(filepath, "w") as f:
f.write(filectx.data())
# then write it as html
_filenameescaped = escapename(filectx.path())
filepath = srcpath(target,ctx,_filenameescaped)
if not force and isfile(filepath):
continue
try:
os.makedirs(dirname(filepath))
except OSError: pass # exists
with open(filepath, "w") as f:
f.write(templates["srchead"].replace("{filetitle}", name+": " + filename))
f.write(parsesrcdata(filectx.data()))
f.write(templates["foot"].replace("{reponame}", name))
# then write manifests for all commits
for c in range(len(repo.changelog)):
ctx = repo.changectx(str(c))
filepath = join(target,"src",ctx.hex(),"index.html")
# skip already existing files
if not force and isfile(filepath):
continue
try:
os.makedirs(dirname(filepath))
except OSError: pass # exists
with open(filepath, "w") as f:
f.write(templates["head"].replace("{reponame}", "<a href='../../'>"+name+"</a>").replace("{title}", name))
f.write(createindex(target, ctx))
f.write(templates["foot"].replace("{reponame}", "<a href='../../'>"+name+"</a>"))
def parsesite(ui, repo, target, **opts):
"""Create the static folder."""
idfile = join(target, _staticidentifier)
if not isdir(target):
# make sure the target exists
os.makedirs(target)
else: # make sure it is a staticrepo
if not isfile(idfile):
if not ui.prompt("The target folder exists is no static repo. Really use it?", default="n").lower() in ["y", "yes"]:
return
with open(idfile, "w") as i:
i.write("")
if opts["name"]:
name = opts["name"]
elif target != "static": name = target
else: name = basename(repo.root)
# first the stylesheets
screenstyle = opts["screenstyle"]
screenfile = join(target, "style.css")
if screenstyle and not samefilecontent(screenstyle, screenfile):
shutil.copyfile(screenstyle, screenfile)
elif not contentequals(screenfile,templates["screenstyle"]):
with open(join(target, "style.css"), "w") as f:
f.write(templates["screenstyle"])
printstyle = opts["printstyle"]
printfile = join(target, "print.css")
if printstyle and not samefilecontent(printstyle, printfile):
shutil.copyfile(printstyle, printfile)
elif not contentequals(printfile, templates["printstyle"]):
with open(printfile, "w") as f:
f.write(templates["printstyle"])
# then the overview
writeoverview(ui, repo, target, name)
# and the log
writelog(ui, repo, target, name)
# and all commit files
writecommits(ui, repo, target, name, force=opts["force"])
# and all file data
writesourcetree(ui, repo, target, name, force=opts["force"])
def addrepo(ui, repo, target, bookmarks):
"""Add the repo to the target and make sure it is up to date."""
try:
commands.init(ui, dest=target)
except mercurial.error.RepoError, e:
# already exists
pass
ui.pushbuffer()
if bookmarks:
commands.push(ui, repo, dest=target, bookmark=repo._bookmarks)
else:
commands.push(ui, repo, dest=target)
ui.popbuffer()
def upload(ui, repo, target, ftpstring, force):
"""upload the repo to the FTP server identified by the ftp string."""
user, password = ftpstring.split("@")[0].split(":")
serverandpath = "@".join(ftpstring.split("@")[1:])
server = serverandpath.split("/")[0]
ftppath = "/".join(serverandpath.split("/")[1:])
timeout = 10
try:
ftp = ftplib.FTP(server, user, password, "", timeout)
except socket.timeout:
ui.warn(_("connection to "), server, _(" timed out after "), timeout, _(" seconds.\n"))
return
ui.status(ftp.getwelcome(), "\n")
# create the target dir.
serverdir = dirname(ftppath)
serverdirparts = ftppath.split("/")
sd = serverdirparts[0]
if not sd in ftp.nlst():
ftp.mkd(sd)
for sdp in serverdirparts[1:]:
sdo = sd
sd = os.path.join(sd, sdp)
if not sd in ftp.nlst(sdo):
ftp.mkd(sd)
ftp.cwd(ftppath)
if not ftp.pwd() == "/" + ftppath:
ui.warn(_("not in the correct ftp directory. Cowardly bailing out.\n"))
return
#ftp.dir()
#return
ftpfeatures = ftp.sendcmd("FEAT")
featuremtime = " MDTM" in ftpfeatures.splitlines()
_ftplistcache = set()
for d, dirnames, filenames in os.walk(target):
for filename in filenames:
localfile = join(d, filename)
serverfile = localfile[len(target)+1:]
serverdir = dirname(serverfile)
serverdirparts = serverdir.split("/")
# print serverdirparts, serverfile
with open(localfile, "rb") as f:
sd = serverdirparts[0]
if sd and not sd in _ftplistcache: # should happen only once per superdir
_ftplistcache.update(set(ftp.nlst()))
if sd and not sd in _ftplistcache:
try:
ui.status(_("creating directory "), sd, "\n")
ftp.mkd(sd)
_ftplistcache.add(sd)
except ftplib.error_perm, resp:
ui.warn(_("could not create directory "), sd, ": " , resp, "\n")
else: _ftplistcache.add(sd)
for sdp in serverdirparts[1:]:
sdold = sd
sd = join(sd, sdp)
#print sd, sdp
#print ftp.nlst(sdold)
if sd and not sd in _ftplistcache: # should happen only once per superdir
_ftplistcache.update(set(ftp.nlst(sdold)))
if sd and not sd in _ftplistcache:
try:
ui.status(_("creating directory "), sd, "\n")
ftp.mkd(sd)
_ftplistcache.add(sd)
except ftplib.error_perm, resp:
ui.warn(_("could not create directory "),
sd, ": " , resp, "\n")
if not serverfile in _ftplistcache: # should happen for existing files only once per dir.
_ftplistcache.update(set(ftp.nlst(serverdir)))
if not serverfile in _ftplistcache or force:
if force:
ui.status(_("uploading "), serverfile,
_(" because I am forced to.\n"))
else:
ui.status(_("uploading "), serverfile,
_(" because it is not yet online.\n"))
ftp.storbinary("STOR "+ serverfile, f)
else:
# reupload the file if the file on the server is older than the local file.
if featuremtime:
ftpmtime = ftp.sendcmd("MDTM " + serverfile).split()[1]
localmtime = os.stat(localfile).st_mtime
localmtimestr = datetime.datetime.utcfromtimestamp(localmtime).strftime("%Y%m%d%H%M%S")
newer = int(localmtimestr) > int(ftpmtime)
if newer:
ui.status(_("uploading "), serverfile,
_(" because it is newer than the file on the FTP server.\n"))
ftp.storbinary("STOR "+ serverfile, f)
def staticsite(ui, repo, target=None, **opts):
"""Create a static copy of the repository and/or upload it to an FTP server."""
if repo.root == target:
ui.warn(_("static target repo can’t be the current repo"))
return
if not target: target = "static"
#print repo["."].branch()
# add the hg repo to the static site
# currently we need to either include all bookmarks or not, because we don’t have the remote repo when parsing the site.
# TODO: I don’t know if that is the correct way to go. Maybe always push all.
bookmarks = opts["bookmarks"]
addrepo(ui, repo, target, bookmarks)
# first: just create the site.
parsesite(ui, repo, target, **opts)
if opts["upload"]:
# upload the repo
upload(ui, repo, target, opts["upload"], opts["force"])
cmdtable = {
# "command-name": (function-call, options-list, help-string)
"site": (staticsite,
[
#('r', 'rev', None, 'parse the given revision'),
#('a', 'all', None, 'parse all revisions (requires much space)'),
('n', 'name', "", 'the repo name. Default: folder or last segment of the repo-path.'),
('u', 'upload', "", 'upload the repo to the given ftp host. Format: user:password@host/path/to/dir'),
('f', 'force', False, 'force recreating all commit files. Slow.'),
('s', 'screenstyle', "", 'use a custom stylesheet for display on screen'),
('p', 'printstyle', "", 'use a custom stylesheet for printing'),
('B', 'bookmarks', False, 'include the bookmarks')],
"[options] [folder]")
}