(Arne Babenhauserheide)
2013-06-20: merge schemes code from bab into wot-code from operhiem1. merge schemes code from bab into wot-code from operhiem1.
diff --git a/.bugs/bugs b/.bugs/bugs
new file mode 100644
--- /dev/null
+++ b/.bugs/bugs
@@ -0,0 +1,2 @@
+set the timezone to UTC on cloning a freenet repo to avoid timezone-based attacks. | owner:Arne Babenhauserheide <bab@draketo.de>, open:True, id:4dfc4cc28a7fa69f040776a7138da78ee89ec819, time:1355764180.36
+mime-type problems | owner:Arne Babenhauserheide <bab@draketo.de>, open:False, id:5916e6e8328e20d8b0276b76b7116dd432730778, time:1353463866.97
diff --git a/.bugs/details/4dfc4cc28a7fa69f040776a7138da78ee89ec819.txt b/.bugs/details/4dfc4cc28a7fa69f040776a7138da78ee89ec819.txt
new file mode 100644
--- /dev/null
+++ b/.bugs/details/4dfc4cc28a7fa69f040776a7138da78ee89ec819.txt
@@ -0,0 +1,564 @@
+# Lines starting with '#' and sections without content
+# are not displayed by a call to 'details'
+#
+[paths]
+# Paths related to this bug.
+# suggested format: REPO_PATH:LINENUMBERS
+
+
+[details]
+# Additional details
+
+
+[expected]
+# The expected result
+
+
+[actual]
+# What happened instead
+
+
+[reproduce]
+# Reproduction steps
+
+
+[comments]
+# Comments and updates - leave your name
+
+Potential patches (export of my experiments to ensure that the code does not get lost):
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1320398320 -3600
+# Node ID d9e348082e48f87c4ecb05f0930b754348168c1b
+# Parent a708b65baeb92ce4971cccaee5b799b88595ddb2
+clone: get all bookmarks before updating
+* * *
+clone: FIX: also get the bookmarks for remote target repos which support pushkey.
+
+diff --git a/mercurial/hg.py b/mercurial/hg.py
+--- a/mercurial/hg.py
++++ b/mercurial/hg.py
+@@ -353,6 +353,21 @@ def clone(ui, peeropts, source, dest=Non
+ if dircleanup:
+ dircleanup.close()
+
++ # clone all bookmarks
++ if destrepo.local() and srcrepo.capable("pushkey"):
++ rb = srcrepo.listkeys('bookmarks')
++ for k, n in rb.iteritems():
++ try:
++ m = destrepo.lookup(n)
++ destrepo._bookmarks[k] = m
++ except error.RepoLookupError:
++ pass
++ if rb:
++ bookmarks.write(destrepo)
++ elif srcrepo.local() and destrepo.capable("pushkey"):
++ for k, n in srcrepo._bookmarks.iteritems():
++ destrepo.pushkey('bookmarks', k, '', hex(n))
++
+ if destrepo.local():
+ fp = destrepo.opener("hgrc", "w", text=True)
+ fp.write("[paths]\n")
+@@ -378,21 +393,6 @@ def clone(ui, peeropts, source, dest=Non
+ destrepo.ui.status(_("updating to branch %s\n") % bn)
+ _update(destrepo, uprev)
+
+- # clone all bookmarks
+- if destrepo.local() and srcrepo.capable("pushkey"):
+- rb = srcrepo.listkeys('bookmarks')
+- for k, n in rb.iteritems():
+- try:
+- m = destrepo.lookup(n)
+- destrepo._bookmarks[k] = m
+- except error.RepoLookupError:
+- pass
+- if rb:
+- bookmarks.write(destrepo)
+- elif srcrepo.local() and destrepo.capable("pushkey"):
+- for k, n in srcrepo._bookmarks.iteritems():
+- destrepo.pushkey('bookmarks', k, '', hex(n))
+-
+ return srcrepo, destrepo
+ finally:
+ release(srclock, destlock)
+
+# HG changeset patch
+# User bab@draketo.de
+# Date 1343931127 -7200
+# Branch stable
+# Node ID f5e211663739e31f2e476c43992ee5335f9d8146
+# Parent 00182b3d087909e3c3ae44761efecdde8f319ef3
+revsets: added branchpoint() for revisions with more than one child.
+
+Reason: Get very terse information via
+
+ hg glog --rev "head() or merge() or branchpoint()"
+
+diff --git a/mercurial/revset.py b/mercurial/revset.py
+--- a/mercurial/revset.py
++++ b/mercurial/revset.py
+@@ -710,6 +710,15 @@ def merge(repo, subset, x):
+ cl = repo.changelog
+ return [r for r in subset if cl.parentrevs(r)[1] != -1]
+
++def branchpoint(repo, subset, x):
++ """``branchpoint()``
++ Changeset has more than one child.
++ """
++ # i18n: "merge" is a keyword
++ getargs(x, 0, 0, _("branchpoint takes no arguments"))
++ cl = repo.changelog
++ return [r for r in subset if cl.children(repo[r].node())[1:]]
++
+ def minrev(repo, subset, x):
+ """``min(set)``
+ Changeset with lowest revision number in set.
+@@ -1137,6 +1146,7 @@ symbols = {
+ "bisected": bisected,
+ "bookmark": bookmark,
+ "branch": branch,
++ "branchpoint": branchpoint,
+ "children": children,
+ "closed": closed,
+ "contains": contains,
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355793798 -3600
+# Node ID c4e74e535082cee18b73f09cf5b0c6b5ffbcd19d
+# Parent 7aa7380691b8815200dda268aa1af19fd56aa741
+Option to enforce using UTC for commit dates.
+
+The timezone entries in commit messages give away location information
+of the commiter, which can be dangerous when Mercurial is used
+anonymously.
+
+To mitigate that danger, this commit adds an rc-option to use UTC
+dates, except when the timezone is requested explicitely via a date
+string or by amending a commit without changing the date and time.
+
+To switch to UTC times, add
+
+ [ui]
+ datetimeutc = True
+
+to your ~/.hgrc or a .hg/hgrc.
+
+Extensions like infocalypse can also set this option when doing the
+initial clone from an anonymous source to ensure that the default
+behaviour of Mercurial is safe.
+
+diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py
+--- a/mercurial/cmdutil.py
++++ b/mercurial/cmdutil.py
+@@ -1586,7 +1586,9 @@ def commit(ui, repo, commitfunc, pats, o
+ '''commit the specified files or all outstanding changes'''
+ date = opts.get('date')
+ if date:
+- opts['date'] = util.parsedate(date)
++ opts['date'] = util.timezoneprivacy(ui.configbool('ui', 'datetimeutc'),
++ date)
++
+ message = logmessage(ui, opts)
+
+ # extract addremove carefully -- this function can be called from a command
+diff --git a/mercurial/commands.py b/mercurial/commands.py
+--- a/mercurial/commands.py
++++ b/mercurial/commands.py
+@@ -1304,9 +1304,12 @@ def commit(ui, repo, *pats, **opts):
+ if not message:
+ message = old.description()
+ editor = cmdutil.commitforceeditor
++ date = util.timezoneprivacy(ui.configbool('ui', 'datetimeutc'),
++ opts.get('date'),
++ old.date())
+ return repo.commit(message,
+ opts.get('user') or old.user(),
+- opts.get('date') or old.date(),
++ date,
+ match,
+ editor=editor,
+ extra=extra)
+diff --git a/mercurial/help/config.txt b/mercurial/help/config.txt
+--- a/mercurial/help/config.txt
++++ b/mercurial/help/config.txt
+@@ -1128,6 +1128,10 @@ User interface controls.
+ changes, abort the commit.
+ Default is False.
+
++``datetimeutc``
++ Whether to always use Universal Time Coordinated (UTC) for date
++ entries when committing.
++
+ ``debug``
+ Print debugging information. True or False. Default is False.
+
+diff --git a/mercurial/util.py b/mercurial/util.py
+--- a/mercurial/util.py
++++ b/mercurial/util.py
+@@ -980,20 +980,20 @@ def shortdate(date=None):
+ """turn (timestamp, tzoff) tuple into iso 8631 date."""
+ return datestr(date, format='%Y-%m-%d')
+
++def timezone(string):
++ tz = string.split()[-1]
++ if tz[0] in "+-" and len(tz) == 5 and tz[1:].isdigit():
++ sign = (tz[0] == "+") and 1 or -1
++ hours = int(tz[1:3])
++ minutes = int(tz[3:5])
++ return -sign * (hours * 60 + minutes) * 60
++ if tz == "GMT" or tz == "UTC":
++ return 0
++ return None
++
+ def strdate(string, format, defaults=[]):
+ """parse a localized time string and return a (unixtime, offset) tuple.
+ if the string cannot be parsed, ValueError is raised."""
+- def timezone(string):
+- tz = string.split()[-1]
+- if tz[0] in "+-" and len(tz) == 5 and tz[1:].isdigit():
+- sign = (tz[0] == "+") and 1 or -1
+- hours = int(tz[1:3])
+- minutes = int(tz[3:5])
+- return -sign * (hours * 60 + minutes) * 60
+- if tz == "GMT" or tz == "UTC":
+- return 0
+- return None
+-
+ # NOTE: unixtime = localunixtime + offset
+ offset, date = timezone(string), string
+ if offset is not None:
+@@ -1151,6 +1151,35 @@ def matchdate(date):
+ start, stop = lower(date), upper(date)
+ return lambda x: x >= start and x <= stop
+
++def timezoneprivacy(privacy, datestring=None, date=None):
++ """Switch to UTC if the timezone could be a risk to
++ privacy and the timezone was not requested explicitly.
++
++ >>> withtz = parsedate("2012-12-23 10:04:23 +0300")
++ >>> localtz = makedate()[1]
++ >>> notz = timezoneprivacy(True, "2012-12-23 07:04:23")
++ >>> notz[1] == 0
++ True
++ >>> notz[0] - localtz == withtz[0]
++ True
++ >>> (notz[0], localtz) == timezoneprivacy(False, "2012-12-23 07:04:23")
++ True
++ >>> (withtz[0], -3600) == timezoneprivacy(True, "2012-12-23 08:04:23 +0100")
++ True
++ >>> (withtz[0], 18000) == timezoneprivacy(True, "2012-12-23 02:04:23 -0500")
++ True
++ """
++ when = parsedate(datestring or date or makedate())
++ if not privacy:
++ return when
++ hastimezone = timezone(datestring) is not None
++ if datestring and not hastimezone:
++ return when[0], 0
++ if datestring or date:
++ return when
++ # no explicit datestring or date: use current UTC
++ return when[0], 0
++
+ def shortuser(user):
+ """Return a short representation of a user name or email address."""
+ f = user.find('@')
+diff --git a/tests/test-commit.t b/tests/test-commit.t
+--- a/tests/test-commit.t
++++ b/tests/test-commit.t
+@@ -90,12 +90,20 @@ commit added file that has been deleted
+ dir/file
+ committed changeset 4:49176991390e
+
+-An empty date was interpreted as epoch origin
++date argument parsing
+
+ $ echo foo >> foo
+ $ hg commit -d '' -m commit-no-date
+ $ hg tip --template '{date|isodate}\n' | grep '1970'
+ [1]
++ $ echo foo >> foo
++ $ hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23' -m commit-utc
++ $ hg tip --template '{date|isodate}\n'
++ 1982-04-23 14:23 +0000
++ $ echo foo >> foo
++ $ hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23 +0100' -m commit-utc
++ $ hg tip --template '{date|isodate}\n'
++ 1982-04-23 14:23 +0100
+
+ Make sure we do not obscure unknown requires file entries (issue2649)
+
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355924730 -3600
+# Node ID 790b40844560e268f9e12a61c313279718bb6f93
+# Parent c4e74e535082cee18b73f09cf5b0c6b5ffbcd19d
+Simpler implementation of enforcing UTC for dates.
+
+diff --git a/mercurial/cmdutil.py b/mercurial/cmdutil.py
+--- a/mercurial/cmdutil.py
++++ b/mercurial/cmdutil.py
+@@ -1586,9 +1586,7 @@ def commit(ui, repo, commitfunc, pats, o
+ '''commit the specified files or all outstanding changes'''
+ date = opts.get('date')
+ if date:
+- opts['date'] = util.timezoneprivacy(ui.configbool('ui', 'datetimeutc'),
+- date)
+-
++ opts['date'] = util.parsedate(date)
+ message = logmessage(ui, opts)
+
+ # extract addremove carefully -- this function can be called from a command
+diff --git a/mercurial/commands.py b/mercurial/commands.py
+--- a/mercurial/commands.py
++++ b/mercurial/commands.py
+@@ -1276,6 +1276,9 @@ def commit(ui, repo, *pats, **opts):
+ raise util.Abort(_('can only close branch heads'))
+ extra['close'] = 1
+
++ if ui.configbool('ui', 'datetimeutc'):
++ pass #time.timezone = "UTC"
++
+ branch = repo[None].branch()
+ bheads = repo.branchheads(branch)
+
+@@ -1304,12 +1307,9 @@ def commit(ui, repo, *pats, **opts):
+ if not message:
+ message = old.description()
+ editor = cmdutil.commitforceeditor
+- date = util.timezoneprivacy(ui.configbool('ui', 'datetimeutc'),
+- opts.get('date'),
+- old.date())
+ return repo.commit(message,
+ opts.get('user') or old.user(),
+- date,
++ opts.get('date') or old.date(),
+ match,
+ editor=editor,
+ extra=extra)
+diff --git a/mercurial/util.py b/mercurial/util.py
+--- a/mercurial/util.py
++++ b/mercurial/util.py
+@@ -980,20 +980,20 @@ def shortdate(date=None):
+ """turn (timestamp, tzoff) tuple into iso 8631 date."""
+ return datestr(date, format='%Y-%m-%d')
+
+-def timezone(string):
+- tz = string.split()[-1]
+- if tz[0] in "+-" and len(tz) == 5 and tz[1:].isdigit():
+- sign = (tz[0] == "+") and 1 or -1
+- hours = int(tz[1:3])
+- minutes = int(tz[3:5])
+- return -sign * (hours * 60 + minutes) * 60
+- if tz == "GMT" or tz == "UTC":
+- return 0
+- return None
+-
+ def strdate(string, format, defaults=[]):
+ """parse a localized time string and return a (unixtime, offset) tuple.
+ if the string cannot be parsed, ValueError is raised."""
++ def timezone(string):
++ tz = string.split()[-1]
++ if tz[0] in "+-" and len(tz) == 5 and tz[1:].isdigit():
++ sign = (tz[0] == "+") and 1 or -1
++ hours = int(tz[1:3])
++ minutes = int(tz[3:5])
++ return -sign * (hours * 60 + minutes) * 60
++ if tz == "GMT" or tz == "UTC":
++ return 0
++ return None
++
+ # NOTE: unixtime = localunixtime + offset
+ offset, date = timezone(string), string
+ if offset is not None:
+@@ -1151,35 +1151,6 @@ def matchdate(date):
+ start, stop = lower(date), upper(date)
+ return lambda x: x >= start and x <= stop
+
+-def timezoneprivacy(privacy, datestring=None, date=None):
+- """Switch to UTC if the timezone could be a risk to
+- privacy and the timezone was not requested explicitly.
+-
+- >>> withtz = parsedate("2012-12-23 10:04:23 +0300")
+- >>> localtz = makedate()[1]
+- >>> notz = timezoneprivacy(True, "2012-12-23 07:04:23")
+- >>> notz[1] == 0
+- True
+- >>> notz[0] - localtz == withtz[0]
+- True
+- >>> (notz[0], localtz) == timezoneprivacy(False, "2012-12-23 07:04:23")
+- True
+- >>> (withtz[0], -3600) == timezoneprivacy(True, "2012-12-23 08:04:23 +0100")
+- True
+- >>> (withtz[0], 18000) == timezoneprivacy(True, "2012-12-23 02:04:23 -0500")
+- True
+- """
+- when = parsedate(datestring or date or makedate())
+- if not privacy:
+- return when
+- hastimezone = timezone(datestring) is not None
+- if datestring and not hastimezone:
+- return when[0], 0
+- if datestring or date:
+- return when
+- # no explicit datestring or date: use current UTC
+- return when[0], 0
+-
+ def shortuser(user):
+ """Return a short representation of a user name or email address."""
+ f = user.find('@')
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355925188 -3600
+# Node ID 8070267ca30f22357f65f96ea0ed99569639f094
+# Parent 790b40844560e268f9e12a61c313279718bb6f93
+test.
+
+diff --git a/mercurial/commands.py b/mercurial/commands.py
+--- a/mercurial/commands.py
++++ b/mercurial/commands.py
+@@ -1277,7 +1277,7 @@ def commit(ui, repo, *pats, **opts):
+ extra['close'] = 1
+
+ if ui.configbool('ui', 'datetimeutc'):
+- pass #time.timezone = "UTC"
++ time.timezone = "UTC"
+
+ branch = repo[None].branch()
+ bheads = repo.branchheads(branch)
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355925219 -3600
+# Node ID 5e3231e4caf46c078a5c0c83f64648dca630f5f1
+# Parent 8070267ca30f22357f65f96ea0ed99569639f094
+test.
+
+diff --git a/mercurial/commands.py b/mercurial/commands.py
+--- a/mercurial/commands.py
++++ b/mercurial/commands.py
+@@ -1277,7 +1277,7 @@ def commit(ui, repo, *pats, **opts):
+ extra['close'] = 1
+
+ if ui.configbool('ui', 'datetimeutc'):
+- time.timezone = "UTC"
++ time.timezone = "GMT+5"
+
+ branch = repo[None].branch()
+ bheads = repo.branchheads(branch)
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355925239 -3600
+# Node ID ce47f904c4c735ad694c4047b674abf30cda0e77
+# Parent 5e3231e4caf46c078a5c0c83f64648dca630f5f1
+test.
+
+diff --git a/mercurial/commands.py b/mercurial/commands.py
+--- a/mercurial/commands.py
++++ b/mercurial/commands.py
+@@ -1278,6 +1278,7 @@ def commit(ui, repo, *pats, **opts):
+
+ if ui.configbool('ui', 'datetimeutc'):
+ time.timezone = "GMT+5"
++ print time.timezone
+
+ branch = repo[None].branch()
+ bheads = repo.branchheads(branch)
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355945885 -3600
+# Node ID 5252b47c54bfde07dc8844e47bb33fc685de3ce8
+# Parent ce47f904c4c735ad694c4047b674abf30cda0e77
+timezone: Fix test to always use a on-utc timezone.
+
+diff --git a/tests/test-commit.t b/tests/test-commit.t
+--- a/tests/test-commit.t
++++ b/tests/test-commit.t
+@@ -97,11 +97,11 @@ date argument parsing
+ $ hg tip --template '{date|isodate}\n' | grep '1970'
+ [1]
+ $ echo foo >> foo
+- $ hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23' -m commit-utc
++ $ TZ="Europe/Berlin" hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23' -m commit-utc
+ $ hg tip --template '{date|isodate}\n'
+ 1982-04-23 14:23 +0000
+ $ echo foo >> foo
+- $ hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23 +0100' -m commit-utc
++ $ TZ="Europe/Berlin" hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23 +0100' -m commit-utc
+ $ hg tip --template '{date|isodate}\n'
+ 1982-04-23 14:23 +0100
+
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355945986 -3600
+# Node ID 909abca4327b214058989204a2afb7d32deb446b
+# Parent 5252b47c54bfde07dc8844e47bb33fc685de3ce8
+enforce UTC: time.timezone to 0
+
+diff --git a/mercurial/commands.py b/mercurial/commands.py
+--- a/mercurial/commands.py
++++ b/mercurial/commands.py
+@@ -1277,8 +1277,7 @@ def commit(ui, repo, *pats, **opts):
+ extra['close'] = 1
+
+ if ui.configbool('ui', 'datetimeutc'):
+- time.timezone = "GMT+5"
+- print time.timezone
++ time.timezone = 0
+
+ branch = repo[None].branch()
+ bheads = repo.branchheads(branch)
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1355945885 -3600
+# Node ID 0b0099e44145183560e7cc01c20e21cd3bea84d0
+# Parent c4e74e535082cee18b73f09cf5b0c6b5ffbcd19d
+timezone: Fix test to always use a on-utc timezone.
+
+diff --git a/tests/test-commit.t b/tests/test-commit.t
+--- a/tests/test-commit.t
++++ b/tests/test-commit.t
+@@ -97,11 +97,11 @@ date argument parsing
+ $ hg tip --template '{date|isodate}\n' | grep '1970'
+ [1]
+ $ echo foo >> foo
+- $ hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23' -m commit-utc
++ $ TZ="Europe/Berlin" hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23' -m commit-utc
+ $ hg tip --template '{date|isodate}\n'
+ 1982-04-23 14:23 +0000
+ $ echo foo >> foo
+- $ hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23 +0100' -m commit-utc
++ $ TZ="Europe/Berlin" hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23 +0100' -m commit-utc
+ $ hg tip --template '{date|isodate}\n'
+ 1982-04-23 14:23 +0100
+
+
+# HG changeset patch
+# User Arne Babenhauserheide <bab@draketo.de>
+# Date 1357663148 -3600
+# Node ID 204a32c7c864f5b1db86c77e037bc895445201d7
+# Parent 0b0099e44145183560e7cc01c20e21cd3bea84d0
+Fix the timezoneprivacy test: You give the local time and it records the UTC according to that.
+
+diff --git a/tests/test-commit.t b/tests/test-commit.t
+--- a/tests/test-commit.t
++++ b/tests/test-commit.t
+@@ -99,7 +99,7 @@ date argument parsing
+ $ echo foo >> foo
+ $ TZ="Europe/Berlin" hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23' -m commit-utc
+ $ hg tip --template '{date|isodate}\n'
+- 1982-04-23 14:23 +0000
++ 1982-04-23 12:23 +0000
+ $ echo foo >> foo
+ $ TZ="Europe/Berlin" hg --config ui.datetimeutc=True commit -d '1982-04-23 14:23 +0100' -m commit-utc
+ $ hg tip --template '{date|isodate}\n'
+
diff --git a/.bugs/details/5916e6e8328e20d8b0276b76b7116dd432730778.txt b/.bugs/details/5916e6e8328e20d8b0276b76b7116dd432730778.txt
new file mode 100644
--- /dev/null
+++ b/.bugs/details/5916e6e8328e20d8b0276b76b7116dd432730778.txt
@@ -0,0 +1,76 @@
+# Lines starting with '#' and sections without content
+# are not displayed by a call to 'details'
+#
+[paths]
+# Paths related to this bug.
+# suggested format: REPO_PATH:LINENUMBERS
+
+
+[details]
+# Additional details
+
+<ArneBab> toad_: I get Does not look like a MIME type: "application/mercurial-bundle;0"
+<ArneBab> when trying to pull with infocalypse
+<ArneBab> freenet.client.MetadataParseException: Does not look like a MIME type: "application/mercurial-bundle;0"
+<TheSeeker> ArneBab: because of the ;0 ?
+<SeekingFor> that may actually be the error
+<ArneBab> yes…
+<ArneBab> but I think ;x is an allowed symbol in headers.
+<SeekingFor> also in mime types?
+<SeekingFor> there is a special rfc which defines the syntax of mime types. i didn't read it yet
+<SeekingFor> the rfc for HTTP 1.0 and HTTP 1.1 say that mime types must be based on that MIME type RFC
+<SeekingFor> that's at least how i remember it
+<SeekingFor> ^ well, prepare for a run from spanish (it is spanish, right?) speaking people :)
+<SeekingFor> its a nice and short tutorial for newcomers
+<SeekingFor> wait, just 33 warnings on current fred-staging? :)
+<SeekingFor> and most of them are generic related or "not used" warnings
+<ArneBab> ech… FCP2.0 doesn't have support for user defined metadata, so we
+<ArneBab> jam the metadata we need into the mime type field.
+<SeekingFor> for my infocalypse repo for flircp fproxy shows this:
+<SeekingFor> Filename: flircp.R1-5.bin
+<SeekingFor> Size: unknown
+<SeekingFor> Expected type: application/octet-stream
+<SeekingFor> why don't you use application/octet-stream too?
+<ArneBab> content := "Content-Type" ":" type "/" subtype
+<ArneBab> *(";" parameter)
+<ArneBab> the ; is the allowed separator for parameters
+<ArneBab> so freenet is being too zealous here
+<ArneBab> http://www.ietf.org/rfc/rfc2045.txt
+<ArneBab> as far as I can tell, infocalypse uses the mime-type to supply additional information…
+<ArneBab> SeekingFor: Did the content-type handling in freenet change in the last few months?
+<ArneBab> toad_: maybe I should ask you that: : Did the content-type handling in freenet change in the last few months?
+<SeekingFor> from freenet.client.DefaultMIMETypes.java:789
+<SeekingFor> private static final String TOP_LEVEL = "(?>[a-zA-Z-]+)";
+<SeekingFor> private static final String CHARS = "(?>[a-zA-Z0-9+_\\-\\.]+)";
+<SeekingFor> private static final String PARAM = "(?>;\\s*"+CHARS+"="+"(("+CHARS+")|(\".*\")))";
+<SeekingFor> private static Pattern MIME_TYPE = Pattern.compile(TOP_LEVEL+"/"+CHARS+"\\s*"+PARAM+"*");
+<SeekingFor> public static boolean isPlausibleMIMEType(String mimeType) {
+<SeekingFor> return MIME_TYPE.matcher(mimeType).matches();
+<SeekingFor> }
+<SeekingFor> this looks like it should be ok to use *(";" paramater)
+<SeekingFor> so the error must be triggered somewhere else
+<ArneBab> yes…
+<ArneBab> does it have a list of allowed mime types?
+<SeekingFor> yes, same file
+<SeekingFor> /* From toad's /etc/mime.types
+<SeekingFor> * cat /etc/mime.types | sed "/^$/d;/#/d" | tr --squeeze '\t' ' ' |
+<SeekingFor> * (y=0; while read x; do echo "$x" |
+<SeekingFor> * sed -n "s/^\([^ ]*\)$/addMIMEType\($y, \"\1\"\);/p;s/^\([^ (),]\+\) \(.*\)$/addMIMEType\($y, \"\1\", \"\2\"\);/p;"; y=$((y+1)); done)
+<SeekingFor> */
+<SeekingFor> but that list does not contain any application/mercurial type
+
+
+[expected]
+# The expected result
+
+
+[actual]
+# What happened instead
+
+
+[reproduce]
+# Reproduction steps
+
+
+[comments]
+# Comments and updates - leave your name
diff --git a/Readme.txt b/Readme.txt
new file mode 100644
--- /dev/null
+++ b/Readme.txt
@@ -0,0 +1,16 @@
+Infocalypse: Anonymous DVCS over Freenet
+========================================
+
+The Infocalypse 2.0 hg extension is an extension for Mercurial that allows you to create, publish
+and maintain incrementally updateable repositories in Freenet.
+
+Your code is then hosted decentrally and anonymously, making it just as censorship-resistant as
+all other content in Freenet.
+
+It includes additional redundancy, fetch-optimization to reduce the number of downloads required
+for getting the code, safe reinsert to keep the repository - by any user, not just the uploader -
+and many other features.
+
+And it works transparently: To publish, just clone to freenet:
+
+ hg clone REPO freenet://USK@/REPO
diff --git a/infocalypse/__init__.py b/infocalypse/__init__.py
--- a/infocalypse/__init__.py
+++ b/infocalypse/__init__.py
@@ -50,7 +50,7 @@ Example .hgrc entry:
[infocalypse]
cfg_file = /mnt/usbkey/s3kr1t/infocalypse.cfg
-The default temp file dirctory is set to:
+The default temp file directory is set to:
~/infocalypse_tmp. It will be created if
it doesn't exist.
@@ -82,7 +82,7 @@ hg fn-push --uri USK@/test.R1/0
Pushes incremental changes from the local
directory into the existing repository.
-You can ommit the --uri argument when
+You can omit the --uri argument when
you run from the same directory the fn-create
was run in because the insert key -> dir
mapping is saved in the config file.
@@ -94,7 +94,7 @@ hg fn-pull --uri <request uri from steps
to pull from the repository in Freenet.
The request uri -> dir mapping is saved after
-the first pull, so you can ommit the --uri
+the first pull, so you can omit the --uri
argument for subsequent fn-pull invocations.
@@ -339,8 +339,13 @@ d kar bott at com cast dot net
import os
from commands import *
+from mercurial import commands, extensions, util, hg, dispatch, discovery
+from mercurial.i18n import _
+import freenetrepo
-from mercurial import commands, util
+_freenetschemes = ('freenet', )
+for _scheme in _freenetschemes:
+ hg.schemes[_scheme] = freenetrepo
#----------------------------------------------------------"
@@ -362,14 +367,15 @@ NOSEARCH_OPT = [('', 'nosearch', None, '
# Allow mercurial naming convention for command table.
# pylint: disable-msg=C0103
+PULL_OPTS = [('', 'hash', [], 'repo hash of repository to pull from'),
+ ('', 'onlytrusted', None, 'only use repo announcements from '
+ + 'known users')]
+
cmdtable = {
"fn-pull": (infocalypse_pull,
[('', 'uri', '', 'request URI to pull from'),
- ('', 'hash', [], 'repo hash of repository to pull from'),
- ('', 'wot', '', 'WoT nick@key/repo to pull from'),
- ('', 'onlytrusted', None, 'only use repo announcements from '
- + 'known users')]
- + WOT_OPTS
+ ('', 'wot', '', 'WoT nick@key/repo to pull from')]
+ + PULL_OPTS
+ FCP_OPTS
+ NOSEARCH_OPT
+ AGGRESSIVE_OPT,
@@ -502,3 +508,304 @@ commands.norepo += ' fn-setupfms'
commands.norepo += ' fn-genkey'
commands.norepo += ' fn-archive'
commands.norepo += ' fn-setupwot'
+
+
+## Wrap core commands for use with freenet keys.
+## Explicitely wrap functions to change local commands in case the remote repo is an FTP repo. See mercurial.extensions for more information.
+# Get the module which holds the functions to wrap
+# the new function: gets the original function as first argument and the originals args and kwds.
+def findcommonoutgoing(orig, *args, **opts):
+ repo = args[0]
+ remoterepo = args[1]
+ capable = getattr(remoterepo, 'capable', lambda x: False)
+ if capable('infocalypse'):
+ class fakeoutgoing(object):
+ def __init__(self):
+ self.excluded = []
+ self.missing = repo.heads()
+ self.missingheads = []
+ self.commonheads = []
+ return fakeoutgoing()
+ else:
+ return orig(*args, **opts)
+# really wrap the functions
+extensions.wrapfunction(discovery, 'findcommonoutgoing', findcommonoutgoing)
+
+# wrap the commands
+
+def freenetpathtouri(path):
+ path = path.replace("%7E", "~").replace("%2C", ",")
+ if path.startswith("freenet://"):
+ return path[len("freenet://"):]
+ if path.startswith("freenet:"):
+ return path[len("freenet:"):]
+ return path
+
+def freenetpull(orig, *args, **opts):
+ def parsepushargs(ui, repo, path=None):
+ return ui, repo, path
+ def isfreenetpath(path):
+ try:
+ if path.startswith("freenet:") or path.startswith("USK@"):
+ return True
+ except AttributeError:
+ return False
+ return False
+ ui, repo, path = parsepushargs(*args)
+ if not path:
+ path = ui.expandpath('default', 'default-push')
+ else:
+ path = ui.expandpath(path)
+ # only act differently, if the target is an infocalypse repo.
+ if not isfreenetpath(path):
+ return orig(*args, **opts)
+ uri = freenetpathtouri(path)
+ opts["uri"] = uri
+ opts["aggressive"] = True # always search for the latest revision.
+ return infocalypse_pull(ui, repo, **opts)
+
+def fixnamepart(namepart):
+ """use redundant keys by default, except if explicitely
+ requested otherwise.
+
+ parse the short form USK@/reponame to upload to a key
+ in the form USK@<key>/reponame.R1/0 - avoids the very easy
+ to make error of forgetting the .R1"""
+ nameparts = namepart.split("/")
+ name = nameparts[0]
+ if nameparts[1:]: # user supplied a number
+ number = nameparts[1]
+ else: number = "0"
+ if not name.endswith(".R0") and not name.endswith(".R1"):
+ name = name + ".R1"
+ namepart = name + "/" + number
+ return namepart
+
+def freenetpush(orig, *args, **opts):
+ def parsepushargs(ui, repo, path=None):
+ return ui, repo, path
+ def isfreenetpath(path):
+ if path and path.startswith("freenet:") or path.startswith("USK@"):
+ return True
+ return False
+ ui, repo, path = parsepushargs(*args)
+ if not path:
+ path = ui.expandpath('default-push', 'default')
+ else:
+ path = ui.expandpath(path)
+ # only act differently, if the target is an infocalypse repo.
+ if not isfreenetpath(path):
+ return orig(*args, **opts)
+ uri = freenetpathtouri(path)
+ # if the uri is the short form (USK@/name/#), generate the key and preprocess the uri.
+ if uri.startswith("USK@/"):
+ ui.status("creating a new key for the repo. For a new repo with an existing key, use clone.\n")
+ from sitecmds import genkeypair
+ fcphost, fcpport = opts["fcphost"], opts["fcpport"]
+ if fcphost == '':
+ fcphost = '127.0.0.1'
+ if fcpport == 0:
+ fcpport = 9481
+
+ # use redundant keys by default, except if explicitely requested otherwise.
+ namepart = uri[5:]
+ namepart = fixnamepart(namepart)
+ insert, request = genkeypair(fcphost, fcpport)
+ uri = "USK"+insert[3:]+namepart
+ opts["uri"] = uri
+ opts["aggressive"] = True # always search for the latest revision.
+ return infocalypse_create(ui, repo, **opts)
+ opts["uri"] = uri
+ opts["aggressive"] = True # always search for the latest revision.
+ return infocalypse_push(ui, repo, **opts)
+
+def freenetclone(orig, *args, **opts):
+ def parsepushargs(ui, repo, path=None):
+ return ui, repo, path
+
+ def isfreenetpath(path):
+ try:
+ if path.startswith("freenet:") or path.startswith("USK@"):
+ return True
+ except AttributeError:
+ return False
+ return False
+ ui, source, dest = parsepushargs(*args)
+ # only act differently, if dest or source is an infocalypse repo.
+ if not isfreenetpath(source) and not isfreenetpath(dest):
+ return orig(*args, **opts)
+
+ if not dest:
+ if not isfreenetpath(source):
+ dest = hg.defaultdest(source)
+ else: # this is a freenet key. It has a /# at the end and
+ # could contain .R1 or .R0 as pure technical identifiers
+ # which we do not need in the local name.
+ dest = source.split("/")[-2]
+ if dest.endswith(".R1") or dest.endswith(".R0"):
+ dest = dest[:-3]
+
+ # check whether to create, pull or copy
+ pulluri, pushuri = None, None
+ if isfreenetpath(source):
+ pulluri = freenetpathtouri(source)
+ if isfreenetpath(dest):
+ pushuri = freenetpathtouri(dest)
+
+ # decide which infocalypse command to use.
+ if pulluri and pushuri:
+ action = "copy"
+ elif pulluri:
+ action = "pull"
+ elif pushuri:
+ action = "create"
+ else:
+ raise util.Abort("""Can't clone without source and target. This message should not be reached. If you see it, this is a bug.""")
+
+ if action == "copy":
+ raise util.Abort("""Cloning without intermediate local repo not yet supported in the simplified commands. Use fn-copy directly.""")
+
+ if action == "create":
+ # if the pushuri is the short form (USK@/name/#), generate the key.
+ if pushuri.startswith("USK@/"):
+ ui.status("creating a new key for the repo. To use your default key, call fn-create.\n")
+ from sitecmds import genkeypair
+ fcphost, fcpport = opts["fcphost"], opts["fcpport"]
+ if fcphost == '':
+ fcphost = '127.0.0.1'
+ if fcpport == 0:
+ fcpport = 9481
+
+ # use redundant keys by default, except if explicitely requested otherwise.
+ namepart = pushuri[5:]
+ namepart = fixnamepart(namepart)
+ insert, request = genkeypair(fcphost, fcpport)
+ pushuri = "USK"+insert[3:]+namepart
+ elif pushuri.endswith("/0"): # initial create, catch the no-.R1 error
+ pass
+ # this rewriting is dangerous here since it could make it
+ # impossible to update old repos when they drop
+ # out. Leaving it commented out for now. TODO: Always
+ # treat a name without .R0 as requesting redundancy *in.
+ # the backend*. Keep it as /name/#, but add /name.Rn/0
+ # backup repos. Needs going into the backend.
+
+ #namepart = pushuri.split("/")[-2] + "/0"
+ #namepartpos = -len(namepart)
+ #namepart2 = fixnamepart(namepart)
+ # if namepart2 != namepart:
+ # ui.status("changed the repo name to " + namepart2 + " to have more redundancy and longer lifetime. This is a small tweak on infocalypse to avoid the frequent error of forgetting to add .R1 to the name. If you really want no additional redundancy for your repo, use NAME.R0 or call hg fn-create directly.\n")
+ #pushuri = pushuri[:namepartpos] + namepart
+ opts["uri"] = pushuri
+ repo = hg.repository(ui, ui.expandpath(source))
+ return infocalypse_create(ui, repo, **opts)
+
+ if action == "pull":
+ if os.path.exists(dest):
+ raise util.Abort(_("destination " + dest + " already exists."))
+ # create the repo
+ req = dispatch.request(["init", dest], ui=ui)
+ dispatch.dispatch(req)
+ # pull the data from freenet
+ origdest = ui.expandpath(dest)
+ dest, branch = hg.parseurl(origdest)
+ destrepo = hg.repository(ui, dest)
+ infocalypse_pull(ui, destrepo, aggressive=True, hash=None, uri=pulluri, **opts)
+ # store the request uri for future updates
+ with destrepo.opener("hgrc", "a", text=True) as f:
+ f.write("""[paths]
+default = freenet://""" + pulluri + """
+
+[ui]
+username = anonymous
+""" )
+ ui.warn("As basic protection, infocalypse automatically set the username 'anonymous' for commits in this repo. To change this, edit " + str(os.path.join(destrepo.root, ".hg", "hgrc")))
+ # and update the repo
+ return hg.update(destrepo, None)
+
+
+# really wrap the command
+entry = extensions.wrapcommand(commands.table, "push", freenetpush)
+entry[1].extend(FCP_OPTS)
+entry = extensions.wrapcommand(commands.table, "pull", freenetpull)
+entry[1].extend(PULL_OPTS)
+entry[1].extend(FCP_OPTS)
+entry = extensions.wrapcommand(commands.table, "clone", freenetclone)
+entry[1].extend(FCP_OPTS)
+
+
+# Starting an FTP repo. Not yet used, except for throwing errors for missing commands and faking the lock.
+
+from mercurial import util
+try:
+ from mercurial.peer import peerrepository
+except ImportError:
+ from mercurial.repo import repository as peerrepository
+try:
+ from mercurial.error import RepoError
+except ImportError:
+ from mercurial.repo import RepoError
+
+class InfocalypseRepository(peerrepository):
+ def __init__(self, ui, path, create):
+ self.create = create
+ self.ui = ui
+ self.path = path
+ self.capabilities = set(["infocalypse"])
+ self.branchmap = {}
+
+ def lock(self):
+ """We cannot really lock Infocalypse repos, yet.
+
+ TODO: Implement as locking the repo in the static site folder."""
+ class DummyLock:
+ def release(self):
+ pass
+ l = DummyLock()
+ return l
+
+ def url(self):
+ return self.path
+
+ def lookup(self, key):
+ return key
+
+ def cancopy(self):
+ return False
+
+ def heads(self, *args, **opts):
+ """
+ Whenever this function is hit, we abort. The traceback is useful for
+ figuring out where to intercept the functionality.
+ """
+ raise util.Abort('command heads unavailable for Infocalypse repositories')
+
+ def pushkey(self, namespace, key, old, new):
+ return False
+
+ def listkeys(self, namespace):
+ return {}
+
+ def push(self, remote, force=False, revs=None, newbranch=None):
+ raise util.Abort('command push unavailable for Infocalypse repositories')
+
+ def pull(self, remote, heads=[], force=False):
+ raise util.Abort('command pull unavailable for Infocalypse repositories')
+
+ def findoutgoing(self, remote, base=None, heads=None, force=False):
+ raise util.Abort('command findoutgoing unavailable for Infocalypse repositories')
+
+
+class RepoContainer(object):
+ def __init__(self):
+ pass
+
+ def __repr__(self):
+ return '<InfocalypseRepository>'
+
+ def instance(self, ui, url, create):
+ # Should this use urlmod.url(), or is manual parsing better?
+ #context = {}
+ return InfocalypseRepository(ui, url, create)
+
+hg.schemes["freenet"] = RepoContainer()
diff --git a/infocalypse/freenetrepo.py b/infocalypse/freenetrepo.py
new file mode 100644
--- /dev/null
+++ b/infocalypse/freenetrepo.py
@@ -0,0 +1,55 @@
+# infocalypse
+#
+# Copyright 2012 Arne Babenhauserheide <arne_bab at web dot de>,
+# though most of this file is taken from hg-git from Scott Chacon
+# <schacon at gmail dot com> also some code (and help) borrowed
+# from durin42
+#
+# This software may be used and distributed according to the terms
+# of the GNU General Public License, incorporated herein by reference.
+
+from mercurial import util
+try:
+ from mercurial.peer import peerrepository
+except ImportError:
+ from mercurial.repo import repository as peerrepository
+try:
+ from mercurial.error import RepoError
+except ImportError:
+ from mercurial.repo import RepoError
+
+class freenetrepo(peerrepository):
+ capabilities = ['lookup']
+
+ def __init__(self, ui, path, create):
+ if create: # pragma: no cover
+ raise util.Abort('Cannot create a freenet repository, yet.')
+ self.ui = ui
+ self.path = path
+
+ def lookup(self, key):
+ if isinstance(key, str):
+ return key
+
+ def local(self):
+ # a freenet repo is never local
+ raise RepoError
+
+ def heads(self):
+ return []
+
+ def listkeys(self, namespace):
+ return {}
+
+ def pushkey(self, namespace, key, old, new):
+ return False
+
+ # used by incoming in hg <= 1.6
+ def branches(self, nodes):
+ return []
+
+instance = freenetrepo
+
+def islocal(path):
+ u = util.url(path)
+ return not u.scheme or u.scheme == 'file'
diff --git a/infocalypse/graph.py b/infocalypse/graph.py
--- a/infocalypse/graph.py
+++ b/infocalypse/graph.py
@@ -52,7 +52,7 @@ FREENET_BLOCK_LEN = 32 * 1024
# Hmmm... arbitrary.
MAX_REDUNDANT_LENGTH = 128 * 1024
-# HACK: Approximate multi-level splitfile boundry
+# HACK: Approximate multi-level splitfile boundary
MAX_METADATA_HACK_LEN = 7 * 1024 * 1024
############################################################
diff --git a/infocalypse/graphutil.py b/infocalypse/graphutil.py
--- a/infocalypse/graphutil.py
+++ b/infocalypse/graphutil.py
@@ -187,8 +187,8 @@ def should_add_head(repo, version_table,
return True
# TRICKY:
-# You need the repository changset DAG in order to determine
-# the base revs. because new changes might have branched from
+# You need the repository changeset DAG in order to determine
+# the base revs, because new changes might have branched from
# a change in the middle of a previous index which doesn't
# appear explictly in the graph.
#
@@ -450,7 +450,7 @@ def get_huge_top_key_edges(graph, extant
alternates) that are too big to salt.
If extant is True, return existing edges.
- If extent is False, return edges that could be added. """
+ If extant is False, return edges that could be added. """
ret = []
edges = graph.get_top_key_edges()
edges += find_redundant_edges(graph, edges, False)[0]
diff --git a/infocalypse/infcmds.py b/infocalypse/infcmds.py
--- a/infocalypse/infcmds.py
+++ b/infocalypse/infcmds.py
@@ -57,16 +57,16 @@ from knownrepos import DEFAULT_TRUST, DE
DEFAULT_PARAMS = {
# FCP params
- 'MaxRetries':3,
+ 'MaxRetries':6,
'PriorityClass':1,
'DontCompress':True, # hg bundles are already compressed.
'Verbosity':1023, # MUST set this to get progress messages.
#'GetCHKOnly':True, # REDFLAG: For testing only. Not sure this still works.
# Non-FCP stuff
- 'N_CONCURRENT':4, # Maximum number of concurrent FCP requests.
- 'CANCEL_TIME_SECS': 20 * 60, # Bound request time.
- 'POLL_SECS':0.25, # Time to sleep in the polling loop.
+ 'N_CONCURRENT':8, # Maximum number of concurrent FCP requests.
+ 'CANCEL_TIME_SECS': 120 * 60, # Bound request time.
+ 'POLL_SECS':1.00, # Time to sleep in the polling loop.
# Testing HACKs
#'TEST_DISABLE_GRAPH': True, # Disable reading the graph.
diff --git a/infocalypse/insertingbundles.py b/infocalypse/insertingbundles.py
--- a/infocalypse/insertingbundles.py
+++ b/infocalypse/insertingbundles.py
@@ -247,8 +247,9 @@ class InsertingBundles(RequestQueueState
self.parent.ctx.ui_.status("Bad CHK: %s %s\n" %
(str(edge), chk1))
self.parent.ctx.ui_.warn("CHK for reinserted edge doesn't "
- + "match!\n")
+ + "match!\nPossibly inserted with a different version of Mercurial.\n")
self.parent.transition(FAILING)
+ return
else:
# REDFLAG: retrying?
diff --git a/infocalypse/sitecmds.py b/infocalypse/sitecmds.py
--- a/infocalypse/sitecmds.py
+++ b/infocalypse/sitecmds.py
@@ -176,13 +176,19 @@ And this is what needs to go into fn-cre
%s
"""
+def genkeypair(fcphost, fcpport):
+ """ Generate a keypair
+
+ :returns: inserturi, requesturi (strings)
+ """
+ client = FCPClient.connect(fcphost, fcpport)
+ client.message_callback = lambda x, y : None # silence.
+ resp = client.generate_ssk()
+ return resp[1]['InsertURI'], resp[1]['RequestURI']
+
def execute_genkey(ui_, params):
""" Run the genkey command. """
- client = FCPClient.connect(params['FCP_HOST'],
- params['FCP_PORT'])
-
- client.message_callback = lambda x, y : None # silence.
- resp = client.generate_ssk()
- ui_.status(MSG_FMT % (resp[1]['InsertURI'], resp[1]['RequestURI'],
- resp[1]['InsertURI'].split('/')[0] +'/',
- "U" + resp[1]['InsertURI'].split('/')[0][1:] +'/NAME/0'))
+ insert, request = genkeypair(params['FCP_HOST'], params['FCP_PORT'])
+ ui_.status(MSG_FMT % (insert, request,
+ insert.split('/')[0] +'/',
+ "U" + insert.split('/')[0][1:] +'/NAME.R1/0'))