As is checkin of reinserting. CLEANUP.
diff --git a/infocalypse/__init__.py b/infocalypse/__init__.py
--- a/infocalypse/__init__.py
+++ b/infocalypse/__init__.py
@@ -121,7 +121,7 @@ import os
from mercurial import commands, util
from infcmds import get_config_info, execute_create, execute_pull, \
- execute_push, execute_setup, execute_copy
+ execute_push, execute_setup, execute_copy, execute_reinsert
def set_target_version(ui_, repo, opts, params, msg_fmt):
""" INTERNAL: Update TARGET_VERSION in params. """
@@ -172,6 +172,32 @@ def infocalypse_copy(ui_, repo, **opts):
params['REQUEST_URI'] = request_uri
execute_copy(ui_, repo, params, stored_cfg)
+def infocalypse_reinsert(ui_, repo, **opts):
+ """ Re-insert Infocalypse repository data. """
+ params, stored_cfg = get_config_info(ui_, opts)
+
+ request_uri = opts['requesturi']
+ if request_uri == '':
+ request_uri = stored_cfg.get_request_uri(repo.root)
+ if not request_uri:
+ ui_.warn("There is no stored request URI for this repo.\n"
+ "Please set one with the --requesturi option.\n")
+ return
+
+ insert_uri = opts['inserturi']
+ if insert_uri == '':
+ insert_uri = stored_cfg.get_dir_insert_uri(repo.root)
+ # REDFLAG: fix parameter definition so that it is required?
+ if not insert_uri:
+ ui_.status("No insert URI specified. Will skip re-insert "
+ +"of top key.\n")
+ insert_uri = None
+
+ params['INSERT_URI'] = insert_uri
+ params['REQUEST_URI'] = request_uri
+ execute_reinsert(ui_, repo, params, stored_cfg)
+
+
def infocalypse_pull(ui_, repo, **opts):
""" Pull from an Infocalypse repository in Freenet.
"""
@@ -257,6 +283,12 @@ cmdtable = {
+ FCP_OPTS,
"[options]"),
+ "fn-reinsert": (infocalypse_reinsert,
+ [('', 'requesturi', '', 'request URI to re-insert data from'),
+ ('', 'inserturi', '', 'insert URI (required to re-insert the top key)'), ]
+ + FCP_OPTS,
+ "[options]"),
+
"fn-setup": (infocalypse_setup,
[('', 'tmpdir', '~/infocalypse_tmp', 'temp directory'),]
+ FCP_OPTS,
diff --git a/infocalypse/graph.py b/infocalypse/graph.py
--- a/infocalypse/graph.py
+++ b/infocalypse/graph.py
@@ -328,10 +328,16 @@ class UpdateGraph:
if not edge_list[ordinal + 1].startswith(PENDING_INSERT):
print "set_chk -- replacing a non pending chk (%i, %i, %i)?" % \
(index_pair[0], index_pair[1], ordinal)
+ if edge_list[ordinal + 1] == chk:
+ print "Values are same."
+ else:
+ print "Values are different:"
+ print "old:", edge_list[ordinal + 1]
+ print "new:", chk
edge_list[ordinal + 1] = chk
self.edge_table[index_pair] = tuple(edge_list)
- def insert_type(self, edge_triple):
+ def insert_type_(self, edge_triple):
""" Return the kind of insert required to insert the CHK
for the edge.
@@ -349,6 +355,34 @@ class UpdateGraph:
return INSERT_PADDED
return INSERT_SALTED_METADATA
+ def insert_type(self, edge_triple):
+ """ Return the kind of insert required to insert the CHK
+ for the edge.
+
+ INSERT_NORMAL -> No modification to the bundle file.
+ INSERT_PADDED -> Add one trailing pad byte.
+ INSERT_SALTED_METADATA -> Copy and salt the Freenet
+ split file metadata for the normal insert. """
+
+ if edge_triple[2] == 0:
+ return INSERT_NORMAL
+
+ assert edge_triple[2] == 1
+
+ length = self.edge_table[edge_triple[:2]][0]
+
+ # REDFLAG: DCI. MUST DEAL WITH ==32k case
+ if length <= FREENET_BLOCK_LEN:
+ # Made redundant path by padding.
+ return INSERT_PADDED
+
+ if length <= MAX_METADATA_HACK_LEN:
+ return INSERT_SALTED_METADATA
+
+ print "insert_type called for edge that's too big to salt???"
+ print edge_triple
+ assert False
+
def insert_length(self, step):
""" Returns the actual length of the data inserted into
Freenet for the edge. """
diff --git a/infocalypse/infcmds.py b/infocalypse/infcmds.py
--- a/infocalypse/infcmds.py
+++ b/infocalypse/infcmds.py
@@ -471,6 +471,38 @@ def execute_copy(ui_, repo, params, stor
finally:
cleanup(update_sm)
+def execute_reinsert(ui_, repo, params, stored_cfg):
+ """ Run the reinsert command. """
+ update_sm = None
+ try:
+ update_sm = setup(ui_, repo, params, stored_cfg)
+ request_uri, is_keypair = handle_key_inversion(ui_, update_sm,
+ params, stored_cfg)
+ params['REQUEST_URI'] = request_uri
+
+ if not params['INSERT_URI'] is None:
+ ui_.status("%sInsert URI:\n%s\n" % (is_redundant(params['INSERT_URI']),
+ params['INSERT_URI']))
+ ui_.status("%sRequest URI:\n%s\n" % (is_redundant(params['REQUEST_URI']),
+ params['REQUEST_URI']))
+
+ update_sm.start_reinserting(params['REQUEST_URI'],
+ params['INSERT_URI'],
+ is_keypair)
+
+ run_until_quiescent(update_sm, params['POLL_SECS'])
+
+ if update_sm.get_state(QUIESCENT).arrived_from(((FINISHING,))):
+ ui_.status("Reinsert finished.\n")
+ else:
+ ui_.status("Reinsert failed.\n")
+
+ # Don't need to update the config.
+ finally:
+ cleanup(update_sm)
+
+
+
# REDFLAG: move into fcpclient?
#def usks_equal(usk_a, usk_b):
# assert is_usk(usk_a) and and is_usk(usk_b)
diff --git a/infocalypse/insertingbundles.py b/infocalypse/insertingbundles.py
--- a/infocalypse/insertingbundles.py
+++ b/infocalypse/insertingbundles.py
@@ -57,7 +57,8 @@ class InsertingBundles(RequestQueueState
"""
#require_state(from_state, QUIESCENT)
- assert not self.parent.ctx['INSERT_URI'] is None
+ assert (self.parent.ctx.get('REINSERT', 0) > 0 or
+ (not self.parent.ctx['INSERT_URI'] is None))
assert not self.parent.ctx.graph is None
graph = self.parent.ctx.graph.clone()
@@ -67,10 +68,7 @@ class InsertingBundles(RequestQueueState
# Update graph.
try:
- self.new_edges = graph.update(self.parent.ctx.repo,
- self.parent.ctx.ui_,
- self.parent.ctx['TARGET_VERSION'],
- self.parent.ctx.bundle_cache)
+ self.set_new_edges(graph)
except UpToDate, err:
# REDFLAG: Later, add FORCE_INSERT parameter?
self.parent.ctx.ui_.warn(str(err) + '\n') # Hmmm
@@ -109,6 +107,7 @@ class InsertingBundles(RequestQueueState
for edge in self.required_edges:
# Will be re-added when the required metadata arrives.
self.new_edges.remove((edge[0], edge[1], 1))
+ print "REQUIRED_EDGES:", self.required_edges, self.new_edges
# REDFLAG: no longer needed?
def leave(self, dummy):
@@ -130,7 +129,15 @@ class InsertingBundles(RequestQueueState
if edge in self.pending:
# Already running.
continue
- if not self.parent.ctx.graph.has_chk(edge):
+
+ # We can't count on the graph when reinserting.
+ # Because the chks are already set.
+
+ #if not self.parent.ctx.graph.has_chk(edge):
+ # # Depends on an edge which hasn't been inserted yet.
+ # continue
+
+ if edge in self.new_edges:
# Depends on an edge which hasn't been inserted yet.
continue
@@ -196,3 +203,21 @@ class InsertingBundles(RequestQueueState
len(self.required_edges) == 0):
self.parent.transition(INSERTING_GRAPH)
+ def set_new_edges(self, graph):
+ """ INTERNAL: Set the list of new edges to insert. """
+ if self.parent.ctx.get('REINSERT', 0) == 0:
+ self.new_edges = graph.update(self.parent.ctx.repo,
+ self.parent.ctx.ui_,
+ self.parent.ctx['TARGET_VERSION'],
+ self.parent.ctx.bundle_cache)
+ return
+
+ # Hmmmm... later support different int values of REINSERT?
+ self.new_edges = graph.get_top_key_edges()
+ redundant = []
+ for edge in self.new_edges:
+ if graph.is_redundant(edge):
+ alternate_edge = (edge[0], edge[1], int(not edge[2]))
+ if not alternate_edge in self.new_edges:
+ redundant.append(alternate_edge)
+ self.new_edges += redundant
diff --git a/infocalypse/updatesm.py b/infocalypse/updatesm.py
--- a/infocalypse/updatesm.py
+++ b/infocalypse/updatesm.py
@@ -87,16 +87,17 @@ def make_search_uris(uri):
return (uri, '/'.join(fields))
# For insert
-def make_insert_uris(uri):
+def make_insert_uris(uri, increment=True):
""" Returns a possibly redundant insert uri tuple.
- NOTE: This increments the version by 1 if uri is a USK.
+ NOTE: This increments the version by 1 if uri is a USK
+ and increment is True.
"""
if uri == 'CHK@':
return (uri,)
assert is_usk_file(uri)
version = get_version(uri)
# REDFLAG: does index increment really belong here?
- return make_redundant_ssk(uri, version + 1)
+ return make_redundant_ssk(uri, version + int(bool(increment)))
def ssk_to_usk(ssk):
""" Convert an SSK for a file USK back into a file USK. """
@@ -511,10 +512,19 @@ class InsertingUri(StaticRequestList):
if not hasattr(from_state, 'get_top_key_tuple'):
raise Exception("Illegal Transition from: %s" % from_state.name)
+ if (self.parent.ctx['INSERT_URI'] is None
+ and self.parent.ctx.get('REINSERT', 0) > 0):
+ # Hmmmm... hackery to deal with reinsert w/o insert uri
+ self.parent.transition(self.success_state)
+ return
+
+ assert not self.parent.ctx['INSERT_URI'] is None
+
top_key_tuple = from_state.get_top_key_tuple()
salt = {0:0x00, 1:0xff} # grrr.... less code.
- insert_uris = make_insert_uris(self.parent.ctx['INSERT_URI'])
+ insert_uris = make_insert_uris(self.parent.ctx['INSERT_URI'],
+ self.parent.ctx.get('REINSERT', 0) < 1)
assert len(insert_uris) < 3
for index, uri in enumerate(insert_uris):
if self.parent.params.get('DUMP_URIS', False):
@@ -529,7 +539,8 @@ class InsertingUri(StaticRequestList):
if to_state.name == self.success_state:
# Hmmm... what about chks?
# Update the index in the insert_uri on success
- if is_usk(self.parent.ctx['INSERT_URI']):
+ if (self.parent.ctx.get('REINSERT', 0) < 1 and
+ is_usk(self.parent.ctx['INSERT_URI'])):
version = get_version(self.parent.ctx['INSERT_URI']) + 1
self.parent.ctx['INSERT_URI'] = (
get_usk_for_usk_version(self.parent.ctx['INSERT_URI'],
@@ -955,6 +966,17 @@ class UpdateStateMachine(RequestQueue, S
self.get_state(INVERTING_URI).insert_uri = insert_uri
self.transition(INVERTING_URI)
+ def start_reinserting(self, request_uri, insert_uri=None, is_keypair=False):
+ """ """
+ self.require_state(QUIESCENT)
+ self.reset()
+ self.ctx['REQUEST_URI'] = request_uri
+ self.ctx['INSERT_URI'] = insert_uri
+ self.ctx['IS_KEYPAIR'] = is_keypair
+ self.ctx['REINSERT'] = 1
+ # REDFLAG: add hack code to InsertingUri to handle reinsert w/o insert uri?
+ self.transition(REQUESTING_URI_4_INSERT)
+
# REDFLAG: UNTESTED
def cancel(self):
""" Start canceling the current operation. """