manifest: add storage-index strings to the json results

docs/frontends/webapi.txt

@@ -964,11 +964,12 @@ POST $DIRURL?t=start-manifest    (must add &ophandle=XYZ)
   by a space.
 
   If output=JSON is added to the queryargs, then the results will be a
-  JSON-formatted dictionary with four keys:
+  JSON-formatted dictionary with five keys:
 
    finished (bool): if False then you must reload the page until True
-   origin_si (str): the storage index of the starting point
+   origin_si (base32 str): the storage index of the starting point
    manifest: list of (path, cap) tuples, where path is a list of strings.
+   storage-index: list of (base32) storage index strings
    stats: a dictionary with the same keys as the t=deep-stats command
           (described below)
 

src/allmydata/dirnode.py

@@ -565,7 +565,7 @@ class DeepStats:
     def set_monitor(self, monitor):
         self.monitor = monitor
         monitor.origin_si = self.origin.get_storage_index()
-        monitor.set_status(self.stats)
+        monitor.set_status(self.get_results())
 
     def add_node(self, node, childpath):
         if IDirectoryNode.providedBy(node):
@@ -640,14 +640,20 @@ class ManifestWalker(DeepStats):
     def __init__(self, origin):
         DeepStats.__init__(self, origin)
         self.manifest = []
+        self.storage_index_strings = set()
 
     def add_node(self, node, path):
         self.manifest.append( (tuple(path), node.get_uri()) )
+        si = node.get_storage_index()
+        if si:
+            self.storage_index_strings.add(base32.b2a(si))
         return DeepStats.add_node(self, node, path)
 
-    def finish(self):
+    def get_results(self):
+        stats = DeepStats.get_results(self)
         return {"manifest": self.manifest,
-                "stats": self.get_results(),
+                "storage-index": self.storage_index_strings,
+                "stats": stats,
                 }
 
 

src/allmydata/interfaces.py

@@ -875,11 +875,18 @@ class IDirectoryNode(IMutableFilesystemNode):
         I also compute deep-stats as described below.
 
         I return a Monitor. The Monitor's results will be a dictionary with
-        two elements. The 'manifest' element is a list of (path, cap) tuples
-        for nodes (directories and files) reachable from this one. 'path'
-        will be a tuple of unicode strings. The origin dirnode will be
-        represented by an empty path tuple. The 'stats' element is a
-        dictionary, the same that is generated by start_deep_stats() below.
+        three elements:
+
+         res['manifest']: a list of (path, cap) tuples for all nodes
+                          (directories and files) reachable from this one.
+                          'path' will be a tuple of unicode strings. The
+                          origin dirnode will be represented by an empty path
+                          tuple.
+         res['storage-index']: a list of (base32) storage index strings,
+                               one for each reachable node. This is a set:
+                               duplicates have been removed.
+         res['stats']: a dictionary, the same that is generated by
+                       start_deep_stats() below.
 
         The Monitor will also have an .origin_si attribute with the (binary)
         storage index of the starting point.

src/allmydata/scripts/cli.py

@@ -200,6 +200,7 @@ class WebopenOptions(VDriveOptions):
 class ManifestOptions(VDriveOptions):
     optFlags = [
         ("storage-index", "s", "Only print storage index strings, not pathname+cap"),
+        ("raw", "r", "Display raw JSON data instead of parsed"),
         ]
     def parseArgs(self, where=''):
         self.where = where

src/allmydata/test/test_dirnode.py

@@ -27,6 +27,7 @@ class Marker:
             nodeuri = nodeuri.to_string()
         self.nodeuri = nodeuri
         si = hashutil.tagged_hash("tag1", nodeuri)[:16]
+        self.storage_index = si
         fp = hashutil.tagged_hash("tag2", nodeuri)
         self.verifieruri = uri.SSKVerifierURI(storage_index=si, fingerprint=fp)
     def get_uri(self):
@@ -35,6 +36,8 @@ class Marker:
         return self.nodeuri
     def get_verifier(self):
         return self.verifieruri
+    def get_storage_index(self):
+        return self.storage_index
 
     def check(self, monitor, verify=False):
         r = CheckerResults("", None)

src/allmydata/test/test_system.py

@@ -2399,6 +2399,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
                            "--node-directory", basedir,
                            self.root_uri])
         def _check((out,err)):
+            self.failUnlessEqual(err, "")
             lines = [l for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 5)
             caps = {}

src/allmydata/test/test_web.py

@@ -940,13 +940,17 @@ class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
         d.addCallback(_got_text)
         d.addCallback(getman, "JSON")
-        def _got_json(manifest):
-            data = manifest["manifest"]
+        def _got_json(res):
+            data = res["manifest"]
             got = {}
             for (path_list, cap) in data:
                 got[tuple(path_list)] = cap
             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
             self.failUnless((u"sub",u"baz.txt") in got)
+            self.failUnless("finished" in res)
+            self.failUnless("origin" in res)
+            self.failUnless("storage-index" in res)
+            self.failUnless("stats" in res)
         d.addCallback(_got_json)
         return d
 

src/allmydata/web/directory.py

@@ -732,6 +732,7 @@ class ManifestResults(rend.Page, ReloadMixin):
         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
         m = self.monitor
         status = {"manifest": m.get_status()["manifest"],
+                  "storage-index": list(m.get_status()["storage-index"]),
                   "stats": m.get_status()["stats"],
                   "finished": m.is_finished(),
                   "origin": base32.b2a(m.origin_si),