Merge pull request #895 from tahoe-lafs/3502.mutable-python-3-part-2

Port allmydata.mutable to Python 3, part 2/2

Fixes ticket:3502

src/allmydata/mutable/layout.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Omit dict so Python 3 changes don't leak into API callers on Python 2.
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
 from past.utils import old_div
 
 import struct
@@ -1744,7 +1756,7 @@ class MDMFSlotReadProxy(object):
 
 
     def _read(self, readvs, force_remote=False):
-        unsatisfiable = list(filter(lambda x: x[0] + x[1] > len(self._data), readvs))
+        unsatisfiable = [x for x in readvs if x[0] + x[1] > len(self._data)]
         # TODO: It's entirely possible to tweak this so that it just
         # fulfills the requests that it can, and not demand that all
         # requests are satisfiable before running it.

src/allmydata/mutable/repairer.py

@@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from zope.interface import implementer
 from twisted.internet import defer

src/allmydata/mutable/retrieve.py

@@ -1,4 +1,15 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Don't import bytes and str, to prevent API leakage
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min  # noqa: F401
 
 import time
 
@@ -749,9 +760,9 @@ class Retrieve(object):
 
         blockhashes = dict(enumerate(blockhashes))
         self.log("the reader gave me the following blockhashes: %s" % \
-                 blockhashes.keys())
+                 list(blockhashes.keys()))
         self.log("the reader gave me the following sharehashes: %s" % \
-                 sharehashes.keys())
+                 list(sharehashes.keys()))
         bht = self._block_hash_trees[reader.shnum]
 
         if bht.needed_hashes(segnum, include_leaf=True):
@@ -908,7 +919,7 @@ class Retrieve(object):
 
 
     def notify_server_corruption(self, server, shnum, reason):
-        if isinstance(reason, unicode):
+        if isinstance(reason, str):
             reason = reason.encode("utf-8")
         storage_server = server.get_storage_server()
         storage_server.advise_corrupt_share(

src/allmydata/mutable/servermap.py

@@ -1,5 +1,15 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
 
+from future.utils import PY2
+if PY2:
+    # Doesn't import str to prevent API leakage on Python 2
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
 from past.builtins import unicode
 
 import sys, time, copy
@@ -188,7 +198,7 @@ class ServerMap(object):
     def dump(self, out=sys.stdout):
         print("servermap:", file=out)
 
-        for ( (server, shnum), (verinfo, timestamp) ) in self._known_shares.items():
+        for ( (server, shnum), (verinfo, timestamp) ) in list(self._known_shares.items()):
             (seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
              offsets_tuple) = verinfo
             print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" %
@@ -226,7 +236,7 @@ class ServerMap(object):
         """Return a dict that maps versionid to sets of (shnum, server,
         timestamp) tuples."""
         versionmap = DictOfSets()
-        for ( (server, shnum), (verinfo, timestamp) ) in self._known_shares.items():
+        for ( (server, shnum), (verinfo, timestamp) ) in list(self._known_shares.items()):
             versionmap.add(verinfo, (shnum, server, timestamp))
         return versionmap
 
@@ -245,7 +255,7 @@ class ServerMap(object):
         (num_distinct_shares, k, N) tuples."""
         versionmap = self.make_versionmap()
         all_shares = {}
-        for verinfo, shares in versionmap.items():
+        for verinfo, shares in list(versionmap.items()):
             s = set()
             for (shnum, server, timestamp) in shares:
                 s.add(shnum)
@@ -271,7 +281,7 @@ class ServerMap(object):
         """Return a string describing which versions we know about."""
         versionmap = self.make_versionmap()
         bits = []
-        for (verinfo, shares) in versionmap.items():
+        for (verinfo, shares) in list(versionmap.items()):
             vstr = self.summarize_version(verinfo)
             shnums = set([shnum for (shnum, server, timestamp) in shares])
             bits.append("%d*%s" % (len(shnums), vstr))
@@ -282,7 +292,7 @@ class ServerMap(object):
         recoverable."""
         versionmap = self.make_versionmap()
         recoverable_versions = set()
-        for (verinfo, shares) in versionmap.items():
+        for (verinfo, shares) in list(versionmap.items()):
             (seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
              offsets_tuple) = verinfo
             shnums = set([shnum for (shnum, server, timestamp) in shares])
@@ -298,7 +308,7 @@ class ServerMap(object):
         versionmap = self.make_versionmap()
 
         unrecoverable_versions = set()
-        for (verinfo, shares) in versionmap.items():
+        for (verinfo, shares) in list(versionmap.items()):
             (seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
              offsets_tuple) = verinfo
             shnums = set([shnum for (shnum, server, timestamp) in shares])
@@ -332,7 +342,7 @@ class ServerMap(object):
         healths = {} # maps verinfo to (found,k)
         unrecoverable = set()
         highest_recoverable_seqnum = -1
-        for (verinfo, shares) in versionmap.items():
+        for (verinfo, shares) in list(versionmap.items()):
             (seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
              offsets_tuple) = verinfo
             shnums = set([shnum for (shnum, server, timestamp) in shares])
@@ -667,7 +677,7 @@ class ServermapUpdater(object):
 
         ds = []
 
-        for shnum,datav in datavs.items():
+        for shnum,datav in list(datavs.items()):
             data = datav[0]
             reader = MDMFSlotReadProxy(ss,
                                        storage_index,

src/allmydata/util/_python3.py

@@ -56,7 +56,11 @@ PORTED_MODULES = [
     "allmydata.mutable.checker",
     "allmydata.mutable.common",
     "allmydata.mutable.filenode",
+    "allmydata.mutable.layout",
     "allmydata.mutable.publish",
+    "allmydata.mutable.repairer",
+    "allmydata.mutable.retrieve",
+    "allmydata.mutable.servermap",
     "allmydata.node",
     "allmydata.storage_client",
     "allmydata.storage.common",