Merge remote-tracking branch 'origin/master' into 3618.bb-tests-python-3

misc/operations_helpers/provisioning/run.py

@@ -30,7 +30,7 @@ class Root(rend.Page):
 
 def run(portnum):
     root = Root()
-    root.putChild("tahoe.css", static.File("tahoe.css"))
+    root.putChild(b"tahoe.css", static.File("tahoe.css"))
     site = appserver.NevowSite(root)
     s = strports.service("tcp:%d" % portnum, site)
     s.startService()

src/allmydata/check_results.py

@@ -1,3 +1,14 @@
+"""Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from past.builtins import unicode
 
 from zope.interface import implementer

src/allmydata/util/_python3.py

@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata._auto_deps",
     "allmydata._monkeypatch",
     "allmydata.blacklist",
+    "allmydata.check_results",
     "allmydata.codec",
     "allmydata.control",
     "allmydata.crypto",
@@ -117,12 +118,16 @@ PORTED_MODULES = [
     "allmydata.util.spans",
     "allmydata.util.statistics",
     "allmydata.util.time_format",
-    "allmydata.web.common",
     "allmydata.web.check_results",
+    "allmydata.web.common",
+    "allmydata.web.directory",
     "allmydata.web.filenode",
     "allmydata.web.info",
     "allmydata.web.introweb",
     "allmydata.web.logs",
+    "allmydata.web.operations",
+    "allmydata.web.private",
+    "allmydata.web.root",
     "allmydata.webish",
 ]
 

src/allmydata/util/netstring.py

@@ -14,14 +14,19 @@ if PY2:
 
 from past.builtins import long
 
+try:
+    from typing import Optional, Tuple, List  # noqa: F401
+except ImportError:
+    pass
 
-def netstring(s):
+
+def netstring(s):  # type: (bytes) -> bytes
     assert isinstance(s, bytes), s # no unicode here
     return b"%d:%s," % (len(s), s,)
 
 def split_netstring(data, numstrings,
                     position=0,
-                    required_trailer=None):
+                    required_trailer=None):  # type (bytes, init, int, Optional[bytes]) -> Tuple[List[bytes], int]
     """like string.split(), but extracts netstrings. Ignore all bytes of data
     before the 'position' byte. Return a tuple of (list of elements (numstrings
     in length), new position index). The new position index points to the first

src/allmydata/web/directory.py

@@ -1,12 +1,16 @@
 """
-TODO: When porting to Python 3, the filename handling logic seems wrong.  On
+Ported to Python 3.
-Python 3 filename will _already_ be correctly decoded.  So only decode if it's
-bytes.
-
-Also there's a lot of code duplication I think.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
-from past.builtins import unicode
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+    # Don't use Future's str so that we don't get leaks into bad byte formatting
+    from past.builtins import unicode as str
 
 from urllib.parse import quote as url_quote
 from datetime import timedelta
@@ -143,7 +147,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name
         nonterminal = not terminal  #len(req.postpath) > 0
 
-        t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
+        t = str(get_arg(req, b"t", b"").strip(), "ascii")
         if isinstance(node_or_failure, Failure):
             f = node_or_failure
             f.trap(NoSuchChildError)
@@ -225,7 +229,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
     @render_exception
     def render_GET(self, req):
         # This is where all of the directory-related ?t=* code goes.
-        t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
+        t = str(get_arg(req, b"t", b"").strip(), "ascii")
 
         # t=info contains variable ophandles, t=rename-form contains the name
         # of the child being renamed. Neither is allowed an ETag.
@@ -263,7 +267,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
 
     @render_exception
     def render_PUT(self, req):
-        t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
+        t = str(get_arg(req, b"t", b"").strip(), "ascii")
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
 
         if t == "mkdir":
@@ -283,7 +287,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
 
     @render_exception
     def render_POST(self, req):
-        t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
+        t = str(get_arg(req, b"t", b"").strip(), "ascii")
 
         if t == "mkdir":
             d = self._POST_mkdir(req)
@@ -372,11 +376,17 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         return d
 
     def _POST_upload(self, req):
-        charset = unicode(get_arg(req, "_charset", b"utf-8"), "utf-8")
+        charset = str(get_arg(req, "_charset", b"utf-8"), "utf-8")
         contents = req.fields["file"]
-        assert contents.filename is None or isinstance(contents.filename, str)
+
-        name = get_arg(req, "name")
+        # The filename embedded in the MIME file upload will be bytes on Python
-        name = name or contents.filename
+        # 2, Unicode on Python 3, or missing (i.e. None). The "name" field in
+        # the upload will be bytes on Python 2, Unicode on Python 3, or missing
+        # (i.e. None). We go through all these variations until we have a name
+        # that is Unicode.
+        assert contents.filename is None or isinstance(contents.filename, (bytes, str))
+        name = get_arg(req, "name")  # returns bytes or None
+        name = name or contents.filename  # unicode, bytes or None
         if name is not None:
             name = name.strip()
         if not name:
@@ -384,9 +394,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
             raise WebError("upload requires a name")
         if isinstance(name, bytes):
             name = name.decode(charset)
+        assert isinstance(name, str)
         if "/" in name:
             raise WebError("name= may not contain a slash", http.BAD_REQUEST)
-        assert isinstance(name, unicode)
 
         # since POST /uri/path/file?t=upload is equivalent to
         # POST /uri/path/dir?t=upload&name=foo, just do the same thing that
@@ -421,7 +431,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         name = get_arg(req, "name")
         if not name:
             raise WebError("set-uri requires a name")
-        charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
+        charset = str(get_arg(req, "_charset", b"utf-8"), "ascii")
         name = name.decode(charset)
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
 
@@ -445,7 +455,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
             # without a name= field. For our own HTML this isn't a big
             # deal, because we create the 'unlink' POST buttons ourselves.
             name = b''
-        charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
+        charset = str(get_arg(req, "_charset", b"utf-8"), "ascii")
         name = name.decode(charset)
         d = self.node.delete(name)
         d.addCallback(lambda res: "thing unlinked")
@@ -461,14 +471,14 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         return self._POST_relink(req)
 
     def _POST_relink(self, req):
-        charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
+        charset = str(get_arg(req, "_charset", b"utf-8"), "ascii")
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
 
         from_name = get_arg(req, "from_name")
         if from_name is not None:
             from_name = from_name.strip()
             from_name = from_name.decode(charset)
-            assert isinstance(from_name, unicode)
+            assert isinstance(from_name, str)
         else:
             raise WebError("from_name= is required")
 
@@ -476,7 +486,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         if to_name is not None:
             to_name = to_name.strip()
             to_name = to_name.decode(charset)
-            assert isinstance(to_name, unicode)
+            assert isinstance(to_name, str)
         else:
             to_name = from_name
 
@@ -493,7 +503,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         if to_dir is not None and to_dir != self.node.get_write_uri():
             to_dir = to_dir.strip()
             to_dir = to_dir.decode(charset)
-            assert isinstance(to_dir, unicode)
+            assert isinstance(to_dir, str)
             to_path = to_dir.split(u"/")
             to_root = self.client.nodemaker.create_from_cap(to_bytes(to_path[0]))
             if not IDirectoryNode.providedBy(to_root):
@@ -632,8 +642,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
             # TODO test handling of bad JSON
             raise
         cs = {}
-        for name, (file_or_dir, mddict) in children.items():
+        for name, (file_or_dir, mddict) in list(children.items()):
-            name = unicode(name) # json returns str *or* unicode
+            name = str(name) # json returns str *or* unicode
             writecap = mddict.get('rw_uri')
             if writecap is not None:
                 writecap = writecap.encode("utf-8")
@@ -705,7 +715,7 @@ class DirectoryAsHTML(Element):
 
     @renderer
     def title(self, req, tag):
-        si_s = unicode(abbreviated_dirnode(self.node), "utf-8")
+        si_s = str(abbreviated_dirnode(self.node), "utf-8")
         header = ["Tahoe-LAFS - Directory SI=%s" % si_s]
         if self.node.is_unknown():
             header.append(" (unknown)")
@@ -719,7 +729,7 @@ class DirectoryAsHTML(Element):
 
     @renderer
     def header(self, req, tag):
-        si_s = unicode(abbreviated_dirnode(self.node), "utf-8")
+        si_s = str(abbreviated_dirnode(self.node), "utf-8")
         header = ["Tahoe-LAFS Directory SI=", tags.span(si_s, class_="data-chars")]
         if self.node.is_unknown():
             header.append(" (unknown)")
@@ -1013,7 +1023,7 @@ def _directory_json_metadata(req, dirnode):
     d = dirnode.list()
     def _got(children):
         kids = {}
-        for name, (childnode, metadata) in children.items():
+        for name, (childnode, metadata) in list(children.items()):
             assert IFilesystemNode.providedBy(childnode), childnode
             rw_uri = childnode.get_write_uri()
             ro_uri = childnode.get_readonly_uri()
@@ -1077,13 +1087,13 @@ class RenameForm(Element, object):
 
     @renderer
     def title(self, req, tag):
-        return tag("Directory SI={}".format(unicode(abbreviated_dirnode(self.original), "ascii")))
+        return tag("Directory SI={}".format(str(abbreviated_dirnode(self.original), "ascii")))
 
     @renderer
     def header(self, req, tag):
         header = [
             "Rename "
-            "in directory SI=%s" % unicode(abbreviated_dirnode(self.original), "ascii"),
+            "in directory SI=%s" % str(abbreviated_dirnode(self.original), "ascii"),
         ]
 
         if self.original.is_readonly():
@@ -1194,7 +1204,7 @@ class ManifestElement(ReloadableMonitorElement):
         si = self.monitor.origin_si
         if not si:
             return "<LIT>"
-        return unicode(base32.b2a(si)[:6], "utf-8")
+        return str(base32.b2a(si)[:6], "utf-8")
 
     @renderer
     def title(self, req, tag):
@@ -1472,7 +1482,7 @@ class UnknownNodeHandler(Resource, object):
 
     @render_exception
     def render_GET(self, req):
-        t = unicode(get_arg(req, "t", "").strip(), "ascii")
+        t = str(get_arg(req, "t", "").strip(), "ascii")
         if t == "info":
             return MoreInfo(self.node)
         if t == "json":

src/allmydata/web/operations.py

@@ -1,4 +1,14 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 from hyperlink import (
@@ -91,7 +101,7 @@ class OphandleTable(resource.Resource, service.Service):
         """
         ophandle = get_arg(req, "ophandle").decode("utf-8")
         assert ophandle
-        here = DecodedURL.from_text(unicode(URLPath.fromRequest(req)))
+        here = DecodedURL.from_text(str(URLPath.fromRequest(req)))
         target = here.click(u"/").child(u"operations", ophandle)
         output = get_arg(req, "output")
         if output:
@@ -102,7 +112,7 @@ class OphandleTable(resource.Resource, service.Service):
     def getChild(self, name, req):
         ophandle = name
         if ophandle not in self.handles:
-            raise WebError("unknown/expired handle '%s'" % escape(unicode(ophandle, "utf-8")),
+            raise WebError("unknown/expired handle '%s'" % escape(str(ophandle, "utf-8")),
                            NOT_FOUND)
         (monitor, renderer, when_added) = self.handles[ophandle]
 

src/allmydata/web/private.py

@@ -1,10 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
-from __future__ import (
+from future.utils import PY2
-    print_function,
+if PY2:
-    unicode_literals,
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-    absolute_import,
-    division,
-)
 
 import attr
 

src/allmydata/web/root.py

@@ -1,5 +1,14 @@
-from future.utils import PY3
+"""
-from past.builtins import unicode
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, PY3
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 import time
@@ -98,7 +107,7 @@ class URIHandler(resource.Resource, object):
         either "PUT /uri" to create an unlinked file, or
         "PUT /uri?t=mkdir" to create an unlinked directory
         """
-        t = unicode(get_arg(req, "t", "").strip(), "utf-8")
+        t = str(get_arg(req, "t", "").strip(), "utf-8")
         if t == "":
             file_format = get_format(req, "CHK")
             mutable_type = get_mutable_type(file_format)
@@ -121,7 +130,7 @@ class URIHandler(resource.Resource, object):
         unlinked file or "POST /uri?t=mkdir" to create a
         new directory
         """
-        t = unicode(get_arg(req, "t", "").strip(), "ascii")
+        t = str(get_arg(req, "t", "").strip(), "ascii")
         if t in ("", "upload"):
             file_format = get_format(req)
             mutable_type = get_mutable_type(file_format)