Merge branch '3607.web-python-3-part-1' into 3611.web-python-3-part-2

src/allmydata/dirnode.py

@@ -74,6 +74,13 @@ ADD_FILE = ActionType(
     u"Add a new file as a child of a directory.",
 )
 
+
+class _OnlyFiles(object):
+    """Marker for replacement option of only replacing files."""
+
+ONLY_FILES = _OnlyFiles()
+
+
 def update_metadata(metadata, new_metadata, now):
     """Updates 'metadata' in-place with the information in 'new_metadata'.
 
@@ -179,7 +186,7 @@ class Adder(object):
         if entries is None:
             entries = {}
         precondition(isinstance(entries, dict), entries)
-        precondition(overwrite in (True, False, "only-files"), overwrite)
+        precondition(overwrite in (True, False, ONLY_FILES), overwrite)
         # keys of 'entries' may not be normalized.
         self.entries = entries
         self.overwrite = overwrite
@@ -205,7 +212,7 @@ class Adder(object):
                 if not self.overwrite:
                     raise ExistingChildError("child %s already exists" % quote_output(name, encoding='utf-8'))
 
-                if self.overwrite == "only-files" and IDirectoryNode.providedBy(children[name][0]):
+                if self.overwrite == ONLY_FILES and IDirectoryNode.providedBy(children[name][0]):
                     raise ExistingChildError("child %s already exists as a directory" % quote_output(name, encoding='utf-8'))
                 metadata = children[name][1].copy()
 
@@ -701,7 +708,7 @@ class DirectoryNode(object):
         'new_child_namex' and 'current_child_namex' need not be normalized.
 
         The overwrite parameter may be True (overwrite any existing child),
-        False (error if the new child link already exists), or "only-files"
+        False (error if the new child link already exists), or ONLY_FILES
         (error if the new child link exists and points to a directory).
         """
         if self.is_readonly() or new_parent.is_readonly():

src/allmydata/test/test_dirnode.py

@@ -1978,12 +1978,12 @@ class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
                                 overwrite=False))
             d.addCallback(lambda res:
                 root_node.set_node(u'file1', filenode,
-                                   overwrite="only-files"))
+                                   overwrite=dirnode.ONLY_FILES))
             d.addCallback(lambda res:
                 self.shouldFail(ExistingChildError, "set_node",
                                 "child 'dir1' already exists",
                                 root_node.set_node, u'dir1', filenode,
-                                overwrite="only-files"))
+                                overwrite=dirnode.ONLY_FILES))
             return d
 
         d.addCallback(_test_adder)

src/allmydata/test/web/test_util.py

@@ -12,17 +12,18 @@ if PY2:
 
 from twisted.trial import unittest
 from allmydata.web import status, common
+from allmydata.dirnode import ONLY_FILES
 from ..common import ShouldFailMixin
 from .. import common_util as testutil
 
 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 
     def test_parse_replace_arg(self):
-        self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
+        self.failUnlessReallyEqual(common.parse_replace_arg(b"true"), True)
-        self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
+        self.failUnlessReallyEqual(common.parse_replace_arg(b"false"), False)
-        self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
+        self.failUnlessReallyEqual(common.parse_replace_arg(b"only-files"),
-                                   "only-files")
+                                   ONLY_FILES)
-        self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
+        self.failUnlessRaises(common.WebError, common.parse_replace_arg, b"only_fles")
 
     def test_abbreviate_time(self):
         self.failUnlessReallyEqual(common.abbreviate_time(None), "")

src/allmydata/util/_python3.py

@@ -115,6 +115,7 @@ PORTED_MODULES = [
     "allmydata.util.spans",
     "allmydata.util.statistics",
     "allmydata.util.time_format",
+    "allmydata.web.common",
     "allmydata.web.logs",
     "allmydata.webish",
 ]

src/allmydata/web/common.py

@@ -1,5 +1,22 @@
-from past.builtins import unicode
+"""
-from six import ensure_text, ensure_str
+Ported to Python 3.
+"""
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+    from past.builtins import unicode as str  # prevent leaking newbytes/newstr into code that can't handle it
+
+from six import ensure_str
+
+try:
+    from typing import Optional, Union, Tuple, Any
+except ImportError:
+    pass
 
 import time
 import json
@@ -51,6 +68,7 @@ from twisted.web.resource import (
     IResource,
 )
 
+from allmydata.dirnode import ONLY_FILES, _OnlyFiles
 from allmydata import blacklist
 from allmydata.interfaces import (
     EmptyPathnameComponentError,
@@ -74,11 +92,13 @@ from allmydata.util.encodingutil import (
     quote_output,
     to_bytes,
 )
+from allmydata.util import abbreviate
 
-# Originally part of this module, so still part of its API:
+
-from .common_py3 import (  # noqa: F401
+class WebError(Exception):
-    get_arg, abbreviate_time, MultiFormatResource, WebError,
+    def __init__(self, text, code=http.BAD_REQUEST):
-)
+        self.text = text
+        self.code = code
 
 
 def get_filenode_metadata(filenode):
@@ -98,17 +118,17 @@ def get_filenode_metadata(filenode):
         metadata['size'] = size
     return metadata
 
-def boolean_of_arg(arg):
+def boolean_of_arg(arg):  # type: (bytes) -> bool
-    # TODO: ""
+    assert isinstance(arg, bytes)
-    arg = ensure_text(arg)
+    if arg.lower() not in (b"true", b"t", b"1", b"false", b"f", b"0", b"on", b"off"):
-    if arg.lower() not in ("true", "t", "1", "false", "f", "0", "on", "off"):
         raise WebError("invalid boolean argument: %r" % (arg,), http.BAD_REQUEST)
-    return arg.lower() in ("true", "t", "1", "on")
+    return arg.lower() in (b"true", b"t", b"1", b"on")
 
-def parse_replace_arg(replace):
+
-    replace = ensure_text(replace)
+def parse_replace_arg(replace):  # type: (bytes) -> Union[bool,_OnlyFiles]
-    if replace.lower() == "only-files":
+    assert isinstance(replace, bytes)
-        return replace
+    if replace.lower() == b"only-files":
+        return ONLY_FILES
     try:
         return boolean_of_arg(replace)
     except WebError:
@@ -145,19 +165,19 @@ def get_mutable_type(file_format): # accepts result of get_format()
         return None
 
 
-def parse_offset_arg(offset):
+def parse_offset_arg(offset):  # type: (bytes) -> Union[int,None]
     # XXX: This will raise a ValueError when invoked on something that
     # is not an integer. Is that okay? Or do we want a better error
     # message? Since this call is going to be used by programmers and
     # their tools rather than users (through the wui), it is not
     # inconsistent to return that, I guess.
     if offset is not None:
-        offset = int(offset)
+        return int(offset)
 
     return offset
 
 
-def get_root(req):
+def get_root(req):  # type: (IRequest) -> str
     """
     Get a relative path with parent directory segments that refers to the root
     location known to the given request.  This seems a lot like the constant
@@ -186,8 +206,8 @@ def convert_children_json(nodemaker, children_json):
     children = {}
     if children_json:
         data = json.loads(children_json)
-        for (namex, (ctype, propdict)) in data.items():
+        for (namex, (ctype, propdict)) in list(data.items()):
-            namex = unicode(namex)
+            namex = str(namex)
             writecap = to_bytes(propdict.get("rw_uri"))
             readcap = to_bytes(propdict.get("ro_uri"))
             metadata = propdict.get("metadata", {})
@@ -208,7 +228,8 @@ def compute_rate(bytes, seconds):
     assert bytes > -1
     assert seconds > 0
 
-    return 1.0 * bytes / seconds
+    return bytes / seconds
+
 
 def abbreviate_rate(data):
     """
@@ -229,6 +250,7 @@ def abbreviate_rate(data):
         return u"%.1fkBps" % (r/1000)
     return u"%.0fBps" % r
 
+
 def abbreviate_size(data):
     """
     Convert number of bytes into human readable strings (unicode).
@@ -265,7 +287,7 @@ def text_plain(text, req):
     return text
 
 def spaces_to_nbsp(text):
-    return unicode(text).replace(u' ', u'\u00A0')
+    return str(text).replace(u' ', u'\u00A0')
 
 def render_time_delta(time_1, time_2):
     return spaces_to_nbsp(format_delta(time_1, time_2))
@@ -283,7 +305,7 @@ def render_time_attr(t):
 # actual exception). The latter is growing increasingly annoying.
 
 def should_create_intermediate_directories(req):
-    t = unicode(get_arg(req, "t", "").strip(), "ascii")
+    t = str(get_arg(req, "t", "").strip(), "ascii")
     return bool(req.method in (b"PUT", b"POST") and
                 t not in ("delete", "rename", "rename-form", "check"))
 
@@ -565,7 +587,7 @@ def _finish(result, render, request):
             resource=fullyQualifiedName(type(result)),
         )
         result.render(request)
-    elif isinstance(result, unicode):
+    elif isinstance(result, str):
         Message.log(
             message_type=u"allmydata:web:common-render:unicode",
         )
@@ -647,7 +669,7 @@ def _renderHTTP_exception(request, failure):
 def _renderHTTP_exception_simple(request, text, code):
     request.setResponseCode(code)
     request.setHeader("content-type", "text/plain;charset=utf-8")
-    if isinstance(text, unicode):
+    if isinstance(text, str):
         text = text.encode("utf-8")
     request.setHeader("content-length", b"%d" % len(text))
     return text
@@ -689,3 +711,124 @@ def url_for_string(req, url_string):
             port=port,
         )
     return url
+
+
+def get_arg(req, argname, default=None, multiple=False):  # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any]
+    """Extract an argument from either the query args (req.args) or the form
+    body fields (req.fields). If multiple=False, this returns a single value
+    (or the default, which defaults to None), and the query args take
+    precedence. If multiple=True, this returns a tuple of arguments (possibly
+    empty), starting with all those in the query args.
+
+    :param TahoeLAFSRequest req: The request to consider.
+
+    :return: Either bytes or tuple of bytes.
+    """
+    if isinstance(argname, str):
+        argname = argname.encode("utf-8")
+    if isinstance(default, str):
+        default = default.encode("utf-8")
+    results = []
+    if argname in req.args:
+        results.extend(req.args[argname])
+    argname_unicode = str(argname, "utf-8")
+    if req.fields and argname_unicode in req.fields:
+        value = req.fields[argname_unicode].value
+        if isinstance(value, str):
+            value = value.encode("utf-8")
+        results.append(value)
+    if multiple:
+        return tuple(results)
+    if results:
+        return results[0]
+    return default
+
+
+class MultiFormatResource(resource.Resource, object):
+    """
+    ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
+    a number of different formats.
+
+    Rendered format is controlled by a query argument (given by
+    ``self.formatArgument``).  Different resources may support different
+    formats but ``json`` is a pretty common one.  ``html`` is the default
+    format if nothing else is given as the ``formatDefault``.
+    """
+    formatArgument = "t"
+    formatDefault = None  # type: Optional[str]
+
+    def render(self, req):
+        """
+        Dispatch to a renderer for a particular format, as selected by a query
+        argument.
+
+        A renderer for the format given by the query argument matching
+        ``formatArgument`` will be selected and invoked.  render_HTML will be
+        used as a default if no format is selected (either by query arguments
+        or by ``formatDefault``).
+
+        :return: The result of the selected renderer.
+        """
+        t = get_arg(req, self.formatArgument, self.formatDefault)
+        # It's either bytes or None.
+        if isinstance(t, bytes):
+            t = str(t, "ascii")
+        renderer = self._get_renderer(t)
+        result = renderer(req)
+        # On Python 3, json.dumps() returns Unicode for example, but
+        # twisted.web expects bytes. Instead of updating every single render
+        # method, just handle Unicode one time here.
+        if isinstance(result, str):
+            result = result.encode("utf-8")
+        return result
+
+    def _get_renderer(self, fmt):
+        """
+        Get the renderer for the indicated format.
+
+        :param str fmt: The format.  If a method with a prefix of ``render_``
+            and a suffix of this format (upper-cased) is found, it will be
+            used.
+
+        :return: A callable which takes a twisted.web Request and renders a
+            response.
+        """
+        renderer = None
+
+        if fmt is not None:
+            try:
+                renderer = getattr(self, "render_{}".format(fmt.upper()))
+            except AttributeError:
+                return resource.ErrorPage(
+                    http.BAD_REQUEST,
+                    "Bad Format",
+                    "Unknown {} value: {!r}".format(self.formatArgument, fmt),
+                ).render
+
+        if renderer is None:
+            renderer = self.render_HTML
+
+        return renderer
+
+
+def abbreviate_time(data):
+    """
+    Convert number of seconds into human readable string.
+
+    :param data: Either ``None`` or integer or float, seconds.
+
+    :return: Unicode string.
+    """
+    # 1.23s, 790ms, 132us
+    if data is None:
+        return u""
+    s = float(data)
+    if s >= 10:
+        return abbreviate.abbreviate_time(data)
+    if s >= 1.0:
+        return u"%.2fs" % s
+    if s >= 0.01:
+        return u"%.0fms" % (1000*s)
+    if s >= 0.001:
+        return u"%.1fms" % (1000*s)
+    return u"%.0fus" % (1000000*s)

src/allmydata/web/common_py3.py

@@ -1,143 +0,0 @@
-"""
-Common utilities that are available from Python 3.
-
-Can eventually be merged back into allmydata.web.common.
-"""
-
-from past.builtins import unicode
-
-try:
-    from typing import Optional
-except ImportError:
-    pass
-
-from twisted.web import resource, http
-
-from allmydata.util import abbreviate
-
-
-class WebError(Exception):
-    def __init__(self, text, code=http.BAD_REQUEST):
-        self.text = text
-        self.code = code
-
-
-def get_arg(req, argname, default=None, multiple=False):
-    """Extract an argument from either the query args (req.args) or the form
-    body fields (req.fields). If multiple=False, this returns a single value
-    (or the default, which defaults to None), and the query args take
-    precedence. If multiple=True, this returns a tuple of arguments (possibly
-    empty), starting with all those in the query args.
-
-    :param TahoeLAFSRequest req: The request to consider.
-
-    :return: Either bytes or tuple of bytes.
-    """
-    if isinstance(argname, unicode):
-        argname = argname.encode("utf-8")
-    if isinstance(default, unicode):
-        default = default.encode("utf-8")
-    results = []
-    if argname in req.args:
-        results.extend(req.args[argname])
-    argname_unicode = unicode(argname, "utf-8")
-    if req.fields and argname_unicode in req.fields:
-        value = req.fields[argname_unicode].value
-        if isinstance(value, unicode):
-            value = value.encode("utf-8")
-        results.append(value)
-    if multiple:
-        return tuple(results)
-    if results:
-        return results[0]
-    return default
-
-
-class MultiFormatResource(resource.Resource, object):
-    """
-    ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
-    a number of different formats.
-
-    Rendered format is controlled by a query argument (given by
-    ``self.formatArgument``).  Different resources may support different
-    formats but ``json`` is a pretty common one.  ``html`` is the default
-    format if nothing else is given as the ``formatDefault``.
-    """
-    formatArgument = "t"
-    formatDefault = None  # type: Optional[str]
-
-    def render(self, req):
-        """
-        Dispatch to a renderer for a particular format, as selected by a query
-        argument.
-
-        A renderer for the format given by the query argument matching
-        ``formatArgument`` will be selected and invoked.  render_HTML will be
-        used as a default if no format is selected (either by query arguments
-        or by ``formatDefault``).
-
-        :return: The result of the selected renderer.
-        """
-        t = get_arg(req, self.formatArgument, self.formatDefault)
-        # It's either bytes or None.
-        if isinstance(t, bytes):
-            t = unicode(t, "ascii")
-        renderer = self._get_renderer(t)
-        result = renderer(req)
-        # On Python 3, json.dumps() returns Unicode for example, but
-        # twisted.web expects bytes. Instead of updating every single render
-        # method, just handle Unicode one time here.
-        if isinstance(result, unicode):
-            result = result.encode("utf-8")
-        return result
-
-    def _get_renderer(self, fmt):
-        """
-        Get the renderer for the indicated format.
-
-        :param str fmt: The format.  If a method with a prefix of ``render_``
-            and a suffix of this format (upper-cased) is found, it will be
-            used.
-
-        :return: A callable which takes a twisted.web Request and renders a
-            response.
-        """
-        renderer = None
-
-        if fmt is not None:
-            try:
-                renderer = getattr(self, "render_{}".format(fmt.upper()))
-            except AttributeError:
-                return resource.ErrorPage(
-                    http.BAD_REQUEST,
-                    "Bad Format",
-                    "Unknown {} value: {!r}".format(self.formatArgument, fmt),
-                ).render
-
-        if renderer is None:
-            renderer = self.render_HTML
-
-        return renderer
-
-
-def abbreviate_time(data):
-    """
-    Convert number of seconds into human readable string.
-
-    :param data: Either ``None`` or integer or float, seconds.
-
-    :return: Unicode string.
-    """
-    # 1.23s, 790ms, 132us
-    if data is None:
-        return u""
-    s = float(data)
-    if s >= 10:
-        return abbreviate.abbreviate_time(data)
-    if s >= 1.0:
-        return u"%.2fs" % s
-    if s >= 0.01:
-        return u"%.0fms" % (1000*s)
-    if s >= 0.001:
-        return u"%.1fms" % (1000*s)
-    return u"%.0fus" % (1000000*s)

src/allmydata/web/storage.py

@@ -9,7 +9,7 @@ from twisted.web.template import (
     renderer,
     renderElement
 )
-from allmydata.web.common_py3 import (
+from allmydata.web.common import (
     abbreviate_time,
     MultiFormatResource
 )