Merge branch 'master' into 3459.test-checker-python-3

Makefile

@@ -51,6 +51,7 @@ test: .tox/create-venvs.log
 ## Run all tests with coverage collection and reporting.
 test-venv-coverage:
 # Special handling for reporting coverage even when the test run fails
+	rm -f ./.coverage.*
 	test_exit=
 	$(VIRTUAL_ENV)/bin/coverage run -m twisted.trial --rterrors --reporter=timing \
 		$(TEST_SUITE) || test_exit="$$?"

misc/python3/Makefile

@@ -1,4 +1,4 @@
-# BBB: Python 3 porting targets
+# Python 3 porting targets
 #
 # NOTE: this Makefile requires GNU make
 

newsfragments/3455.minor

@@ -0,0 +1 @@
+Begin porting the `node` module to Python 3.

newsfragments/3464.minor

@@ -0,0 +1 @@
+Cleanup comments that don't match the project convention.

src/allmydata/__init__.py

@@ -42,3 +42,9 @@ __full_version__ = __appname__ + '/' + str(__version__)
 # Install Python 3 module locations in Python 2:
 from future import standard_library
 standard_library.install_aliases()
+
+
+# Monkey-patch 3rd party libraries:
+from ._monkeypatch import patch
+patch()
+del patch

src/allmydata/_monkeypatch.py

@@ -0,0 +1,48 @@
+"""
+Monkey-patching of third party libraries.
+
+Ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from warnings import catch_warnings
+
+
+def patch():
+    """Path third-party libraries to make Tahoe-LAFS work."""
+    # Make sure Foolscap always get native strings passed to method names in callRemote.
+    # This can be removed when any one of the following happens:
+    #
+    # 1. Tahoe-LAFS on Python 2 switches to version of Foolscap that fixes
+    #    https://github.com/warner/foolscap/issues/72
+    # 2. Foolscap is dropped as a dependency.
+    # 3. Tahoe-LAFS drops Python 2 support.
+
+    if not PY2:
+        # Python 3 doesn't need to monkey patch Foolscap
+        return
+
+    # We need to suppress warnings so as to prevent unexpected output from
+    # breaking some integration tests.
+    with catch_warnings(record=True):
+        # Only tested with this version; ensure correctness with new releases,
+        # and then either update the assert or hopefully drop the monkeypatch.
+        from foolscap import __version__
+        assert __version__ == "0.13.1", "Wrong version %s of Foolscap" % (__version__,)
+
+        from foolscap.referenceable import RemoteReference
+        original_getMethodInfo = RemoteReference._getMethodInfo
+
+        def _getMethodInfo(self, name):
+            if isinstance(name, str):
+                name = name.encode("utf-8")
+            return original_getMethodInfo(self, name)
+        RemoteReference._getMethodInfo = _getMethodInfo

src/allmydata/client.py

@@ -69,8 +69,8 @@ def _is_valid_section(section_name):
     Currently considers all possible storage server plugin sections valid.
     """
     return (
-        section_name.startswith(b"storageserver.plugins.") or
+        section_name.startswith("storageserver.plugins.") or
-        section_name.startswith(b"storageclient.plugins.")
+        section_name.startswith("storageclient.plugins.")
     )
 
 

src/allmydata/immutable/downloader/share.py

@@ -6,7 +6,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2, native_str
+from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
@@ -765,9 +765,7 @@ class Share(object):
                                  level=log.WEIRD, umid="qZu0wg"))
 
     def _send_request(self, start, length):
-        # For some reason tests fail on Python 2 if this is not a native
+        return self._rref.callRemote("read", start, length)
-        # string...
-        return self._rref.callRemote(native_str("read"), start, length)
 
     def _got_data(self, data, start, length, block_ev, lp):
         block_ev.finished(len(data), now())

src/allmydata/immutable/encode.py

@@ -1,5 +1,18 @@
 # -*- test-case-name: allmydata.test.test_encode -*-
 
+"""
+Ported to Python 3.
+"""
+
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 from zope.interface import implementer
 from twisted.internet import defer
@@ -468,7 +481,7 @@ class Encoder(object):
                      (self,
                       self.segment_size*(segnum+1),
                       self.segment_size*self.num_segments,
-                      100 * (segnum+1) / self.num_segments,
+                      100 * (segnum+1) // self.num_segments,
                       ),
                      level=log.OPERATIONAL)
             elapsed = time.time() - start

src/allmydata/immutable/filenode.py

@@ -1,3 +1,16 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from functools import reduce
 import binascii
 from time import time as now

src/allmydata/immutable/layout.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import struct
 from zope.interface import implementer
 from twisted.internet import defer

src/allmydata/immutable/upload.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, native_str
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long, unicode
 
 import os, time, weakref, itertools
@@ -66,7 +78,7 @@ def _serialize_existing_shares(existing_shares):
     return {
         server: list(shares)
         for (server, shares)
-        in existing_shares.iteritems()
+        in existing_shares.items()
     }
 
 _EXISTING_SHARES = Field(
@@ -79,7 +91,7 @@ def _serialize_happiness_mappings(happiness_mappings):
     return {
         sharenum: base32.b2a(serverid)
         for (sharenum, serverid)
-        in happiness_mappings.iteritems()
+        in happiness_mappings.items()
     }
 
 _HAPPINESS_MAPPINGS = Field(
@@ -150,7 +162,9 @@ class HelperUploadResults(Copyable, RemoteCopy):
     # note: don't change this string, it needs to match the value used on the
     # helper, and it does *not* need to match the fully-qualified
     # package/module/class name
-    typeToCopy = "allmydata.upload.UploadResults.tahoe.allmydata.com"
+    #
+    # Needs to be native string to make Foolscap happy.
+    typeToCopy = native_str("allmydata.upload.UploadResults.tahoe.allmydata.com")
     copytype = typeToCopy
 
     # also, think twice about changing the shape of any existing attribute,
@@ -283,7 +297,7 @@ class ServerTracker(object):
         #log.msg("%s._got_reply(%s)" % (self, (alreadygot, buckets)))
         (alreadygot, buckets) = alreadygot_and_buckets
         b = {}
-        for sharenum, rref in buckets.items():
+        for sharenum, rref in list(buckets.items()):
             bp = self.wbp_class(rref, self._server, self.sharesize,
                                 self.blocksize,
                                 self.num_segments,
@@ -780,7 +794,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
 
         shares_to_ask = set()
         servermap = self._share_placements
-        for shnum, tracker_id in servermap.items():
+        for shnum, tracker_id in list(servermap.items()):
             if tracker_id == None:
                 continue
             if tracker.get_serverid() == tracker_id:
@@ -1574,7 +1588,7 @@ class AssistedUploader(object):
         # abbreviated), so if we detect old results, just clobber them.
 
         sharemap = upload_results.sharemap
-        if str in [type(v) for v in sharemap.values()]:
+        if any(isinstance(v, (bytes, unicode)) for v in sharemap.values()):
             upload_results.sharemap = None
 
     def _build_verifycap(self, helper_upload_results):

src/allmydata/node.py

@@ -9,11 +9,16 @@ import os.path
 import re
 import types
 import errno
-from six.moves import configparser
+from io import StringIO
 import tempfile
-from io import BytesIO
 from base64 import b32decode, b32encode
 
+# Python 2 compatibility
+from six.moves import configparser
+from future.utils import PY2
+if PY2:
+    from io import BytesIO as StringIO  # noqa: F811
+
 from twisted.python import log as twlog
 from twisted.application import service
 from twisted.python.failure import Failure
@@ -70,7 +75,7 @@ def _common_valid_config():
 # Add our application versions to the data that Foolscap's LogPublisher
 # reports.
 for thing, things_version in get_package_versions().items():
-    app_versions.add_version(thing, str(things_version))
+    app_versions.add_version(thing, things_version)
 
 # group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
 ADDR_RE = re.compile("^([1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*)(:([1-9][0-9]*))?$")
@@ -206,7 +211,7 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
 
     # load configuration from in-memory string
     parser = configparser.SafeConfigParser()
-    parser.readfp(BytesIO(config_str))
+    parser.readfp(StringIO(config_str))
 
     fname = "<in-memory>"
     configutil.validate_config(fname, parser, _valid_config)
@@ -821,9 +826,9 @@ class Node(service.MultiService):
         for o in twlog.theLogPublisher.observers:
             # o might be a FileLogObserver's .emit method
             if type(o) is type(self.setup_logging): # bound method
-                ob = o.im_self
+                ob = o.__self__
                 if isinstance(ob, twlog.FileLogObserver):
-                    newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__)
+                    newmeth = types.MethodType(formatTimeTahoeStyle, ob)
                     ob.formatTime = newmeth
         # TODO: twisted >2.5.0 offers maxRotatedFiles=50
 

src/allmydata/scripts/common.py

@@ -4,7 +4,7 @@ import os, sys, urllib, textwrap
 import codecs
 from os.path import join
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/scripts/stats_gatherer.py

@@ -2,7 +2,7 @@ from __future__ import print_function
 
 import os
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/scripts/tahoe_check.py

@@ -3,7 +3,7 @@ from __future__ import print_function
 import urllib
 import json
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/stats.py

@@ -6,7 +6,7 @@ import pprint
 import time
 from collections import deque
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/test/check_load.py

@@ -37,7 +37,7 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
 import os, sys, httplib, binascii
 import urllib, json, random, time, urlparse
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/test/check_memory.py

@@ -2,7 +2,7 @@ from __future__ import print_function
 
 import os, shutil, sys, urllib, time, stat, urlparse
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/test/common.py

@@ -46,6 +46,7 @@ from testtools.twistedsupport import (
     flush_logged_errors,
 )
 
+from twisted.application import service
 from twisted.plugin import IPlugin
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks, returnValue
@@ -87,16 +88,15 @@ from ..crypto import (
 from .eliotutil import (
     EliotLoggedRunTest,
 )
-# Backwards compatibility imports:
+from .common_util import ShouldFailMixin  # noqa: F401
-from .common_py3 import LoggingServiceParent, ShouldFailMixin  # noqa: F401
 
 
 TEST_RSA_KEY_SIZE = 522
 
 EMPTY_CLIENT_CONFIG = config_from_string(
-    b"/dev/null",
+    "/dev/null",
-    b"tub.port",
+    "tub.port",
-    b""
+    ""
 )
 
 
@@ -249,8 +249,8 @@ class UseNode(object):
 
         self.config = config_from_string(
             self.basedir.asTextMode().path,
-            u"tub.port",
+            "tub.port",
-b"""
+"""
 [node]
 {node_config}
 
@@ -781,6 +781,11 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
     return filenode
 
 
+class LoggingServiceParent(service.MultiService):
+    def log(self, *args, **kwargs):
+        return log.msg(*args, **kwargs)
+
+
 TEST_DATA=b"\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1)
 
 

src/allmydata/test/common_py3.py

@@ -1,170 +0,0 @@
-"""
-Common utilities that have been ported to Python 3.
-
-Ported to Python 3.
-"""
-
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-from past.builtins import unicode
-
-import os
-import time
-import signal
-
-from twisted.internet import defer, reactor
-from twisted.application import service
-from twisted.python import failure
-from twisted.trial import unittest
-
-from ..util.assertutil import precondition
-from ..util.encodingutil import unicode_platform, get_filesystem_encoding
-from ..util import log
-
-
-class TimezoneMixin(object):
-
-    def setTimezone(self, timezone):
-        def tzset_if_possible():
-            # Windows doesn't have time.tzset().
-            if hasattr(time, 'tzset'):
-                time.tzset()
-
-        unset = object()
-        originalTimezone = os.environ.get('TZ', unset)
-        def restoreTimezone():
-            if originalTimezone is unset:
-                del os.environ['TZ']
-            else:
-                os.environ['TZ'] = originalTimezone
-            tzset_if_possible()
-
-        os.environ['TZ'] = timezone
-        self.addCleanup(restoreTimezone)
-        tzset_if_possible()
-
-    def have_working_tzset(self):
-        return hasattr(time, 'tzset')
-
-
-class SignalMixin(object):
-    # This class is necessary for any code which wants to use Processes
-    # outside the usual reactor.run() environment. It is copied from
-    # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
-    # something rather different.
-    sigchldHandler = None
-
-    def setUp(self):
-        # make sure SIGCHLD handler is installed, as it should be on
-        # reactor.run(). problem is reactor may not have been run when this
-        # test runs.
-        if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
-            self.sigchldHandler = signal.signal(signal.SIGCHLD,
-                                                reactor._handleSigchld)
-        return super(SignalMixin, self).setUp()
-
-    def tearDown(self):
-        if self.sigchldHandler:
-            signal.signal(signal.SIGCHLD, self.sigchldHandler)
-        return super(SignalMixin, self).tearDown()
-
-
-class ShouldFailMixin(object):
-
-    def shouldFail(self, expected_failure, which, substring,
-                   callable, *args, **kwargs):
-        """Assert that a function call raises some exception. This is a
-        Deferred-friendly version of TestCase.assertRaises() .
-
-        Suppose you want to verify the following function:
-
-         def broken(a, b, c):
-             if a < 0:
-                 raise TypeError('a must not be negative')
-             return defer.succeed(b+c)
-
-        You can use:
-            d = self.shouldFail(TypeError, 'test name',
-                                'a must not be negative',
-                                broken, -4, 5, c=12)
-        in your test method. The 'test name' string will be included in the
-        error message, if any, because Deferred chains frequently make it
-        difficult to tell which assertion was tripped.
-
-        The substring= argument, if not None, must appear in the 'repr'
-        of the message wrapped by this Failure, or the test will fail.
-        """
-
-        assert substring is None or isinstance(substring, (bytes, unicode))
-        d = defer.maybeDeferred(callable, *args, **kwargs)
-        def done(res):
-            if isinstance(res, failure.Failure):
-                res.trap(expected_failure)
-                if substring:
-                    self.failUnless(substring in str(res),
-                                    "%s: substring '%s' not in '%s'"
-                                    % (which, substring, str(res)))
-                # return the Failure for further analysis, but in a form that
-                # doesn't make the Deferred chain think that we failed.
-                return [res]
-            else:
-                self.fail("%s was supposed to raise %s, not get '%s'" %
-                          (which, expected_failure, res))
-        d.addBoth(done)
-        return d
-
-
-class ReallyEqualMixin(object):
-    def failUnlessReallyEqual(self, a, b, msg=None):
-        self.assertEqual(a, b, msg)
-        self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
-
-
-def skip_if_cannot_represent_filename(u):
-    precondition(isinstance(u, unicode))
-
-    enc = get_filesystem_encoding()
-    if not unicode_platform():
-        try:
-            u.encode(enc)
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
-
-
-class Marker(object):
-    pass
-
-class FakeCanary(object):
-    """For use in storage tests.
-
-    Can be moved back to test_storage.py once enough Python 3 porting has been
-    done.
-    """
-    def __init__(self, ignore_disconnectors=False):
-        self.ignore = ignore_disconnectors
-        self.disconnectors = {}
-    def notifyOnDisconnect(self, f, *args, **kwargs):
-        if self.ignore:
-            return
-        m = Marker()
-        self.disconnectors[m] = (f, args, kwargs)
-        return m
-    def dontNotifyOnDisconnect(self, marker):
-        if self.ignore:
-            return
-        del self.disconnectors[marker]
-    def getRemoteTubID(self):
-        return None
-    def getPeer(self):
-        return "<fake>"
-
-
-class LoggingServiceParent(service.MultiService):
-    def log(self, *args, **kwargs):
-        return log.msg(*args, **kwargs)

src/allmydata/test/common_util.py

@@ -1,22 +1,33 @@
 from __future__ import print_function
 
 import os
+import time
+import signal
 from random import randrange
 from six.moves import StringIO
 
 from twisted.internet import reactor, defer
+from twisted.python import failure
 from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..scripts import runner
-from allmydata.util.encodingutil import get_io_encoding
+from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding
 # Imported for backwards compatibility:
 from future.utils import bord, bchr, binary_type
-from .common_py3 import (
+from past.builtins import unicode
-    SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, ShouldFailMixin
-)
 
 
+def skip_if_cannot_represent_filename(u):
+    precondition(isinstance(u, unicode))
+
+    enc = get_filesystem_encoding()
+    if not unicode_platform():
+        try:
+            u.encode(enc)
+        except UnicodeEncodeError:
+            raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
+
 def skip_if_cannot_represent_argv(u):
     precondition(isinstance(u, unicode))
     try:
@@ -78,6 +89,34 @@ def flip_one_bit(s, offset=0, size=None):
     return result
 
 
+class ReallyEqualMixin(object):
+    def failUnlessReallyEqual(self, a, b, msg=None):
+        self.assertEqual(a, b, msg)
+        self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
+
+
+class SignalMixin(object):
+    # This class is necessary for any code which wants to use Processes
+    # outside the usual reactor.run() environment. It is copied from
+    # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
+    # something rather different.
+    sigchldHandler = None
+
+    def setUp(self):
+        # make sure SIGCHLD handler is installed, as it should be on
+        # reactor.run(). problem is reactor may not have been run when this
+        # test runs.
+        if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
+            self.sigchldHandler = signal.signal(signal.SIGCHLD,
+                                                reactor._handleSigchld)
+        return super(SignalMixin, self).setUp()
+
+    def tearDown(self):
+        if self.sigchldHandler:
+            signal.signal(signal.SIGCHLD, self.sigchldHandler)
+        return super(SignalMixin, self).tearDown()
+
+
 class StallMixin(object):
     def stall(self, res=None, delay=1):
         d = defer.Deferred()
@@ -85,6 +124,76 @@ class StallMixin(object):
         return d
 
 
+class Marker(object):
+    pass
+
+class FakeCanary(object):
+    """For use in storage tests.
+    """
+    def __init__(self, ignore_disconnectors=False):
+        self.ignore = ignore_disconnectors
+        self.disconnectors = {}
+    def notifyOnDisconnect(self, f, *args, **kwargs):
+        if self.ignore:
+            return
+        m = Marker()
+        self.disconnectors[m] = (f, args, kwargs)
+        return m
+    def dontNotifyOnDisconnect(self, marker):
+        if self.ignore:
+            return
+        del self.disconnectors[marker]
+    def getRemoteTubID(self):
+        return None
+    def getPeer(self):
+        return "<fake>"
+
+
+class ShouldFailMixin(object):
+
+    def shouldFail(self, expected_failure, which, substring,
+                   callable, *args, **kwargs):
+        """Assert that a function call raises some exception. This is a
+        Deferred-friendly version of TestCase.assertRaises() .
+
+        Suppose you want to verify the following function:
+
+         def broken(a, b, c):
+             if a < 0:
+                 raise TypeError('a must not be negative')
+             return defer.succeed(b+c)
+
+        You can use:
+            d = self.shouldFail(TypeError, 'test name',
+                                'a must not be negative',
+                                broken, -4, 5, c=12)
+        in your test method. The 'test name' string will be included in the
+        error message, if any, because Deferred chains frequently make it
+        difficult to tell which assertion was tripped.
+
+        The substring= argument, if not None, must appear in the 'repr'
+        of the message wrapped by this Failure, or the test will fail.
+        """
+
+        assert substring is None or isinstance(substring, (bytes, unicode))
+        d = defer.maybeDeferred(callable, *args, **kwargs)
+        def done(res):
+            if isinstance(res, failure.Failure):
+                res.trap(expected_failure)
+                if substring:
+                    self.failUnless(substring in str(res),
+                                    "%s: substring '%s' not in '%s'"
+                                    % (which, substring, str(res)))
+                # return the Failure for further analysis, but in a form that
+                # doesn't make the Deferred chain think that we failed.
+                return [res]
+            else:
+                self.fail("%s was supposed to raise %s, not get '%s'" %
+                          (which, expected_failure, res))
+        d.addBoth(done)
+        return d
+
+
 class TestMixin(SignalMixin):
     def setUp(self):
         return super(TestMixin, self).setUp()
@@ -132,6 +241,31 @@ class TestMixin(SignalMixin):
             self.fail("Reactor was still active when it was required to be quiescent.")
 
 
+class TimezoneMixin(object):
+
+    def setTimezone(self, timezone):
+        def tzset_if_possible():
+            # Windows doesn't have time.tzset().
+            if hasattr(time, 'tzset'):
+                time.tzset()
+
+        unset = object()
+        originalTimezone = os.environ.get('TZ', unset)
+        def restoreTimezone():
+            if originalTimezone is unset:
+                del os.environ['TZ']
+            else:
+                os.environ['TZ'] = originalTimezone
+            tzset_if_possible()
+
+        os.environ['TZ'] = timezone
+        self.addCleanup(restoreTimezone)
+        tzset_if_possible()
+
+    def have_working_tzset(self):
+        return hasattr(time, 'tzset')
+
+
 try:
     import win32file
     import win32con

src/allmydata/test/eliotutil.py

@@ -2,7 +2,7 @@
 Tools aimed at the interaction between tests and Eliot.
 """
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 # Can't use `builtins.str` because it's not JSON encodable:
 # `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
 from past.builtins import unicode as str
@@ -106,7 +106,7 @@ def eliot_logged_test(f):
 
         # Begin an action that should comprise all messages from the decorated
         # test method.
-        with RUN_TEST(name=self.id().decode("utf-8")).context() as action:
+        with RUN_TEST(name=self.id()).context() as action:
             # When the test method Deferred fires, the RUN_TEST action is
             # done.  However, we won't have re-published the MemoryLogger
             # messages into the global/default logger when this Deferred

src/allmydata/test/mutable/test_version.py

@@ -2,7 +2,7 @@ from __future__ import print_function
 
 import os
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/test/test_auth.py

@@ -1,4 +1,4 @@
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401

src/allmydata/test/test_client.py

@@ -252,11 +252,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         is not set.
         """
         config = client.config_from_string(
-            b"test_storage_default_anonymous_enabled",
+            "test_storage_default_anonymous_enabled",
-            b"tub.port",
+            "tub.port",
             BASECONFIG + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
             )
         )
         self.assertTrue(client.anonymous_storage_enabled(config))
@@ -268,11 +268,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         config = client.config_from_string(
             self.id(),
-            b"tub.port",
+            "tub.port",
             BASECONFIG + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
-                b"anonymous = true\n"
+                "anonymous = true\n"
             )
         )
         self.assertTrue(client.anonymous_storage_enabled(config))
@@ -284,11 +284,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         config = client.config_from_string(
             self.id(),
-            b"tub.port",
+            "tub.port",
             BASECONFIG + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
-                b"anonymous = false\n"
+                "anonymous = false\n"
             )
         )
         self.assertFalse(client.anonymous_storage_enabled(config))
@@ -300,11 +300,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         config = client.config_from_string(
             self.id(),
-            b"tub.port",
+            "tub.port",
             BASECONFIG + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = false\n"
+                "enabled = false\n"
-                b"anonymous = true\n"
+                "anonymous = true\n"
             )
         )
         self.assertFalse(client.anonymous_storage_enabled(config))
@@ -680,11 +680,11 @@ class AnonymousStorage(SyncTestCase):
         os.makedirs(basedir + b"/private")
         config = client.config_from_string(
             basedir,
-            b"tub.port",
+            "tub.port",
             BASECONFIG_I % (SOME_FURL,) + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
-                b"anonymous = true\n"
+                "anonymous = true\n"
             )
         )
         node = yield client.create_client_from_config(
@@ -711,11 +711,11 @@ class AnonymousStorage(SyncTestCase):
         os.makedirs(basedir + b"/private")
         config = client.config_from_string(
             basedir,
-            b"tub.port",
+            "tub.port",
             BASECONFIG_I % (SOME_FURL,) + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
-                b"anonymous = false\n"
+                "anonymous = false\n"
             )
         )
         node = yield client.create_client_from_config(
@@ -732,7 +732,7 @@ class AnonymousStorage(SyncTestCase):
             ]),
         )
         self.expectThat(
-            config.get_private_config(b"storage.furl", default=None),
+            config.get_private_config("storage.furl", default=None),
             Is(None),
         )
 
@@ -748,18 +748,18 @@ class AnonymousStorage(SyncTestCase):
         os.makedirs(basedir + b"/private")
         enabled_config = client.config_from_string(
             basedir,
-            b"tub.port",
+            "tub.port",
             BASECONFIG_I % (SOME_FURL,) + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
-                b"anonymous = true\n"
+                "anonymous = true\n"
             )
         )
         node = yield client.create_client_from_config(
             enabled_config,
             _introducer_factory=MemoryIntroducerClient,
         )
-        anonymous_storage_furl = enabled_config.get_private_config(b"storage.furl")
+        anonymous_storage_furl = enabled_config.get_private_config("storage.furl")
         def check_furl():
             return node.tub.getReferenceForURL(anonymous_storage_furl)
         # Perform a sanity check that our test code makes sense: is this a
@@ -772,11 +772,11 @@ class AnonymousStorage(SyncTestCase):
 
         disabled_config = client.config_from_string(
             basedir,
-            b"tub.port",
+            "tub.port",
             BASECONFIG_I % (SOME_FURL,) + (
-                b"[storage]\n"
+                "[storage]\n"
-                b"enabled = true\n"
+                "enabled = true\n"
-                b"anonymous = false\n"
+                "anonymous = false\n"
             )
         )
         node = yield client.create_client_from_config(
@@ -1137,8 +1137,8 @@ class StorageAnnouncementTests(SyncTestCase):
         create_node_dir(self.basedir, u"")
 
 
-    def get_config(self, storage_enabled, more_storage=b"", more_sections=b""):
+    def get_config(self, storage_enabled, more_storage="", more_sections=""):
-        return b"""
+        return """
 [node]
 tub.location = tcp:192.0.2.0:1234
 
@@ -1163,7 +1163,7 @@ introducer.furl = pb://abcde@nowhere/fake
         """
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(storage_enabled=False),
         )
         self.assertThat(
@@ -1185,7 +1185,7 @@ introducer.furl = pb://abcde@nowhere/fake
         """
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(storage_enabled=True),
         )
         client_deferred = client.create_client_from_config(
@@ -1217,13 +1217,13 @@ introducer.furl = pb://abcde@nowhere/fake
         value = u"thing"
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(
                 storage_enabled=True,
-                more_storage=b"plugins=tahoe-lafs-dummy-v1",
+                more_storage="plugins=tahoe-lafs-dummy-v1",
                 more_sections=(
-                    b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
+                    "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
-                    b"some = {}\n".format(value)
+                    "some = {}\n".format(value)
                 ),
             ),
         )
@@ -1258,15 +1258,15 @@ introducer.furl = pb://abcde@nowhere/fake
 
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(
                 storage_enabled=True,
-                more_storage=b"plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2",
+                more_storage="plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2",
                 more_sections=(
-                    b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
+                    "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
-                    b"some = thing-1\n"
+                    "some = thing-1\n"
-                    b"[storageserver.plugins.tahoe-lafs-dummy-v2]\n"
+                    "[storageserver.plugins.tahoe-lafs-dummy-v2]\n"
-                    b"some = thing-2\n"
+                    "some = thing-2\n"
                 ),
             ),
         )
@@ -1306,13 +1306,13 @@ introducer.furl = pb://abcde@nowhere/fake
 
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(
                 storage_enabled=True,
-                more_storage=b"plugins=tahoe-lafs-dummy-v1",
+                more_storage="plugins=tahoe-lafs-dummy-v1",
                 more_sections=(
-                    b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
+                    "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
-                    b"some = thing\n"
+                    "some = thing\n"
                 ),
             ),
         )
@@ -1342,10 +1342,10 @@ introducer.furl = pb://abcde@nowhere/fake
 
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(
                 storage_enabled=True,
-                more_storage=b"plugins=tahoe-lafs-dummy-v1",
+                more_storage="plugins=tahoe-lafs-dummy-v1",
             ),
         )
         self.assertThat(
@@ -1380,14 +1380,14 @@ introducer.furl = pb://abcde@nowhere/fake
 
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(
                 storage_enabled=True,
-                more_storage=b"plugins=tahoe-lafs-dummy-v1",
+                more_storage="plugins=tahoe-lafs-dummy-v1",
                 more_sections=(
-                    b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
+                    "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
                     # This will make it explode on instantiation.
-                    b"invalid = configuration\n"
+                    "invalid = configuration\n"
                 )
             ),
         )
@@ -1407,10 +1407,10 @@ introducer.furl = pb://abcde@nowhere/fake
         """
         config = client.config_from_string(
             self.basedir,
-            u"tub.port",
+            "tub.port",
             self.get_config(
                 storage_enabled=True,
-                more_storage=b"plugins=tahoe-lafs-dummy-vX",
+                more_storage="plugins=tahoe-lafs-dummy-vX",
             ),
         )
         self.assertThat(

src/allmydata/test/test_crawler.py

@@ -27,8 +27,7 @@ from allmydata.util import fileutil, hashutil, pollmixin
 from allmydata.storage.server import StorageServer, si_b2a
 from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded
 
-from allmydata.test.common_py3 import FakeCanary
+from allmydata.test.common_util import StallMixin, FakeCanary
-from allmydata.test.common_util import StallMixin
 
 class BucketEnumeratingCrawler(ShareCrawler):
     cpu_slice = 500 # make sure it can complete in a single slice

src/allmydata/test/test_deepcheck.py

@@ -1,6 +1,6 @@
 import os, json, urllib
 
-# BBB: Python 2 compatibility
+# Python 2 compatibility
 # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
 # a `future.types.newstr.newstr` as a *Python 3* byte string representation under
 # *Python 2*:

src/allmydata/test/test_encodingutil.py

@@ -77,7 +77,7 @@ from twisted.trial import unittest
 
 from twisted.python.filepath import FilePath
 
-from allmydata.test.common_py3 import (
+from allmydata.test.common_util import (
     ReallyEqualMixin, skip_if_cannot_represent_filename,
 )
 from allmydata.util import encodingutil, fileutil

src/allmydata/test/test_happiness.py

@@ -23,7 +23,7 @@ from hypothesis.strategies import text, sets
 from allmydata.immutable import happiness_upload
 from allmydata.util.happinessutil import servers_of_happiness, \
     shares_by_server, merge_servers
-from allmydata.test.common_py3 import ShouldFailMixin
+from allmydata.test.common import ShouldFailMixin
 
 
 class HappinessUploadUtils(unittest.TestCase):

src/allmydata/test/test_iputil.py

@@ -23,7 +23,7 @@ from tenacity import retry, stop_after_attempt
 from foolscap.api import Tub
 
 from allmydata.util import iputil, gcutil
-import allmydata.test.common_py3 as testutil
+import allmydata.test.common_util as testutil
 from allmydata.util.namespace import Namespace
 
 

src/allmydata/test/test_node.py

@@ -520,7 +520,6 @@ introducer.furl = empty
 enabled = false
 [i2p]
 enabled = false
-[node]
 """
 
 NOLISTEN = """
@@ -566,6 +565,7 @@ class Listeners(unittest.TestCase):
         create_node_dir(basedir, "testing")
         with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
             f.write(BASE_CONFIG)
+            f.write("[node]\n")
             f.write("tub.port = tcp:0\n")
             f.write("tub.location = AUTO\n")
 
@@ -594,6 +594,7 @@ class Listeners(unittest.TestCase):
         location = "tcp:localhost:%d,tcp:localhost:%d" % (port1, port2)
         with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
             f.write(BASE_CONFIG)
+            f.write("[node]\n")
             f.write("tub.port = %s\n" % port)
             f.write("tub.location = %s\n" % location)
 
@@ -617,6 +618,7 @@ class Listeners(unittest.TestCase):
         os.mkdir(os.path.join(basedir, "private"))
         with open(config_fname, "w") as f:
             f.write(BASE_CONFIG)
+            f.write("[node]\n")
             f.write("tub.port = listen:i2p,listen:tor\n")
             f.write("tub.location = tcp:example.org:1234\n")
         config = client.read_config(basedir, "client.port")

src/allmydata/test/test_storage.py

@@ -51,7 +51,8 @@ from allmydata.test.no_network import NoNetworkServer
 from allmydata.storage_client import (
     _StorageServer,
 )
-from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin
+from .common import LoggingServiceParent, ShouldFailMixin
+from .common_util import FakeCanary
 
 
 class UtilTests(unittest.TestCase):

src/allmydata/test/test_storage_web.py

@@ -50,7 +50,7 @@ from allmydata.web.storage import (
     StorageStatusElement,
     remove_prefix
 )
-from .common_py3 import FakeCanary
+from .common_util import FakeCanary
 
 def remove_tags(s):
     s = re.sub(br'<[^>]*>', b' ', s)

src/allmydata/test/test_time_format.py

@@ -16,7 +16,7 @@ import time
 
 from twisted.trial import unittest
 
-from allmydata.test.common_py3 import TimezoneMixin
+from allmydata.test.common_util import TimezoneMixin
 from allmydata.util import time_format
 
 

src/allmydata/test/test_upload.py

@@ -28,12 +28,12 @@ from allmydata.util import log, base32
 from allmydata.util.assertutil import precondition
 from allmydata.util.deferredutil import DeferredListShouldSucceed
 from allmydata.test.no_network import GridTestMixin
-from allmydata.test.common_py3 import ShouldFailMixin
 from allmydata.storage_client import StorageFarmBroker
 from allmydata.storage.server import storage_index_to_dir
 from allmydata.client import _Client
 from .common import (
     EMPTY_CLIENT_CONFIG,
+    ShouldFailMixin,
 )
 from functools import reduce
 

src/allmydata/test/web/test_introducer.py

@@ -33,7 +33,7 @@ from .common import (
 from ..common import (
     SameProcessStreamEndpointAssigner,
 )
-from ..common_py3 import (
+from ..common_util import (
     FakeCanary,
 )
 from ..common_web import (

src/allmydata/test/web/test_web.py

@@ -52,7 +52,7 @@ from allmydata.interfaces import (
 )
 from allmydata.mutable import servermap, publish, retrieve
 from .. import common_util as testutil
-from ..common_py3 import TimezoneMixin
+from ..common_util import TimezoneMixin
 from ..common_web import (
     do_http,
     Error,

src/allmydata/util/_python3.py

@@ -24,6 +24,7 @@ if PY2:
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
+    "allmydata._monkeypatch",
     "allmydata.codec",
     "allmydata.crypto",
     "allmydata.crypto.aes",
@@ -40,8 +41,12 @@ PORTED_MODULES = [
     "allmydata.immutable.downloader.segmentation",
     "allmydata.immutable.downloader.share",
     "allmydata.immutable.downloader.status",
+    "allmydata.immutable.encode",
+    "allmydata.immutable.filenode",
     "allmydata.immutable.happiness_upload",
+    "allmydata.immutable.layout",
     "allmydata.immutable.literal",
+    "allmydata.immutable.upload",
     "allmydata.interfaces",
     "allmydata.introducer.interfaces",
     "allmydata.monitor",
@@ -53,7 +58,6 @@ PORTED_MODULES = [
     "allmydata.storage.mutable",
     "allmydata.storage.server",
     "allmydata.storage.shares",
-    "allmydata.test.common_py3",
     "allmydata.test.no_network",
     "allmydata.uri",
     "allmydata.util._python3",