tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

Merge branch 'master' into 3459.test-checker-python-3

This commit is contained in:
Itamar Turner-Trauring 2020-10-09 10:22:17 -04:00 committed by GitHub
parent c680b1d971 cc8c9c0bdf
commit 17f0676b3f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 372 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -51,6 +51,7 @@ test: .tox/create-venvs.log
## Run all tests with coverage collection and reporting.
test-venv-coverage:
# Special handling for reporting coverage even when the test run fails
rm -f ./.coverage.*
test_exit=
$(VIRTUAL_ENV)/bin/coverage run -m twisted.trial --rterrors --reporter=timing \
$(TEST_SUITE) || test_exit="$$?"

2
misc/python3/Makefile
View File

@ -1,4 +1,4 @@
# BBB: Python 3 porting targets
# Python 3 porting targets
#
# NOTE: this Makefile requires GNU make

1
newsfragments/3455.minor Normal file
View File

@ -0,0 +1 @@
Begin porting the `node` module to Python 3.

0
newsfragments/3456.minor Normal file
View File

0
newsfragments/3458.minor Normal file
View File

0
newsfragments/3462.minor Normal file
View File

0
newsfragments/3463.minor Normal file
View File

1
newsfragments/3464.minor Normal file
View File

@ -0,0 +1 @@
Cleanup comments that don't match the project convention.

6
src/allmydata/__init__.py
View File

@ -42,3 +42,9 @@ __full_version__ = __appname__ + '/' + str(__version__)
# Install Python 3 module locations in Python 2:
from future import standard_library
standard_library.install_aliases()
# Monkey-patch 3rd party libraries:
from ._monkeypatch import patch
patch()
del patch

48
src/allmydata/_monkeypatch.py Normal file
View File

@ -0,0 +1,48 @@
"""
Monkey-patching of third party libraries.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from warnings import catch_warnings
def patch():
"""Path third-party libraries to make Tahoe-LAFS work."""
# Make sure Foolscap always get native strings passed to method names in callRemote.
# This can be removed when any one of the following happens:
#
# 1. Tahoe-LAFS on Python 2 switches to version of Foolscap that fixes
# https://github.com/warner/foolscap/issues/72
# 2. Foolscap is dropped as a dependency.
# 3. Tahoe-LAFS drops Python 2 support.
if not PY2:
# Python 3 doesn't need to monkey patch Foolscap
return
# We need to suppress warnings so as to prevent unexpected output from
# breaking some integration tests.
with catch_warnings(record=True):
# Only tested with this version; ensure correctness with new releases,
# and then either update the assert or hopefully drop the monkeypatch.
from foolscap import __version__
assert __version__ == "0.13.1", "Wrong version %s of Foolscap" % (__version__,)
from foolscap.referenceable import RemoteReference
original_getMethodInfo = RemoteReference._getMethodInfo
def _getMethodInfo(self, name):
if isinstance(name, str):
name = name.encode("utf-8")
return original_getMethodInfo(self, name)
RemoteReference._getMethodInfo = _getMethodInfo

4
src/allmydata/client.py
View File

@ -69,8 +69,8 @@ def _is_valid_section(section_name):
Currently considers all possible storage server plugin sections valid.
"""
return (
section_name.startswith(b"storageserver.plugins.") or
section_name.startswith(b"storageclient.plugins.")
section_name.startswith("storageserver.plugins.") or
section_name.startswith("storageclient.plugins.")
)

6
src/allmydata/immutable/downloader/share.py
View File

@ -6,7 +6,7 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, native_str
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
@ -765,9 +765,7 @@ class Share(object):
level=log.WEIRD, umid="qZu0wg"))
def _send_request(self, start, length):
# For some reason tests fail on Python 2 if this is not a native
# string...
return self._rref.callRemote(native_str("read"), start, length)
return self._rref.callRemote("read", start, length)
def _got_data(self, data, start, length, block_ev, lp):
block_ev.finished(len(data), now())

15
src/allmydata/immutable/encode.py
View File

@ -1,5 +1,18 @@
# -*- test-case-name: allmydata.test.test_encode -*-
"""
Ported to Python 3.
"""
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
from zope.interface import implementer
from twisted.internet import defer
@ -468,7 +481,7 @@ class Encoder(object):
(self,
self.segment_size*(segnum+1),
self.segment_size*self.num_segments,
100 * (segnum+1) / self.num_segments,
100 * (segnum+1) // self.num_segments,
),
level=log.OPERATIONAL)
elapsed = time.time() - start

13
src/allmydata/immutable/filenode.py
View File

@ -1,3 +1,16 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from functools import reduce
import binascii
from time import time as now

12
src/allmydata/immutable/layout.py
View File

@ -1,3 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import struct
from zope.interface import implementer
from twisted.internet import defer

26
src/allmydata/immutable/upload.py
View File

@ -1,3 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, native_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long, unicode
import os, time, weakref, itertools
@ -66,7 +78,7 @@ def _serialize_existing_shares(existing_shares):
return {
server: list(shares)
for (server, shares)
in existing_shares.iteritems()
in existing_shares.items()
}
_EXISTING_SHARES = Field(
@ -79,7 +91,7 @@ def _serialize_happiness_mappings(happiness_mappings):
return {
sharenum: base32.b2a(serverid)
for (sharenum, serverid)
in happiness_mappings.iteritems()
in happiness_mappings.items()
}
_HAPPINESS_MAPPINGS = Field(
@ -150,7 +162,9 @@ class HelperUploadResults(Copyable, RemoteCopy):
# note: don't change this string, it needs to match the value used on the
# helper, and it does *not* need to match the fully-qualified
# package/module/class name
typeToCopy = "allmydata.upload.UploadResults.tahoe.allmydata.com"
#
# Needs to be native string to make Foolscap happy.
typeToCopy = native_str("allmydata.upload.UploadResults.tahoe.allmydata.com")
copytype = typeToCopy
# also, think twice about changing the shape of any existing attribute,
@ -283,7 +297,7 @@ class ServerTracker(object):
#log.msg("%s._got_reply(%s)" % (self, (alreadygot, buckets)))
(alreadygot, buckets) = alreadygot_and_buckets
b = {}
for sharenum, rref in buckets.items():
for sharenum, rref in list(buckets.items()):
bp = self.wbp_class(rref, self._server, self.sharesize,
self.blocksize,
self.num_segments,
@ -780,7 +794,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
shares_to_ask = set()
servermap = self._share_placements
for shnum, tracker_id in servermap.items():
for shnum, tracker_id in list(servermap.items()):
if tracker_id == None:
continue
if tracker.get_serverid() == tracker_id:
@ -1574,7 +1588,7 @@ class AssistedUploader(object):
# abbreviated), so if we detect old results, just clobber them.
sharemap = upload_results.sharemap
if str in [type(v) for v in sharemap.values()]:
if any(isinstance(v, (bytes, unicode)) for v in sharemap.values()):
upload_results.sharemap = None
def _build_verifycap(self, helper_upload_results):

17
src/allmydata/node.py
View File

@ -9,11 +9,16 @@ import os.path
import re
import types
import errno
from six.moves import configparser
from io import StringIO
import tempfile
from io import BytesIO
from base64 import b32decode, b32encode
# Python 2 compatibility
from six.moves import configparser
from future.utils import PY2
if PY2:
from io import BytesIO as StringIO # noqa: F811
from twisted.python import log as twlog
from twisted.application import service
from twisted.python.failure import Failure
@ -70,7 +75,7 @@ def _common_valid_config():
# Add our application versions to the data that Foolscap's LogPublisher
# reports.
for thing, things_version in get_package_versions().items():
app_versions.add_version(thing, str(things_version))
app_versions.add_version(thing, things_version)
# group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
ADDR_RE = re.compile("^([1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*)(:([1-9][0-9]*))?$")
@ -206,7 +211,7 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
# load configuration from in-memory string
parser = configparser.SafeConfigParser()
parser.readfp(BytesIO(config_str))
parser.readfp(StringIO(config_str))
fname = "<in-memory>"
configutil.validate_config(fname, parser, _valid_config)
@ -821,9 +826,9 @@ class Node(service.MultiService):
for o in twlog.theLogPublisher.observers:
# o might be a FileLogObserver's .emit method
if type(o) is type(self.setup_logging): # bound method
ob = o.im_self
ob = o.__self__
if isinstance(ob, twlog.FileLogObserver):
newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__)
newmeth = types.MethodType(formatTimeTahoeStyle, ob)
ob.formatTime = newmeth
# TODO: twisted >2.5.0 offers maxRotatedFiles=50

2
src/allmydata/scripts/common.py
View File

@ -4,7 +4,7 @@ import os, sys, urllib, textwrap
import codecs
from os.path import join
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

2
src/allmydata/scripts/stats_gatherer.py
View File

@ -2,7 +2,7 @@ from __future__ import print_function
import os
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

2
src/allmydata/scripts/tahoe_check.py
View File

@ -3,7 +3,7 @@ from __future__ import print_function
import urllib
import json
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

2
src/allmydata/stats.py
View File

@ -6,7 +6,7 @@ import pprint
import time
from collections import deque
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

2
src/allmydata/test/check_load.py
View File

@ -37,7 +37,7 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
import os, sys, httplib, binascii
import urllib, json, random, time, urlparse
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

2
src/allmydata/test/check_memory.py
View File

@ -2,7 +2,7 @@ from __future__ import print_function
import os, shutil, sys, urllib, time, stat, urlparse
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

19
src/allmydata/test/common.py
View File

@ -46,6 +46,7 @@ from testtools.twistedsupport import (
flush_logged_errors,
)
from twisted.application import service
from twisted.plugin import IPlugin
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks, returnValue
@ -87,16 +88,15 @@ from ..crypto import (
from .eliotutil import (
EliotLoggedRunTest,
)
# Backwards compatibility imports:
from .common_py3 import LoggingServiceParent, ShouldFailMixin # noqa: F401
from .common_util import ShouldFailMixin # noqa: F401
TEST_RSA_KEY_SIZE = 522
EMPTY_CLIENT_CONFIG = config_from_string(
b"/dev/null",
b"tub.port",
b""
"/dev/null",
"tub.port",
""
)
@ -249,8 +249,8 @@ class UseNode(object):
self.config = config_from_string(
self.basedir.asTextMode().path,
u"tub.port",
b"""
"tub.port",
"""
[node]
{node_config}
@ -781,6 +781,11 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
return filenode
class LoggingServiceParent(service.MultiService):
def log(self, *args, **kwargs):
return log.msg(*args, **kwargs)
TEST_DATA=b"\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1)

170
src/allmydata/test/common_py3.py
View File

@ -1,170 +0,0 @@
"""
Common utilities that have been ported to Python 3.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import unicode
import os
import time
import signal
from twisted.internet import defer, reactor
from twisted.application import service
from twisted.python import failure
from twisted.trial import unittest
from ..util.assertutil import precondition
from ..util.encodingutil import unicode_platform, get_filesystem_encoding
from ..util import log
class TimezoneMixin(object):
def setTimezone(self, timezone):
def tzset_if_possible():
# Windows doesn't have time.tzset().
if hasattr(time, 'tzset'):
time.tzset()
unset = object()
originalTimezone = os.environ.get('TZ', unset)
def restoreTimezone():
if originalTimezone is unset:
del os.environ['TZ']
else:
os.environ['TZ'] = originalTimezone
tzset_if_possible()
os.environ['TZ'] = timezone
self.addCleanup(restoreTimezone)
tzset_if_possible()
def have_working_tzset(self):
return hasattr(time, 'tzset')
class SignalMixin(object):
# This class is necessary for any code which wants to use Processes
# outside the usual reactor.run() environment. It is copied from
# Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
# something rather different.
sigchldHandler = None
def setUp(self):
# make sure SIGCHLD handler is installed, as it should be on
# reactor.run(). problem is reactor may not have been run when this
# test runs.
if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
self.sigchldHandler = signal.signal(signal.SIGCHLD,
reactor._handleSigchld)
return super(SignalMixin, self).setUp()
def tearDown(self):
if self.sigchldHandler:
signal.signal(signal.SIGCHLD, self.sigchldHandler)
return super(SignalMixin, self).tearDown()
class ShouldFailMixin(object):
def shouldFail(self, expected_failure, which, substring,
callable, *args, **kwargs):
"""Assert that a function call raises some exception. This is a
Deferred-friendly version of TestCase.assertRaises() .
Suppose you want to verify the following function:
def broken(a, b, c):
if a < 0:
raise TypeError('a must not be negative')
return defer.succeed(b+c)
You can use:
d = self.shouldFail(TypeError, 'test name',
'a must not be negative',
broken, -4, 5, c=12)
in your test method. The 'test name' string will be included in the
error message, if any, because Deferred chains frequently make it
difficult to tell which assertion was tripped.
The substring= argument, if not None, must appear in the 'repr'
of the message wrapped by this Failure, or the test will fail.
"""
assert substring is None or isinstance(substring, (bytes, unicode))
d = defer.maybeDeferred(callable, *args, **kwargs)
def done(res):
if isinstance(res, failure.Failure):
res.trap(expected_failure)
if substring:
self.failUnless(substring in str(res),
"%s: substring '%s' not in '%s'"
% (which, substring, str(res)))
# return the Failure for further analysis, but in a form that
# doesn't make the Deferred chain think that we failed.
return [res]
else:
self.fail("%s was supposed to raise %s, not get '%s'" %
(which, expected_failure, res))
d.addBoth(done)
return d
class ReallyEqualMixin(object):
def failUnlessReallyEqual(self, a, b, msg=None):
self.assertEqual(a, b, msg)
self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
def skip_if_cannot_represent_filename(u):
precondition(isinstance(u, unicode))
enc = get_filesystem_encoding()
if not unicode_platform():
try:
u.encode(enc)
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
class Marker(object):
pass
class FakeCanary(object):
"""For use in storage tests.
Can be moved back to test_storage.py once enough Python 3 porting has been
done.
"""
def __init__(self, ignore_disconnectors=False):
self.ignore = ignore_disconnectors
self.disconnectors = {}
def notifyOnDisconnect(self, f, *args, **kwargs):
if self.ignore:
return
m = Marker()
self.disconnectors[m] = (f, args, kwargs)
return m
def dontNotifyOnDisconnect(self, marker):
if self.ignore:
return
del self.disconnectors[marker]
def getRemoteTubID(self):
return None
def getPeer(self):
return "<fake>"
class LoggingServiceParent(service.MultiService):
def log(self, *args, **kwargs):
return log.msg(*args, **kwargs)

142
src/allmydata/test/common_util.py
View File

@ -1,22 +1,33 @@
from __future__ import print_function
import os
import time
import signal
from random import randrange
from six.moves import StringIO
from twisted.internet import reactor, defer
from twisted.python import failure
from twisted.trial import unittest
from ..util.assertutil import precondition
from ..scripts import runner
from allmydata.util.encodingutil import get_io_encoding
from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding
# Imported for backwards compatibility:
from future.utils import bord, bchr, binary_type
from .common_py3 import (
SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, ShouldFailMixin
)
from past.builtins import unicode
def skip_if_cannot_represent_filename(u):
precondition(isinstance(u, unicode))
enc = get_filesystem_encoding()
if not unicode_platform():
try:
u.encode(enc)
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
def skip_if_cannot_represent_argv(u):
precondition(isinstance(u, unicode))
try:
@ -78,6 +89,34 @@ def flip_one_bit(s, offset=0, size=None):
return result
class ReallyEqualMixin(object):
def failUnlessReallyEqual(self, a, b, msg=None):
self.assertEqual(a, b, msg)
self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
class SignalMixin(object):
# This class is necessary for any code which wants to use Processes
# outside the usual reactor.run() environment. It is copied from
# Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
# something rather different.
sigchldHandler = None
def setUp(self):
# make sure SIGCHLD handler is installed, as it should be on
# reactor.run(). problem is reactor may not have been run when this
# test runs.
if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
self.sigchldHandler = signal.signal(signal.SIGCHLD,
reactor._handleSigchld)
return super(SignalMixin, self).setUp()
def tearDown(self):
if self.sigchldHandler:
signal.signal(signal.SIGCHLD, self.sigchldHandler)
return super(SignalMixin, self).tearDown()
class StallMixin(object):
def stall(self, res=None, delay=1):
d = defer.Deferred()
@ -85,6 +124,76 @@ class StallMixin(object):
return d
class Marker(object):
pass
class FakeCanary(object):
"""For use in storage tests.
"""
def __init__(self, ignore_disconnectors=False):
self.ignore = ignore_disconnectors
self.disconnectors = {}
def notifyOnDisconnect(self, f, *args, **kwargs):
if self.ignore:
return
m = Marker()
self.disconnectors[m] = (f, args, kwargs)
return m
def dontNotifyOnDisconnect(self, marker):
if self.ignore:
return
del self.disconnectors[marker]
def getRemoteTubID(self):
return None
def getPeer(self):
return "<fake>"
class ShouldFailMixin(object):
def shouldFail(self, expected_failure, which, substring,
callable, *args, **kwargs):
"""Assert that a function call raises some exception. This is a
Deferred-friendly version of TestCase.assertRaises() .
Suppose you want to verify the following function:
def broken(a, b, c):
if a < 0:
raise TypeError('a must not be negative')
return defer.succeed(b+c)
You can use:
d = self.shouldFail(TypeError, 'test name',
'a must not be negative',
broken, -4, 5, c=12)
in your test method. The 'test name' string will be included in the
error message, if any, because Deferred chains frequently make it
difficult to tell which assertion was tripped.
The substring= argument, if not None, must appear in the 'repr'
of the message wrapped by this Failure, or the test will fail.
"""
assert substring is None or isinstance(substring, (bytes, unicode))
d = defer.maybeDeferred(callable, *args, **kwargs)
def done(res):
if isinstance(res, failure.Failure):
res.trap(expected_failure)
if substring:
self.failUnless(substring in str(res),
"%s: substring '%s' not in '%s'"
% (which, substring, str(res)))
# return the Failure for further analysis, but in a form that
# doesn't make the Deferred chain think that we failed.
return [res]
else:
self.fail("%s was supposed to raise %s, not get '%s'" %
(which, expected_failure, res))
d.addBoth(done)
return d
class TestMixin(SignalMixin):
def setUp(self):
return super(TestMixin, self).setUp()
@ -132,6 +241,31 @@ class TestMixin(SignalMixin):
self.fail("Reactor was still active when it was required to be quiescent.")
class TimezoneMixin(object):
def setTimezone(self, timezone):
def tzset_if_possible():
# Windows doesn't have time.tzset().
if hasattr(time, 'tzset'):
time.tzset()
unset = object()
originalTimezone = os.environ.get('TZ', unset)
def restoreTimezone():
if originalTimezone is unset:
del os.environ['TZ']
else:
os.environ['TZ'] = originalTimezone
tzset_if_possible()
os.environ['TZ'] = timezone
self.addCleanup(restoreTimezone)
tzset_if_possible()
def have_working_tzset(self):
return hasattr(time, 'tzset')
try:
import win32file
import win32con

4
src/allmydata/test/eliotutil.py
View File

@ -2,7 +2,7 @@
Tools aimed at the interaction between tests and Eliot.
"""
# BBB: Python 2 compatibility
# Python 2 compatibility
# Can't use `builtins.str` because it's not JSON encodable:
# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
from past.builtins import unicode as str
@ -106,7 +106,7 @@ def eliot_logged_test(f):
# Begin an action that should comprise all messages from the decorated
# test method.
with RUN_TEST(name=self.id().decode("utf-8")).context() as action:
with RUN_TEST(name=self.id()).context() as action:
# When the test method Deferred fires, the RUN_TEST action is
# done. However, we won't have re-published the MemoryLogger
# messages into the global/default logger when this Deferred

2
src/allmydata/test/mutable/test_version.py
View File

@ -2,7 +2,7 @@ from __future__ import print_function
import os
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

2
src/allmydata/test/test_auth.py
View File

@ -1,4 +1,4 @@
# BBB: Python 2 compatibility
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401

120
src/allmydata/test/test_client.py
View File

@ -252,11 +252,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
is not set.
"""
config = client.config_from_string(
b"test_storage_default_anonymous_enabled",
b"tub.port",
"test_storage_default_anonymous_enabled",
"tub.port",
BASECONFIG + (
b"[storage]\n"
b"enabled = true\n"
"[storage]\n"
"enabled = true\n"
)
)
self.assertTrue(client.anonymous_storage_enabled(config))
@ -268,11 +268,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
"""
config = client.config_from_string(
self.id(),
b"tub.port",
"tub.port",
BASECONFIG + (
b"[storage]\n"
b"enabled = true\n"
b"anonymous = true\n"
"[storage]\n"
"enabled = true\n"
"anonymous = true\n"
)
)
self.assertTrue(client.anonymous_storage_enabled(config))
@ -284,11 +284,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
"""
config = client.config_from_string(
self.id(),
b"tub.port",
"tub.port",
BASECONFIG + (
b"[storage]\n"
b"enabled = true\n"
b"anonymous = false\n"
"[storage]\n"
"enabled = true\n"
"anonymous = false\n"
)
)
self.assertFalse(client.anonymous_storage_enabled(config))
@ -300,11 +300,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
"""
config = client.config_from_string(
self.id(),
b"tub.port",
"tub.port",
BASECONFIG + (
b"[storage]\n"
b"enabled = false\n"
b"anonymous = true\n"
"[storage]\n"
"enabled = false\n"
"anonymous = true\n"
)
)
self.assertFalse(client.anonymous_storage_enabled(config))
@ -680,11 +680,11 @@ class AnonymousStorage(SyncTestCase):
os.makedirs(basedir + b"/private")
config = client.config_from_string(
basedir,
b"tub.port",
"tub.port",
BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n"
b"enabled = true\n"
b"anonymous = true\n"
"[storage]\n"
"enabled = true\n"
"anonymous = true\n"
)
)
node = yield client.create_client_from_config(
@ -711,11 +711,11 @@ class AnonymousStorage(SyncTestCase):
os.makedirs(basedir + b"/private")
config = client.config_from_string(
basedir,
b"tub.port",
"tub.port",
BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n"
b"enabled = true\n"
b"anonymous = false\n"
"[storage]\n"
"enabled = true\n"
"anonymous = false\n"
)
)
node = yield client.create_client_from_config(
@ -732,7 +732,7 @@ class AnonymousStorage(SyncTestCase):
]),
)
self.expectThat(
config.get_private_config(b"storage.furl", default=None),
config.get_private_config("storage.furl", default=None),
Is(None),
)
@ -748,18 +748,18 @@ class AnonymousStorage(SyncTestCase):
os.makedirs(basedir + b"/private")
enabled_config = client.config_from_string(
basedir,
b"tub.port",
"tub.port",
BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n"
b"enabled = true\n"
b"anonymous = true\n"
"[storage]\n"
"enabled = true\n"
"anonymous = true\n"
)
)
node = yield client.create_client_from_config(
enabled_config,
_introducer_factory=MemoryIntroducerClient,
)
anonymous_storage_furl = enabled_config.get_private_config(b"storage.furl")
anonymous_storage_furl = enabled_config.get_private_config("storage.furl")
def check_furl():
return node.tub.getReferenceForURL(anonymous_storage_furl)
# Perform a sanity check that our test code makes sense: is this a
@ -772,11 +772,11 @@ class AnonymousStorage(SyncTestCase):
disabled_config = client.config_from_string(
basedir,
b"tub.port",
"tub.port",
BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n"
b"enabled = true\n"
b"anonymous = false\n"
"[storage]\n"
"enabled = true\n"
"anonymous = false\n"
)
)
node = yield client.create_client_from_config(
@ -1137,8 +1137,8 @@ class StorageAnnouncementTests(SyncTestCase):
create_node_dir(self.basedir, u"")
def get_config(self, storage_enabled, more_storage=b"", more_sections=b""):
return b"""
def get_config(self, storage_enabled, more_storage="", more_sections=""):
return """
[node]
tub.location = tcp:192.0.2.0:1234
@ -1163,7 +1163,7 @@ introducer.furl = pb://abcde@nowhere/fake
"""
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(storage_enabled=False),
)
self.assertThat(
@ -1185,7 +1185,7 @@ introducer.furl = pb://abcde@nowhere/fake
"""
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(storage_enabled=True),
)
client_deferred = client.create_client_from_config(
@ -1217,13 +1217,13 @@ introducer.furl = pb://abcde@nowhere/fake
value = u"thing"
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(
storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1",
more_storage="plugins=tahoe-lafs-dummy-v1",
more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
b"some = {}\n".format(value)
"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
"some = {}\n".format(value)
),
),
)
@ -1258,15 +1258,15 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(
storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2",
more_storage="plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2",
more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
b"some = thing-1\n"
b"[storageserver.plugins.tahoe-lafs-dummy-v2]\n"
b"some = thing-2\n"
"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
"some = thing-1\n"
"[storageserver.plugins.tahoe-lafs-dummy-v2]\n"
"some = thing-2\n"
),
),
)
@ -1306,13 +1306,13 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(
storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1",
more_storage="plugins=tahoe-lafs-dummy-v1",
more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
b"some = thing\n"
"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
"some = thing\n"
),
),
)
@ -1342,10 +1342,10 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(
storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1",
more_storage="plugins=tahoe-lafs-dummy-v1",
),
)
self.assertThat(
@ -1380,14 +1380,14 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(
storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1",
more_storage="plugins=tahoe-lafs-dummy-v1",
more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
"[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
# This will make it explode on instantiation.
b"invalid = configuration\n"
"invalid = configuration\n"
)
),
)
@ -1407,10 +1407,10 @@ introducer.furl = pb://abcde@nowhere/fake
"""
config = client.config_from_string(
self.basedir,
u"tub.port",
"tub.port",
self.get_config(
storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-vX",
more_storage="plugins=tahoe-lafs-dummy-vX",
),
)
self.assertThat(

3
src/allmydata/test/test_crawler.py
View File

@ -27,8 +27,7 @@ from allmydata.util import fileutil, hashutil, pollmixin
from allmydata.storage.server import StorageServer, si_b2a
from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded
from allmydata.test.common_py3 import FakeCanary
from allmydata.test.common_util import StallMixin
from allmydata.test.common_util import StallMixin, FakeCanary
class BucketEnumeratingCrawler(ShareCrawler):
cpu_slice = 500 # make sure it can complete in a single slice

2
src/allmydata/test/test_deepcheck.py
View File

@ -1,6 +1,6 @@
import os, json, urllib
# BBB: Python 2 compatibility
# Python 2 compatibility
# Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
# a `future.types.newstr.newstr` as a *Python 3* byte string representation under
# *Python 2*:

2
src/allmydata/test/test_encodingutil.py
View File

@ -77,7 +77,7 @@ from twisted.trial import unittest
from twisted.python.filepath import FilePath
from allmydata.test.common_py3 import (
from allmydata.test.common_util import (
ReallyEqualMixin, skip_if_cannot_represent_filename,
)
from allmydata.util import encodingutil, fileutil

2
src/allmydata/test/test_happiness.py
View File

@ -23,7 +23,7 @@ from hypothesis.strategies import text, sets
from allmydata.immutable import happiness_upload
from allmydata.util.happinessutil import servers_of_happiness, \
shares_by_server, merge_servers
from allmydata.test.common_py3 import ShouldFailMixin
from allmydata.test.common import ShouldFailMixin
class HappinessUploadUtils(unittest.TestCase):

2
src/allmydata/test/test_iputil.py
View File

@ -23,7 +23,7 @@ from tenacity import retry, stop_after_attempt
from foolscap.api import Tub
from allmydata.util import iputil, gcutil
import allmydata.test.common_py3 as testutil
import allmydata.test.common_util as testutil
from allmydata.util.namespace import Namespace

4
src/allmydata/test/test_node.py
View File

@ -520,7 +520,6 @@ introducer.furl = empty
enabled = false
[i2p]
enabled = false
[node]
"""
NOLISTEN = """
@ -566,6 +565,7 @@ class Listeners(unittest.TestCase):
create_node_dir(basedir, "testing")
with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
f.write(BASE_CONFIG)
f.write("[node]\n")
f.write("tub.port = tcp:0\n")
f.write("tub.location = AUTO\n")
@ -594,6 +594,7 @@ class Listeners(unittest.TestCase):
location = "tcp:localhost:%d,tcp:localhost:%d" % (port1, port2)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
f.write(BASE_CONFIG)
f.write("[node]\n")
f.write("tub.port = %s\n" % port)
f.write("tub.location = %s\n" % location)
@ -617,6 +618,7 @@ class Listeners(unittest.TestCase):
os.mkdir(os.path.join(basedir, "private"))
with open(config_fname, "w") as f:
f.write(BASE_CONFIG)
f.write("[node]\n")
f.write("tub.port = listen:i2p,listen:tor\n")
f.write("tub.location = tcp:example.org:1234\n")
config = client.read_config(basedir, "client.port")

3
src/allmydata/test/test_storage.py
View File

@ -51,7 +51,8 @@ from allmydata.test.no_network import NoNetworkServer
from allmydata.storage_client import (
_StorageServer,
)
from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin
from .common import LoggingServiceParent, ShouldFailMixin
from .common_util import FakeCanary
class UtilTests(unittest.TestCase):

2
src/allmydata/test/test_storage_web.py
View File

@ -50,7 +50,7 @@ from allmydata.web.storage import (
StorageStatusElement,
remove_prefix
)
from .common_py3 import FakeCanary
from .common_util import FakeCanary
def remove_tags(s):
s = re.sub(br'<[^>]*>', b' ', s)

2
src/allmydata/test/test_time_format.py
View File

@ -16,7 +16,7 @@ import time
from twisted.trial import unittest
from allmydata.test.common_py3 import TimezoneMixin
from allmydata.test.common_util import TimezoneMixin
from allmydata.util import time_format

2
src/allmydata/test/test_upload.py
View File

@ -28,12 +28,12 @@ from allmydata.util import log, base32
from allmydata.util.assertutil import precondition
from allmydata.util.deferredutil import DeferredListShouldSucceed
from allmydata.test.no_network import GridTestMixin
from allmydata.test.common_py3 import ShouldFailMixin
from allmydata.storage_client import StorageFarmBroker
from allmydata.storage.server import storage_index_to_dir
from allmydata.client import _Client
from .common import (
EMPTY_CLIENT_CONFIG,
ShouldFailMixin,
)
from functools import reduce

2
src/allmydata/test/web/test_introducer.py
View File

@ -33,7 +33,7 @@ from .common import (
from ..common import (
SameProcessStreamEndpointAssigner,
)
from ..common_py3 import (
from ..common_util import (
FakeCanary,
)
from ..common_web import (

2
src/allmydata/test/web/test_web.py
View File

@ -52,7 +52,7 @@ from allmydata.interfaces import (
)
from allmydata.mutable import servermap, publish, retrieve
from .. import common_util as testutil
from ..common_py3 import TimezoneMixin
from ..common_util import TimezoneMixin
from ..common_web import (
do_http,
Error,

6
src/allmydata/util/_python3.py
View File

@ -24,6 +24,7 @@ if PY2:
# Keep these sorted alphabetically, to reduce merge conflicts:
PORTED_MODULES = [
"allmydata._monkeypatch",
"allmydata.codec",
"allmydata.crypto",
"allmydata.crypto.aes",
@ -40,8 +41,12 @@ PORTED_MODULES = [
"allmydata.immutable.downloader.segmentation",
"allmydata.immutable.downloader.share",
"allmydata.immutable.downloader.status",
"allmydata.immutable.encode",
"allmydata.immutable.filenode",
"allmydata.immutable.happiness_upload",
"allmydata.immutable.layout",
"allmydata.immutable.literal",
"allmydata.immutable.upload",
"allmydata.interfaces",
"allmydata.introducer.interfaces",
"allmydata.monitor",
@ -53,7 +58,6 @@ PORTED_MODULES = [
"allmydata.storage.mutable",
"allmydata.storage.server",
"allmydata.storage.shares",
"allmydata.test.common_py3",
"allmydata.test.no_network",
"allmydata.uri",
"allmydata.util._python3",