From 0d093c45df583bed9400004bebd2f3becaa7523d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 May 2021 10:06:12 -0400
Subject: [PATCH 01/37] Test fails on Python 3, instead of spinning until
 timeout.

---
 integration/conftest.py                  | 2 +-
 integration/test_servers_of_happiness.py | 2 +-
 integration/util.py                      | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 533cbdb67..918f2d4c9 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -28,7 +28,7 @@ from twisted.internet.error import (
 import pytest
 import pytest_twisted
 
-from util import (
+from .util import (
     _CollectOutputProtocol,
     _MagicTextProtocol,
     _DumpOutputProtocol,
diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py
index 1f350eb8e..f8f69b1cc 100644
--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@@ -3,7 +3,7 @@ from os.path import join
 
 from twisted.internet.error import ProcessTerminated
 
-import util
+from . import util
 
 import pytest_twisted
 
diff --git a/integration/util.py b/integration/util.py
index cd7f15e84..2052d2ae6 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -116,6 +116,7 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
@@ -123,6 +124,7 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
+        data = unicode(data, sys.stderr.encoding)
         sys.stdout.write(data)
 
 

From 6df076dc10b22e4b42905875df33a929b95937e1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 May 2021 10:16:46 -0400
Subject: [PATCH 02/37] Bit more progress.

---
 integration/util.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index 2052d2ae6..bb2f9e8e4 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -5,7 +5,7 @@ import time
 import json
 from os import mkdir, environ
 from os.path import exists, join
-from six.moves import StringIO
+from io import StringIO, BytesIO
 from functools import partial
 from subprocess import check_output
 
@@ -59,7 +59,7 @@ class _CollectOutputProtocol(ProcessProtocol):
     """
     def __init__(self):
         self.done = Deferred()
-        self.output = StringIO()
+        self.output = BytesIO()
 
     def processEnded(self, reason):
         if not self.done.called:
@@ -73,7 +73,7 @@ class _CollectOutputProtocol(ProcessProtocol):
         self.output.write(data)
 
     def errReceived(self, data):
-        print("ERR: {}".format(data))
+        print("ERR: {!r}".format(data))
         self.output.write(data)
 
 
@@ -94,9 +94,11 @@ class _DumpOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         self._out.write(data)
 
     def errReceived(self, data):
+        data = unicode(data, sys.stdout.encoding)
         self._out.write(data)
 
 
@@ -284,7 +286,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
                 config,
                 u'node',
                 u'log_gatherer.furl',
-                flog_gatherer.decode("utf-8"),
+                flog_gatherer,
             )
             write_config(FilePath(config_path), config)
         created_d.addCallback(created)

From b11cc9137b87c6ca443b7244f4de4353be5bd250 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 May 2021 10:48:50 -0400
Subject: [PATCH 03/37] Integration test passed on Python 3.

---
 integration/test_servers_of_happiness.py | 2 +-
 src/allmydata/web/root.py                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py
index f8f69b1cc..594397999 100644
--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@@ -42,4 +42,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
         assert isinstance(e, ProcessTerminated)
 
     output = proto.output.getvalue()
-    assert "shares could be placed on only" in output
+    assert b"shares could be placed on only" in output
diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index b7dc8b5f4..1debc1d10 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -318,7 +318,7 @@ class Root(MultiFormatResource):
         }
         version = server.get_version()
         if version is not None:
-            description[u"version"] = version["application-version"]
+            description[u"version"] = version[b"application-version"]
 
         return description
 

From 04fc8e704650e1758a2eac6e5f146092aa04f310 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 May 2021 13:21:23 -0400
Subject: [PATCH 04/37] Port to Python 3.

---
 integration/test_servers_of_happiness.py | 12 ++++++++++++
 src/allmydata/util/_python3.py           |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py
index 594397999..b9de0c075 100644
--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import sys
 from os.path import join
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 5f78af626..41270b430 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -22,6 +22,12 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+
+PORTED_INTEGRATION_TESTS = [
+    "integration.test_servers_of_happiness",
+]
+
+
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
     "allmydata",

From add20d802430113c949c89d6bdccac8d463b653f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 May 2021 13:42:38 -0400
Subject: [PATCH 05/37] First pass at integration support on Python 3.

---
 src/allmydata/util/_python3.py |  3 +++
 tox.ini                        | 13 ++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 41270b430..c265424c6 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -23,6 +23,9 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
+# Every time a module is added here, also add it to tox.ini environment
+# integrations3. Bit of duplication, but it's only a handful of files so quite
+# temporary.
 PORTED_INTEGRATION_TESTS = [
     "integration.test_servers_of_happiness",
 ]
diff --git a/tox.ini b/tox.ini
index a6e5e0cc3..6a0c628d7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,7 @@
 [gh-actions]
 python =
     2.7: py27-coverage,codechecks
-    3.6: py36-coverage
+    3.6: py36-coverage,integration3
     3.7: py37-coverage
     3.8: py38-coverage
     3.9: py39-coverage,typechecks,codechecks3
@@ -97,6 +97,17 @@ commands =
          coverage report
 
 
+[testenv:integration3]
+basepython = python3
+setenv =
+         COVERAGE_PROCESS_START=.coveragerc
+commands =
+         # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+         py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py}
+         coverage combine
+         coverage report
+
+
 [testenv:codechecks]
 basepython = python2.7
 # On macOS, git inside of towncrier needs $HOME.

From 01baa13fca6f71931682210dfa1b7170207244c5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 10:10:02 -0400
Subject: [PATCH 06/37] News file

---
 newsfragments/3703.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3703.minor

diff --git a/newsfragments/3703.minor b/newsfragments/3703.minor
new file mode 100644
index 000000000..e69de29bb

From 7c2f9e7d8c1e5e02d5ee46ded0971b891d7bb26d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 10:19:21 -0400
Subject: [PATCH 07/37] Fix test to match real-world types.

---
 src/allmydata/test/web/test_web.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index e73cc12f8..1c9d6b65c 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -218,7 +218,7 @@ class FakeDisplayableServer(StubServer):  # type: ignore  # tahoe-lafs/ticket/35
         return self.connected
     def get_version(self):
         return {
-            "application-version": "1.0"
+            b"application-version": b"1.0"
         }
     def get_permutation_seed(self):
         return b""

From 886c5007b5746fef93f2153e6231ce159d02b8a3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 10:38:14 -0400
Subject: [PATCH 08/37] News file.

---
 newsfragments/3707.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3707.minor

diff --git a/newsfragments/3707.minor b/newsfragments/3707.minor
new file mode 100644
index 000000000..e69de29bb

From 1257fc18c8816e024d519f1db6ad9bae79feccc0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 11:10:52 -0400
Subject: [PATCH 09/37] Additional test coverage and corresponding bug fixes
 for password auth on Python 3.

---
 src/allmydata/frontends/auth.py |  2 +-
 src/allmydata/test/test_auth.py | 25 ++++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py
index f2ac99b8f..376cdebe4 100644
--- a/src/allmydata/frontends/auth.py
+++ b/src/allmydata/frontends/auth.py
@@ -80,5 +80,5 @@ class AccountFileChecker(object):
             return defer.fail(error.UnauthorizedLogin())
 
         d = defer.maybeDeferred(creds.checkPassword, correct)
-        d.addCallback(self._cbPasswordMatch, str(creds.username))
+        d.addCallback(self._cbPasswordMatch, creds.username)
         return d
diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py
index f808f72ab..4922ca5f3 100644
--- a/src/allmydata/test/test_auth.py
+++ b/src/allmydata/test/test_auth.py
@@ -65,7 +65,7 @@ class AccountFileCheckerKeyTests(unittest.TestCase):
         avatarId = self.checker.requestAvatarId(key_credentials)
         return self.assertFailure(avatarId, error.UnauthorizedLogin)
 
-    def test_password_auth_user(self):
+    def test_password_auth_user_with_ssh_key(self):
         """
         AccountFileChecker.requestAvatarId returns a Deferred that fires with
         UnauthorizedLogin if called with an SSHPrivateKey object for a username
@@ -76,6 +76,29 @@ class AccountFileCheckerKeyTests(unittest.TestCase):
         avatarId = self.checker.requestAvatarId(key_credentials)
         return self.assertFailure(avatarId, error.UnauthorizedLogin)
 
+    def test_password_auth_user_with_correct_password(self):
+        """
+        AccountFileChecker.requestAvatarId returns a Deferred that fires with
+        the user if the correct password is given.
+        """
+        key_credentials = credentials.UsernamePassword(b"alice", b"password")
+        d = self.checker.requestAvatarId(key_credentials)
+        def authenticated(avatarId):
+            self.assertEqual(
+                (b"alice",
+                 b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"),
+                (avatarId.username, avatarId.rootcap))
+        return d
+
+    def test_password_auth_user_with_wrong_password(self):
+        """
+        AccountFileChecker.requestAvatarId returns a Deferred that fires with
+        UnauthorizedLogin if the wrong password is given.
+        """
+        key_credentials = credentials.UsernamePassword(b"alice", b"WRONG")
+        avatarId = self.checker.requestAvatarId(key_credentials)
+        return self.assertFailure(avatarId, error.UnauthorizedLogin)
+
     def test_unrecognized_key(self):
         """
         AccountFileChecker.requestAvatarId returns a Deferred that fires with

From 1c6a324a9258d5a5ad58601696dc70e27c16ba20 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 11:11:11 -0400
Subject: [PATCH 10/37] Fix a byte-to-string bug on Python 3.

---
 integration/util.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration/util.py b/integration/util.py
index bb2f9e8e4..4f3c40666 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -532,7 +532,8 @@ def generate_ssh_key(path):
     key = RSAKey.generate(2048)
     key.write_private_key_file(path)
     with open(path + ".pub", "wb") as f:
-        f.write(b"%s %s" % (key.get_name(), key.get_base64()))
+        s = "%s %s" % (key.get_name(), key.get_base64())
+        f.write(s.encode("ascii"))
 
 
 def run_in_thread(f):

From 430bc51e2979937b8c282ce411563494c6f81380 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 11:41:59 -0400
Subject: [PATCH 11/37] SFTP integration tests pass on Python 3.

---
 src/allmydata/frontends/sftpd.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py
index 17eca993e..d2d614c77 100644
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@@ -1011,8 +1011,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
         PrefixingLogMixin.__init__(self, facility="tahoe.sftp", prefix=username)
         if noisy: self.log(".__init__(%r, %r, %r)" % (client, rootnode, username), level=NOISY)
 
-        self.channelLookup["session"] = session.SSHSession
-        self.subsystemLookup["sftp"] = FileTransferServer
+        self.channelLookup[b"session"] = session.SSHSession
+        self.subsystemLookup[b"sftp"] = FileTransferServer
 
         self._client = client
         self._root = rootnode

From 02e699347ed7d53508fea773272ad78d69e150f8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 11:49:21 -0400
Subject: [PATCH 12/37] Mark tests as ported.

---
 src/allmydata/util/_python3.py | 1 +
 tox.ini                        | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f821045d5..17e74aa4f 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -22,6 +22,7 @@ if PY2:
 # temporary.
 PORTED_INTEGRATION_TESTS = [
     "integration.test_servers_of_happiness",
+    "integration.test_sftp",
 ]
 
 
diff --git a/tox.ini b/tox.ini
index b8deeeb60..32252a346 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,7 +102,7 @@ setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py}
+         py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py integration/test_sftp.py}
          coverage combine
          coverage report
 

From 37bcaf6c88f3f42ac580e36590759a1109cc3c94 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 11:51:41 -0400
Subject: [PATCH 13/37] Port initialization "tests".

---
 integration/test_aaa_aardvark.py | 9 +++++++++
 src/allmydata/util/_python3.py   | 1 +
 tox.ini                          | 2 +-
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py
index 4a2ef71a6..28ac4c412 100644
--- a/integration/test_aaa_aardvark.py
+++ b/integration/test_aaa_aardvark.py
@@ -5,6 +5,15 @@
 # You can safely skip any of these tests, it'll just appear to "take
 # longer" to start the first test as the fixtures get built
 
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 
 def test_create_flogger(flog_gatherer):
     print("Created flog_gatherer")
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 17e74aa4f..9b166f5e0 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -21,6 +21,7 @@ if PY2:
 # integrations3. Bit of duplication, but it's only a handful of files so quite
 # temporary.
 PORTED_INTEGRATION_TESTS = [
+    "integration.test_aaa_aardvark",
     "integration.test_servers_of_happiness",
     "integration.test_sftp",
 ]
diff --git a/tox.ini b/tox.ini
index 32252a346..fac52ad43 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,7 +102,7 @@ setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py integration/test_sftp.py}
+         py.test --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py}
          coverage combine
          coverage report
 

From 109829dfcba9a0d840f9a2f43dc5174f1909f3fa Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 11:58:29 -0400
Subject: [PATCH 14/37] BytesWarnings.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index fac52ad43..7f438ef2d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,7 +102,7 @@ setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         py.test --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py}
+         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py}
          coverage combine
          coverage report
 

From 3f2571480c49d40dc1c54e5ee645d95ddf573135 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 12:03:58 -0400
Subject: [PATCH 15/37] Some progress towards passing on Python 3.

---
 integration/test_streaming_logs.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py
index 52c813f9b..e5a3acedd 100644
--- a/integration/test_streaming_logs.py
+++ b/integration/test_streaming_logs.py
@@ -5,12 +5,14 @@ from __future__ import (
     division,
 )
 
+from six import ensure_text
+
 import json
 
 from os.path import (
     join,
 )
-from urlparse import (
+from urllib.parse import (
     urlsplit,
 )
 
@@ -68,7 +70,7 @@ def _connect_client(reactor, api_auth_token, ws_url):
     factory = WebSocketClientFactory(
         url=ws_url,
         headers={
-            "Authorization": "{} {}".format(SCHEME, api_auth_token),
+            "Authorization": "{} {}".format(str(SCHEME, "ascii"), api_auth_token),
         }
     )
     factory.protocol = _StreamingLogClientProtocol
@@ -127,7 +129,7 @@ def _test_streaming_logs(reactor, temp_dir, alice):
     node_url = cfg.get_config_from_file("node.url")
     api_auth_token = cfg.get_private_config("api_auth_token")
 
-    ws_url = node_url.replace("http://", "ws://")
+    ws_url = ensure_text(node_url).replace("http://", "ws://")
     log_url = ws_url + "private/logs/v1"
 
     print("Connecting to {}".format(log_url))

From d80b6d6c56956dc1e44bd3086ba8cfaa073669cf Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 May 2021 13:18:08 -0400
Subject: [PATCH 16/37] Add a couple missing environments.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 6a0c628d7..8ffba7713 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,7 +17,7 @@ python =
 twisted = 1
 
 [tox]
-envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3
+envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3
 minversion = 2.4
 
 [testenv]

From 9baedc97dbdc827198dcd461ef8d47033bd3d4fe Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 11:09:59 -0400
Subject: [PATCH 17/37] Auth token needs to be bytes, apparently.

---
 src/allmydata/client.py           | 3 ++-
 src/allmydata/test/test_client.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index a6c45643f..aabae9065 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -928,7 +928,8 @@ class _Client(node.Node, pollmixin.PollMixin):
         random data in "api_auth_token" which must be echoed to API
         calls.
         """
-        return self.config.get_private_config('api_auth_token')
+        return self.config.get_private_config(
+            'api_auth_token').encode("ascii")
 
     def _create_auth_token(self):
         """
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 957be5197..fd2837f1d 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -415,7 +415,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
             f.write("deadbeef")
 
         token = c.get_auth_token()
-        self.assertEqual("deadbeef", token)
+        self.assertEqual(b"deadbeef", token)
 
     @defer.inlineCallbacks
     def test_web_staticdir(self):

From b8b00fa2da2914020e7fe72ccb9767a04d918e2a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 11:13:52 -0400
Subject: [PATCH 18/37] Port to Python 3.

---
 integration/test_streaming_logs.py | 7 +++++++
 src/allmydata/util/_python3.py     | 5 +++--
 tox.ini                            | 2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py
index e5a3acedd..036d30715 100644
--- a/integration/test_streaming_logs.py
+++ b/integration/test_streaming_logs.py
@@ -1,3 +1,6 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import (
     print_function,
     unicode_literals,
@@ -5,6 +8,10 @@ from __future__ import (
     division,
 )
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from six import ensure_text
 
 import json
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 9b166f5e0..4bbfc0e68 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -18,12 +18,13 @@ if PY2:
 
 
 # Every time a module is added here, also add it to tox.ini environment
-# integrations3. Bit of duplication, but it's only a handful of files so quite
-# temporary.
+# integrations3. Bit of duplication, but it's only a handful of files and quite
+# temporary, just until we've ported them all.
 PORTED_INTEGRATION_TESTS = [
     "integration.test_aaa_aardvark",
     "integration.test_servers_of_happiness",
     "integration.test_sftp",
+    "integration.test_streaming_logs",
 ]
 
 
diff --git a/tox.ini b/tox.ini
index 7f438ef2d..e6b745491 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,7 +102,7 @@ setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py}
+         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py}
          coverage combine
          coverage report
 

From 9b6067dcd1d05f8e4296c83157bd9447df1c0699 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 11:39:32 -0400
Subject: [PATCH 19/37] Make test match reality, and fix corresponding bug.

---
 src/allmydata/test/test_tor_provider.py | 4 +++-
 src/allmydata/util/tor_provider.py      | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py
index 148d813f5..86d54803a 100644
--- a/src/allmydata/test/test_tor_provider.py
+++ b/src/allmydata/test/test_tor_provider.py
@@ -14,6 +14,7 @@ import os
 from twisted.trial import unittest
 from twisted.internet import defer, error
 from six.moves import StringIO
+from six import ensure_str
 import mock
 from ..util import tor_provider
 from ..scripts import create_node, runner
@@ -185,7 +186,8 @@ class CreateOnion(unittest.TestCase):
                                                            protocol)))
         txtorcon = mock.Mock()
         ehs = mock.Mock()
-        ehs.private_key = b"privkey"
+        # This appears to be a native string in the real txtorcon object...
+        ehs.private_key = ensure_str("privkey")
         ehs.hostname = "ONION.onion"
         txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs)
         ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py
index c4c63f61a..4ca19c01c 100644
--- a/src/allmydata/util/tor_provider.py
+++ b/src/allmydata/util/tor_provider.py
@@ -211,6 +211,8 @@ def create_config(reactor, cli_config):
                                                               "tor_onion.privkey")
     privkeyfile = os.path.join(private_dir, "tor_onion.privkey")
     with open(privkeyfile, "wb") as f:
+        if isinstance(privkey, str):
+            privkey = privkey.encode("ascii")
         f.write(privkey)
 
     # tahoe_config_tor: this is a dictionary of keys/values to add to the

From 070691caa2a8f204eeda04f815162f2f9baa4c79 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 11:54:30 -0400
Subject: [PATCH 20/37] Make sure it always goes to stderr.

---
 src/allmydata/scripts/runner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index 7122e499e..f63b9b330 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -185,8 +185,8 @@ def _maybe_enable_eliot_logging(options, reactor):
 
 def run():
     if six.PY3:
-        warnings.warn("Support for Python 3 is an incomplete work-in-progress."
-                      " Use at your own risk.")
+        print("Support for Python 3 is an incomplete work-in-progress."
+              " Use at your own risk.", file=sys.stderr)
 
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize

From 4043b2fe1fffdef95fb6efa449cd1b5f20dc0dce Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 11:56:21 -0400
Subject: [PATCH 21/37] Tests pass on Python 3.

---
 integration/test_tor.py | 12 +++++++-----
 integration/util.py     |  6 ++++--
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 3b374f669..bea48ba5d 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -6,7 +6,7 @@ from os.path import join
 import pytest
 import pytest_twisted
 
-import util
+from . import util
 
 from twisted.python.filepath import (
     FilePath,
@@ -55,7 +55,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     cap = proto.output.getvalue().strip().split()[-1]
     print("TEH CAP!", cap)
 
-    proto = util._CollectOutputProtocol()
+    proto = util._CollectOutputProtocol(capture_stderr=False)
     reactor.spawnProcess(
         proto,
         sys.executable,
@@ -68,7 +68,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     yield proto.done
 
     dave_got = proto.output.getvalue().strip()
-    assert dave_got == open(gold_path, 'r').read().strip()
+    assert dave_got == open(gold_path, 'rb').read().strip()
 
 
 @pytest_twisted.inlineCallbacks
@@ -100,7 +100,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     # Which services should this client connect to?
     write_introducer(node_dir, "default", introducer_furl)
     with node_dir.child('tahoe.cfg').open('w') as f:
-        f.write('''
+        node_config = '''
 [node]
 nickname = %(name)s
 web.port = %(web_port)s
@@ -125,7 +125,9 @@ shares.total = 2
     'log_furl': flog_gatherer,
     'control_port': control_port,
     'local_port': control_port + 1000,
-})
+}
+        node_config = node_config.encode("utf-8")
+        f.write(node_config)
 
     print("running")
     yield util._run_node(reactor, node_dir.path, request, None)
diff --git a/integration/util.py b/integration/util.py
index 4f3c40666..a233471c7 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -57,9 +57,10 @@ class _CollectOutputProtocol(ProcessProtocol):
     self.output, and callback's on done with all of it after the
     process exits (for any reason).
     """
-    def __init__(self):
+    def __init__(self, capture_stderr=True):
         self.done = Deferred()
         self.output = BytesIO()
+        self.capture_stderr = capture_stderr
 
     def processEnded(self, reason):
         if not self.done.called:
@@ -74,7 +75,8 @@ class _CollectOutputProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         print("ERR: {!r}".format(data))
-        self.output.write(data)
+        if self.capture_err:
+            self.output.write(data)
 
 
 class _DumpOutputProtocol(ProcessProtocol):

From 02897b9968ef5b1e341c8d0600b2b7dd8fde1be6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 13:21:47 -0400
Subject: [PATCH 22/37] Fix typo.

---
 integration/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/util.py b/integration/util.py
index a233471c7..c8ba5d16b 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -75,7 +75,7 @@ class _CollectOutputProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         print("ERR: {!r}".format(data))
-        if self.capture_err:
+        if self.capture_stderr:
             self.output.write(data)
 
 

From fba23ef91cc95fdc0da8429fd38fed802c96c9fc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 13:34:35 -0400
Subject: [PATCH 23/37] Port to Python 3.

---
 integration/test_tor.py        | 10 ++++++++++
 src/allmydata/util/_python3.py |  1 +
 tox.ini                        |  2 +-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index bea48ba5d..15d888e36 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -1,5 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import sys
 from os.path import join
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 4bbfc0e68..2bde5941e 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -25,6 +25,7 @@ PORTED_INTEGRATION_TESTS = [
     "integration.test_servers_of_happiness",
     "integration.test_sftp",
     "integration.test_streaming_logs",
+    "integration.test_tor",
 ]
 
 
diff --git a/tox.ini b/tox.ini
index 7d8e90040..04eae7833 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,7 +102,7 @@ setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py}
+         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py integration/test_tor.py}
          coverage combine
          coverage report
 

From d517304a7914ee3a73ced2462d2743b70b6e617b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 May 2021 13:47:49 -0400
Subject: [PATCH 24/37] A lot closer to passing on Python 3.

---
 integration/test_web.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/integration/test_web.py b/integration/test_web.py
index d48ad3aa6..8fac6909d 100644
--- a/integration/test_web.py
+++ b/integration/test_web.py
@@ -12,12 +12,12 @@ exists anywhere, however.
 from past.builtins import unicode
 
 import time
-import json
-import urllib2
+from urllib.parse import unquote as url_unquote, quote as url_quote
 
 import allmydata.uri
+from allmydata.util import jsonbytes as json
 
-import util
+from . import util
 
 import requests
 import html5lib
@@ -66,7 +66,7 @@ def test_upload_download(alice):
             u"filename": u"boom",
         }
     )
-    assert data == FILE_CONTENTS
+    assert unicode(data, "utf-8") == FILE_CONTENTS
 
 
 def test_put(alice):
@@ -117,10 +117,10 @@ def test_deep_stats(alice):
 
     # when creating a directory, we'll be re-directed to a URL
     # containing our writecap..
-    uri = urllib2.unquote(resp.url)
+    uri = url_unquote(resp.url)
     assert 'URI:DIR2:' in uri
     dircap = uri[uri.find("URI:DIR2:"):].rstrip('/')
-    dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap)))
+    dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap)))
 
     # POST a file into this directory
     FILE_CONTENTS = u"a file in a directory"
@@ -147,7 +147,7 @@ def test_deep_stats(alice):
     k, data = d
     assert k == u"dirnode"
     assert len(data['children']) == 1
-    k, child = data['children'].values()[0]
+    k, child = list(data['children'].values())[0]
     assert k == u"filenode"
     assert child['size'] == len(FILE_CONTENTS)
 
@@ -198,11 +198,11 @@ def test_status(alice):
 
     print("Uploaded data, cap={}".format(cap))
     resp = requests.get(
-        util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))),
+        util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))),
     )
 
     print("Downloaded {} bytes of data".format(len(resp.content)))
-    assert resp.content == FILE_CONTENTS
+    assert unicode(resp.content, "ascii") == FILE_CONTENTS
 
     resp = requests.get(
         util.node_url(alice.node_dir, "status"),
@@ -221,12 +221,12 @@ def test_status(alice):
             continue
         resp = requests.get(util.node_url(alice.node_dir, href))
         if href.startswith(u"/status/up"):
-            assert "File Upload Status" in resp.content
-            if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content:
+            assert b"File Upload Status" in resp.content
+            if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content:
                 found_upload = True
         elif href.startswith(u"/status/down"):
-            assert "File Download Status" in resp.content
-            if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content:
+            assert b"File Download Status" in resp.content
+            if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content:
                 found_download = True
 
                 # download the specialized event information
@@ -299,7 +299,7 @@ def test_directory_deep_check(alice):
     print("Uploaded data1, cap={}".format(cap1))
 
     resp = requests.get(
-        util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))),
+        util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))),
         params={u"t": u"info"},
     )
 
@@ -440,7 +440,7 @@ def test_introducer_info(introducer):
     resp = requests.get(
         util.node_url(introducer.node_dir, u""),
     )
-    assert "Introducer" in resp.content
+    assert b"Introducer" in resp.content
 
     resp = requests.get(
         util.node_url(introducer.node_dir, u""),
@@ -513,6 +513,6 @@ def test_mkdir_with_children(alice):
         params={u"t": "mkdir-with-children"},
         data=json.dumps(meta),
     )
-    assert resp.startswith("URI:DIR2")
+    assert resp.startswith(b"URI:DIR2")
     cap = allmydata.uri.from_string(resp)
     assert isinstance(cap, allmydata.uri.DirectoryURI)

From e2ce207fdd4d94de3b785ca22080c87ecaf6646e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 09:13:39 -0400
Subject: [PATCH 25/37] Don't convert bytes to unicode implicitly.

---
 src/allmydata/web/status.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py
index 0401fb586..158d897f9 100644
--- a/src/allmydata/web/status.py
+++ b/src/allmydata/web/status.py
@@ -1173,7 +1173,8 @@ class MapupdateStatusElement(Element):
     def privkey_from(self, req, tag):
         server = self._update_status.get_privkey_from()
         if server:
-            return tag(tags.li("Got privkey from: [%s]" % server.get_name()))
+            return tag(tags.li("Got privkey from: [%s]" % str(
+                server.get_name(), "utf-8")))
         else:
             return tag
 

From 079041fc1eb6d284382f39bd307ad8fba81b487e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 09:20:49 -0400
Subject: [PATCH 26/37] Port to Python 3.

---
 integration/test_web.py        | 21 +++++++++++++++------
 src/allmydata/util/_python3.py |  4 +---
 tox.ini                        |  2 +-
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/integration/test_web.py b/integration/test_web.py
index 8fac6909d..22f08da82 100644
--- a/integration/test_web.py
+++ b/integration/test_web.py
@@ -7,9 +7,18 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did
 at the time of testing' -- not necessarily a cohesive idea of what the
 WebAPI *should* do in every situation. It's not clear the latter
 exists anywhere, however.
+
+Ported to Python 3.
 """
 
-from past.builtins import unicode
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 from urllib.parse import unquote as url_unquote, quote as url_quote
@@ -66,7 +75,7 @@ def test_upload_download(alice):
             u"filename": u"boom",
         }
     )
-    assert unicode(data, "utf-8") == FILE_CONTENTS
+    assert str(data, "utf-8") == FILE_CONTENTS
 
 
 def test_put(alice):
@@ -97,7 +106,7 @@ def test_helper_status(storage_nodes):
     resp = requests.get(url)
     assert resp.status_code >= 200 and resp.status_code < 300
     dom = BeautifulSoup(resp.content, "html5lib")
-    assert unicode(dom.h1.string) == u"Helper Status"
+    assert str(dom.h1.string) == u"Helper Status"
 
 
 def test_deep_stats(alice):
@@ -202,7 +211,7 @@ def test_status(alice):
     )
 
     print("Downloaded {} bytes of data".format(len(resp.content)))
-    assert unicode(resp.content, "ascii") == FILE_CONTENTS
+    assert str(resp.content, "ascii") == FILE_CONTENTS
 
     resp = requests.get(
         util.node_url(alice.node_dir, "status"),
@@ -400,9 +409,9 @@ def test_directory_deep_check(alice):
     for _ in range(5):
         resp = requests.get(deepcheck_uri)
         dom = BeautifulSoup(resp.content, "html5lib")
-        if dom.h1 and u'Results' in unicode(dom.h1.string):
+        if dom.h1 and u'Results' in str(dom.h1.string):
             break
-        if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string):
+        if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string):
             dom = None
             time.sleep(1)
     assert dom is not None, "Operation never completed"
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 2bde5941e..c15301c3c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -17,15 +17,13 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 
-# Every time a module is added here, also add it to tox.ini environment
-# integrations3. Bit of duplication, but it's only a handful of files and quite
-# temporary, just until we've ported them all.
 PORTED_INTEGRATION_TESTS = [
     "integration.test_aaa_aardvark",
     "integration.test_servers_of_happiness",
     "integration.test_sftp",
     "integration.test_streaming_logs",
     "integration.test_tor",
+    "integration.test_web",
 ]
 
 
diff --git a/tox.ini b/tox.ini
index 04eae7833..365c5d0e9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,7 +102,7 @@ setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration/test_aaa_aardvark.py integration/test_servers_of_happiness.py integration/test_sftp.py integration/test_streaming_logs.py integration/test_tor.py}
+         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration}
          coverage combine
          coverage report
 

From 22f8b9b4281e15d3851eb7097e2c076907c621be Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 09:25:52 -0400
Subject: [PATCH 27/37] Port to Python 3.

---
 integration/conftest.py        | 10 ++++++++++
 integration/util.py            | 26 ++++++++++++++++++--------
 src/allmydata/util/_python3.py |  4 ++++
 3 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 918f2d4c9..39ff3b42b 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -1,5 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import sys
 import shutil
 from time import sleep
diff --git a/integration/util.py b/integration/util.py
index c8ba5d16b..7c7a1efd2 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -1,4 +1,14 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import sys
 import time
@@ -96,11 +106,11 @@ class _DumpOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
-        data = unicode(data, sys.stdout.encoding)
+        data = str(data, sys.stdout.encoding)
         self._out.write(data)
 
     def errReceived(self, data):
-        data = unicode(data, sys.stdout.encoding)
+        data = str(data, sys.stdout.encoding)
         self._out.write(data)
 
 
@@ -120,7 +130,7 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
-        data = unicode(data, sys.stdout.encoding)
+        data = str(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
@@ -128,7 +138,7 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
-        data = unicode(data, sys.stderr.encoding)
+        data = str(data, sys.stderr.encoding)
         sys.stdout.write(data)
 
 
@@ -269,9 +279,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
             '--hostname', 'localhost',
             '--listen', 'tcp',
             '--webport', web_port,
-            '--shares-needed', unicode(needed),
-            '--shares-happy', unicode(happy),
-            '--shares-total', unicode(total),
+            '--shares-needed', str(needed),
+            '--shares-happy', str(happy),
+            '--shares-total', str(total),
             '--helper',
         ]
         if not storage:
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index c15301c3c..f6cd9752c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -279,4 +279,8 @@ PORTED_TEST_MODULES = [
     "allmydata.test.web.test_web",
     "allmydata.test.web.test_webish",
     "allmydata.test.test_windows",
+
+    "integration",
+    "integration.conftest",
+    "integration.util",
 ]

From ba03a7b436a6f1f9b26c11ff2209f3e71953bc03 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 09:26:14 -0400
Subject: [PATCH 28/37] News file.

---
 newsfragments/3709.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3709.minor

diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor
new file mode 100644
index 000000000..e69de29bb

From d7ad72f651cd16158364921a573170771142d36c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 11:21:22 -0400
Subject: [PATCH 29/37] Just do integration tests as part of normal CI runners.

---
 .github/workflows/ci.yml | 89 ++++++----------------------------------
 tox.ini                  |  4 +-
 2 files changed, 15 insertions(+), 78 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1f3d57310..6fd542e0c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -33,6 +33,19 @@ jobs:
             python-version: 2.7
 
     steps:
+      - name: Install Tor [Ubuntu]
+        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt install tor
+
+      - name: Install Tor [macOS]
+        if: matrix.os == 'macos-latest'
+        run: brew install tor
+
+      - name: Install Tor [Windows]
+        if: matrix.os == 'windows-latest'
+        uses: crazy-max/ghaction-chocolatey@v1
+        with:
+          args: install tor
 
       # See https://github.com/actions/checkout. A fetch-depth of 0
       # fetches all tags and branches.
@@ -172,82 +185,6 @@ jobs:
           # Some magic value required for some magic reason.
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
-  integration:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - macos-latest
-          - windows-latest
-        python-version:
-          - 2.7
-
-    steps:
-
-      - name: Install Tor [Ubuntu]
-        if: matrix.os == 'ubuntu-latest'
-        run: sudo apt install tor
-
-      - name: Install Tor [macOS]
-        if: matrix.os == 'macos-latest'
-        run: brew install tor
-
-      - name: Install Tor [Windows]
-        if: matrix.os == 'windows-latest'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install tor
-
-      - name: Check out Tahoe-LAFS sources
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      # See this step under coverage job.
-      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: 'x86'
-
-      - name: Get pip cache directory
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Use pip cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install Python packages
-        run: |
-          pip install --upgrade tox
-          pip list
-
-      - name: Display tool versions
-        run: python misc/build_helpers/show-tool-versions.py
-
-      - name: Run "tox -e integration"
-        run: tox -e integration
-
-      - name: Upload eliot.log in case of failure
-        uses: actions/upload-artifact@v1
-        if: failure()
-        with:
-          name: integration.eliot.json
-          path: integration.eliot.json
 
   packaging:
     runs-on: ${{ matrix.os }}
diff --git a/tox.ini b/tox.ini
index 365c5d0e9..2eddfd766 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,11 +6,11 @@
 # Map Python versions in GitHub Actions to tox environments to run.
 [gh-actions]
 python =
-    2.7: py27-coverage,codechecks
+    2.7: py27-coverage,codechecks,integration
     3.6: py36-coverage,integration3
     3.7: py37-coverage
     3.8: py38-coverage
-    3.9: py39-coverage,typechecks,codechecks3
+    3.9: py39-coverage,typechecks,codechecks3,integration3
     pypy-3.7: pypy3
 
 [pytest]

From c4dd8dde27237592c87a90d6502bc8fd4a9153b1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 11:51:47 -0400
Subject: [PATCH 30/37] Flake.

---
 src/allmydata/scripts/runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index f63b9b330..a17ce3ef4 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-import warnings
+
 import os, sys
 from six.moves import StringIO
 import six

From 8741c33b3562391e9035ff469621d74d8282fb1d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 11:58:02 -0400
Subject: [PATCH 31/37] Needs to be in separate list since it's not checked.

---
 src/allmydata/util/_python3.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f6cd9752c..53e5998fb 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -26,6 +26,11 @@ PORTED_INTEGRATION_TESTS = [
     "integration.test_web",
 ]
 
+PORTED_INTEGRATION_MODULES = [
+    "integration",
+    "integration.conftest",
+    "integration.util",
+]
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
@@ -279,8 +284,4 @@ PORTED_TEST_MODULES = [
     "allmydata.test.web.test_web",
     "allmydata.test.web.test_webish",
     "allmydata.test.test_windows",
-
-    "integration",
-    "integration.conftest",
-    "integration.util",
 ]

From c4c053a37d5da1398ab18884436f6856098c9685 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 12 May 2021 12:02:22 -0400
Subject: [PATCH 32/37] Fix failing tests on Python 3.

---
 src/allmydata/scripts/runner.py   | 6 ++++--
 src/allmydata/test/test_system.py | 5 +++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index a17ce3ef4..43a3b8807 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -183,10 +183,12 @@ def _maybe_enable_eliot_logging(options, reactor):
     # Pass on the options so we can dispatch the subcommand.
     return options
 
+PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress."
+                    " Use at your own risk.")
+
 def run():
     if six.PY3:
-        print("Support for Python 3 is an incomplete work-in-progress."
-              " Use at your own risk.", file=sys.stderr)
+        print(PYTHON_3_WARNING, file=sys.stderr)
 
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 12ae846eb..627b6ef29 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -43,6 +43,7 @@ from allmydata.monitor import Monitor
 from allmydata.mutable.common import NotWriteableError
 from allmydata.mutable import layout as mutable_layout
 from allmydata.mutable.publish import MutableData
+from allmydata.scripts.runner import PYTHON_3_WARNING
 
 from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue
 from twisted.python.failure import Failure
@@ -2635,7 +2636,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             out, err, rc_or_sig = res
             self.failUnlessEqual(rc_or_sig, 0, str(res))
             if check_stderr:
-                self.failUnlessEqual(err, b"")
+                self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii")))
 
         d.addCallback(_run_in_subprocess, "create-alias", "newalias")
         d.addCallback(_check_succeeded)
@@ -2655,7 +2656,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_ls(res):
             out, err, rc_or_sig = res
             self.failUnlessEqual(rc_or_sig, 0, str(res))
-            self.failUnlessEqual(err, b"", str(res))
+            self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii")))
             self.failUnlessIn(b"tahoe-moved", out)
             self.failIfIn(b"tahoe-file", out)
         d.addCallback(_check_ls)

From 914f1d171cf241b3c55b69f1572a9b9f7ba28011 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 May 2021 10:19:11 -0400
Subject: [PATCH 33/37] Let's not run integration tests as part of unit tests.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 8ffba7713..ca5a2c689 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,7 @@
 [gh-actions]
 python =
     2.7: py27-coverage,codechecks
-    3.6: py36-coverage,integration3
+    3.6: py36-coverage
     3.7: py37-coverage
     3.8: py38-coverage
     3.9: py39-coverage,typechecks,codechecks3

From dc7caa00c5b17467b87c709c4e55df7817ad8d9a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 May 2021 10:25:48 -0400
Subject: [PATCH 34/37] Run Python 3 integration tests.

---
 .github/workflows/ci.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1f3d57310..89d35fdc1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -182,6 +182,9 @@ jobs:
           - windows-latest
         python-version:
           - 2.7
+        include:
+          - os: ubuntu-latest
+            python-version: 3.6
 
     steps:
 
@@ -239,9 +242,14 @@ jobs:
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e integration"
+      - name: Run "Python 2 integration tests"
+        if: ${{ matrix.python-version == '2.7' }}
         run: tox -e integration
 
+      - name: Run "Python 3 integration tests"
+        if: ${{ matrix.python-version != '2.7' }}
+        run: tox -e integration3
+
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1
         if: failure()

From 2f7ba77a9f3433f5b3be6856e0459670b44161a8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 May 2021 10:26:30 -0400
Subject: [PATCH 35/37] Note the version being run.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index ca5a2c689..34698e8be 100644
--- a/tox.ini
+++ b/tox.ini
@@ -102,6 +102,7 @@ basepython = python3
 setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
+         python --version
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
          py.test --timeout=1800 --coverage -v {posargs:integration/test_servers_of_happiness.py}
          coverage combine

From 73a3e82856047954103890c6e30e429051591473 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 May 2021 11:29:11 -0400
Subject: [PATCH 36/37] Don't run integration tests here either.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index cb134ffd2..964403a40 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,7 @@ python =
     3.6: py36-coverage
     3.7: py37-coverage
     3.8: py38-coverage
-    3.9: py39-coverage,typechecks,codechecks3,integration3
+    3.9: py39-coverage,typechecks,codechecks3
     pypy-3.7: pypy3
 
 [pytest]

From 6a619b1ec74cf8c4d76dbb20fd51f97d24c582aa Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 14 May 2021 10:05:08 -0400
Subject: [PATCH 37/37] Make sure not to install Python 3-only version of
 pefile.

---
 newsfragments/3717.minor | 0
 tox.ini                  | 2 ++
 2 files changed, 2 insertions(+)
 create mode 100644 newsfragments/3717.minor

diff --git a/newsfragments/3717.minor b/newsfragments/3717.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/tox.ini b/tox.ini
index 5f500b49d..e7ce93f88 100644
--- a/tox.ini
+++ b/tox.ini
@@ -281,6 +281,8 @@ deps =
     # PyInstaller 4.0 drops Python 2 support.  When we finish porting to
     # Python 3 we can reconsider this constraint.
     pyinstaller < 4.0
+    # 2021.5.13 broke on Windows. See https://github.com/erocarrera/pefile/issues/318
+    pefile < 2021.5.13 ; platform_system == "Windows"
 # Setting PYTHONHASHSEED to a known value assists with reproducible builds.
 # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build
 setenv=PYTHONHASHSEED=1