Merge remote-tracking branch 'origin/master' into tor-integration-test-improvements

2023-07-21 12:25:32 -04:00 · 2023-07-21 12:25:32 -04:00 · eb3f8f6219
parent d7ac368aa6 8b8903c44f
commit eb3f8f6219
153 changed files with 3721 additions and 2131 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -11,22 +11,36 @@
 #
 version: 2.1

-# A template that can be shared between the two different image-building
+# Every job that pushes a Docker image from Docker Hub must authenticate to
+# it.  Define a couple yaml anchors that can be used to supply the necessary
+# credentials.
+
+# First is a CircleCI job context which makes Docker Hub credentials available
+# in the environment.
+#
+# Contexts are managed in the CircleCI web interface:
+#
+#  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
+dockerhub-context-template: &DOCKERHUB_CONTEXT
+  context: "dockerhub-auth"
+
+# Next is a Docker executor template that gets the credentials from the
+# environment and supplies them to the executor.
+dockerhub-auth-template: &DOCKERHUB_AUTH
+  - auth:
+      username: $DOCKERHUB_USERNAME
+      password: $DOCKERHUB_PASSWORD
+
+  # A template that can be shared between the two different image-building
 # workflows.
 .images: &IMAGES
  jobs:
-    # Every job that pushes a Docker image from Docker Hub needs to provide
-    # credentials.  Use this first job to define a yaml anchor that can be
-    # used to supply a CircleCI job context which makes Docker Hub credentials
-    # available in the environment.
-    #
-    # Contexts are managed in the CircleCI web interface:
-    #
-    #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
-    - "build-image-debian-11": &DOCKERHUB_CONTEXT
+    - "build-image-debian-11":
        <<: *DOCKERHUB_CONTEXT
    - "build-image-ubuntu-20-04":
        <<: *DOCKERHUB_CONTEXT
+    - "build-image-ubuntu-22-04":
+        <<: *DOCKERHUB_CONTEXT
    - "build-image-fedora-35":
        <<: *DOCKERHUB_CONTEXT
    - "build-image-oraclelinux-8":
@ -66,17 +80,30 @@ workflows:
      - "ubuntu-20-04":
          {}

+      - "ubuntu-22-04":
+          {}
+
      # Equivalent to RHEL 8; CentOS 8 is dead.
      - "oraclelinux-8":
          {}

      - "nixos":
-          name: "NixOS 21.05"
-          nixpkgs: "21.05"
+          name: "<<matrix.pythonVersion>>"
+          nixpkgs: "22.11"
+          matrix:
+            parameters:
+              pythonVersion:
+                - "python38"
+                - "python39"
+                - "python310"

      - "nixos":
-          name: "NixOS 21.11"
-          nixpkgs: "21.11"
+          name: "<<matrix.pythonVersion>>"
+          nixpkgs: "unstable"
+          matrix:
+            parameters:
+              pythonVersion:
+                - "python311"

      # Eventually, test against PyPy 3.8
      #- "pypy27-buster":
@ -113,30 +140,7 @@ workflows:
    # Build as part of the workflow but only if requested.
    when: "<< pipeline.parameters.build-images >>"

-
 jobs:
-  dockerhub-auth-template:
-    # This isn't a real job.  It doesn't get scheduled as part of any
-    # workflow.  Instead, it's just a place we can hang a yaml anchor to
-    # finish the Docker Hub authentication configuration.  Workflow jobs using
-    # the DOCKERHUB_CONTEXT anchor will have access to the environment
-    # variables used here.  These variables will allow the Docker Hub image
-    # pull to be authenticated and hopefully avoid hitting and rate limits.
-    docker: &DOCKERHUB_AUTH
-      - image: "null"
-        auth:
-          username: $DOCKERHUB_USERNAME
-          password: $DOCKERHUB_PASSWORD
-
-    steps:
-      - run:
-          name: "CircleCI YAML schema conformity"
-          command: |
-            # This isn't a real command.  We have to have something in this
-            # space, though, or the CircleCI yaml schema validator gets angry.
-            # Since this job is never scheduled this step is never run so the
-            # actual value here is irrelevant.
-
  codechecks:
    docker:
      - <<: *DOCKERHUB_AUTH
@ -256,7 +260,7 @@ jobs:
          name: "Submit coverage results"
          command: |
            if [ -n "${UPLOAD_COVERAGE}" ]; then
-              /tmp/venv/bin/codecov
+              echo "TODO: Need a new coverage solution, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4011"
            fi

    docker:
@ -336,6 +340,16 @@ jobs:
      <<: *UTF_8_ENVIRONMENT
      TAHOE_LAFS_TOX_ENVIRONMENT: "py39"

+  ubuntu-22-04:
+    <<: *DEBIAN
+    docker:
+      - <<: *DOCKERHUB_AUTH
+        image: "tahoelafsci/ubuntu:22.04-py3.10"
+        user: "nobody"
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py310"
+
  oraclelinux-8: &RHEL_DERIV
    docker:
      - <<: *DOCKERHUB_AUTH
@ -374,71 +388,29 @@ jobs:
          Reference the name of a niv-managed nixpkgs source (see `niv show`
          and nix/sources.json)
        type: "string"
+      pythonVersion:
+        description: >-
+          Reference the name of a Python package in nixpkgs to use.
+        type: "string"

-    docker:
-      # Run in a highly Nix-capable environment.
-      - <<: *DOCKERHUB_AUTH
-        image: "nixos/nix:2.10.3"
-
-    environment:
-      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
-      # allows us to push to CACHIX_NAME.  We only need this set for
-      # `cachix use` in this step.
-      CACHIX_NAME: "tahoe-lafs-opensource"
+    executor: "nix"

    steps:
-      - "run":
-          # Get cachix for Nix-friendly caching.
-          name: "Install Basic Dependencies"
-          command: |
-            NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
-            nix-env \
-              --file $NIXPKGS \
-              --install \
-              -A cachix bash
-              # Activate it for "binary substitution".  This sets up
-              # configuration tht lets Nix download something from the cache
-              # instead of building it locally, if possible.
-              cachix use "${CACHIX_NAME}"
-
-      - "checkout"
-
-      - "run":
-          # The Nix package doesn't know how to do this part, unfortunately.
-          name: "Generate version"
-          command: |
-            nix-shell \
-              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
-              --run 'python setup.py update_version'
-
-      - "run":
-          name: "Build"
-          command: |
-            # CircleCI build environment looks like it has a zillion and a
-            # half cores.  Don't let Nix autodetect this high core count
-            # because it blows up memory usage and fails the test run.  Pick a
-            # number of cores that suites the build environment we're paying
-            # for (the free one!).
-            #
-            # Also, let it run more than one job at a time because we have to
-            # build a couple simple little dependencies that don't take
-            # advantage of multiple cores and we get a little speedup by doing
-            # them in parallel.
-            source .circleci/lib.sh
-            cache_if_able nix-build \
-              --cores 3 \
-              --max-jobs 2 \
-              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
-
-      - "run":
-          name: "Test"
-          command: |
-            # Let it go somewhat wild for the test suite itself
-            source .circleci/lib.sh
-            cache_if_able nix-build \
-              --cores 8 \
-              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
-              tests.nix
+      - "nix-build":
+          nixpkgs: "<<parameters.nixpkgs>>"
+          pythonVersion: "<<parameters.pythonVersion>>"
+          buildSteps:
+            - "run":
+                name: "Unit Test"
+                command: |
+                  # The dependencies are all built so we can allow more
+                  # parallelism here.
+                  source .circleci/lib.sh
+                  cache_if_able nix-build \
+                    --cores 8 \
+                    --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+                    --argstr pythonVersion "<<parameters.pythonVersion>>" \
+                    nix/tests.nix

  typechecks:
    docker:
@ -524,6 +496,15 @@ jobs:
      PYTHON_VERSION: "3.9"


+  build-image-ubuntu-22-04:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "22.04"
+      PYTHON_VERSION: "3.10"
+
+
  build-image-oraclelinux-8:
    <<: *BUILD_IMAGE

@ -542,7 +523,6 @@ jobs:

  # build-image-pypy27-buster:
  #   <<: *BUILD_IMAGE
-
  #   environment:
  #     DISTRO: "pypy"
  #     TAG: "buster"
@ -550,3 +530,87 @@ jobs:
  #     # setting up PyPy 3 in the image building toolchain.  This value is just
  #     # for constructing the right Docker image tag.
  #     PYTHON_VERSION: "2"
+
+executors:
+  nix:
+    docker:
+      # Run in a highly Nix-capable environment.
+      - <<: *DOCKERHUB_AUTH
+        image: "nixos/nix:2.10.3"
+    environment:
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
+      # to push to CACHIX_NAME.  CACHIX_NAME tells cachix which cache to push
+      # to.
+      CACHIX_NAME: "tahoe-lafs-opensource"
+
+commands:
+  nix-build:
+    parameters:
+      nixpkgs:
+        description: >-
+          Reference the name of a niv-managed nixpkgs source (see `niv show`
+          and nix/sources.json)
+        type: "string"
+      pythonVersion:
+        description: >-
+          Reference the name of a Python package in nixpkgs to use.
+        type: "string"
+      buildSteps:
+        description: >-
+          The build steps to execute after setting up the build environment.
+        type: "steps"
+
+    steps:
+      - "run":
+          # Get cachix for Nix-friendly caching.
+          name: "Install Basic Dependencies"
+          command: |
+            NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
+            nix-env \
+              --file $NIXPKGS \
+              --install \
+              -A cachix bash
+              # Activate it for "binary substitution".  This sets up
+              # configuration tht lets Nix download something from the cache
+              # instead of building it locally, if possible.
+              cachix use "${CACHIX_NAME}"
+
+      - "checkout"
+
+      - "run":
+          # The Nix package doesn't know how to do this part, unfortunately.
+          name: "Generate version"
+          command: |
+            nix-shell \
+              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
+              --run 'python setup.py update_version'
+
+      - "run":
+          name: "Build Dependencies"
+          command: |
+            # CircleCI build environment looks like it has a zillion and a
+            # half cores.  Don't let Nix autodetect this high core count
+            # because it blows up memory usage and fails the test run.  Pick a
+            # number of cores that suits the build environment we're paying
+            # for (the free one!).
+            source .circleci/lib.sh
+            # nix-shell will build all of the dependencies of the target but
+            # not the target itself.
+            cache_if_able nix-shell \
+              --run "" \
+              --cores 3 \
+              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+              --argstr pythonVersion "<<parameters.pythonVersion>>" \
+              ./default.nix
+
+      - "run":
+          name: "Build Package"
+          command: |
+            source .circleci/lib.sh
+            cache_if_able nix-build \
+              --cores 4 \
+              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+              --argstr pythonVersion "<<parameters.pythonVersion>>" \
+              ./default.nix
+
+      - steps: "<<parameters.buildSteps>>"
--- a/.circleci/create-virtualenv.sh
+++ b/.circleci/create-virtualenv.sh
@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 # above, it may still not be able to get us a compatible version unless we
 # explicitly ask for one.
 "${PIP}" install --upgrade setuptools==44.0.0 wheel
+
+# Just about every user of this image wants to use tox from the bootstrap
+# virtualenv so go ahead and install it now.
+"${PIP}" install "tox~=3.0"
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@ -3,18 +3,6 @@
 # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail

-# Basic Python packages that you just need to have around to do anything,
-# practically speaking.
-BASIC_DEPS="pip wheel"
-
-# Python packages we need to support the test infrastructure.  *Not* packages
-# Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox~=3.0 codecov"
-
-# Python packages we need to generate test reports for CI infrastructure.
-# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
-REPORTING_DEPS="python-subunit junitxml subunitreporter"
-
 # The filesystem location of the wheelhouse which we'll populate with wheels
 # for all of our dependencies.
 WHEELHOUSE_PATH="$1"
@ -41,15 +29,5 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 LANG="en_US.UTF-8" "${PIP}" \
    wheel \
    --wheel-dir "${WHEELHOUSE_PATH}" \
-    "${PROJECT_ROOT}"[test] \
-    ${BASIC_DEPS} \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
-
-# Not strictly wheelhouse population but ... Note we omit basic deps here.
-# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
-# have to ask.
-"${PIP}" \
-    install \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
+    "${PROJECT_ROOT}"[testenv] \
+    "${PROJECT_ROOT}"[test]
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@ -79,9 +79,10 @@ else
    alternative="false"
 fi

+WORKDIR=/tmp/tahoe-lafs.tox
 ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \
    -c ${PROJECT_ROOT}/tox.ini \
-    --workdir /tmp/tahoe-lafs.tox \
+    --workdir "${WORKDIR}" \
    -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
    ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"

@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then

    # Create a junitxml results area.
    mkdir -p "$(dirname "${JUNITXML}")"
-    "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+
+    "${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@ -26,12 +26,7 @@ shift || :

 # Tell pip where it can find any existing wheels.
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
-
-# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
-# between the time dependencies change and the images are re-built and (b) the
-# upcoming-deprecations job wants to install some dependencies from github and
-# it's awkward to get that done any earlier than the tox run.  So, we don't
-# set it.
+export PIP_NO_INDEX="1"

 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -46,7 +46,6 @@ jobs:
      matrix:
        os:
          - windows-latest
-          - ubuntu-latest
        python-version:
          - "3.8"
          - "3.9"
@ -54,9 +53,9 @@ jobs:
          - "3.11"
        include:
          # On macOS don't bother with 3.8, just to get faster builds.
-          - os: macos-latest
+          - os: macos-12
            python-version: "3.9"
-          - os: macos-latest
+          - os: macos-12
            python-version: "3.11"
          # We only support PyPy on Linux at the moment.
          - os: ubuntu-latest
@ -80,7 +79,7 @@ jobs:

      - name: Install Python packages
        run: |
-          pip install --upgrade codecov "tox<4" tox-gh-actions setuptools
+          pip install --upgrade "tox<4" tox-gh-actions setuptools
          pip list

      - name: Display tool versions
@ -165,18 +164,20 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        include:
-          - os: macos-latest
-            python-version: "3.9"
-            force-foolscap: false
-          - os: windows-latest
-            python-version: "3.9"
-            force-foolscap: false
+        os:
          # 22.04 has some issue with Tor at the moment:
          # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
+          - ubuntu-20.04
+          - macos-12
+          - windows-latest
+        python-version:
+          - "3.11"
+        force-foolscap:
+          - false
+        include:
          - os: ubuntu-20.04
-            python-version: "3.11"
-            force-foolscap: false
+            python-version: "3.10"
+            force-foolscap: true
    steps:

      - name: Install Tor [Ubuntu]
@ -249,7 +250,7 @@ jobs:
      fail-fast: false
      matrix:
        os:
-          - macos-10.15
+          - macos-12
          - windows-latest
          - ubuntu-latest
        python-version:
--- a/.gitignore
+++ b/.gitignore
@ -53,3 +53,5 @@ zope.interface-*.egg
 # This is the plaintext of the private environment needed for some CircleCI
 # operations.  It's never supposed to be checked in.
 secret-env-plain
+
+.ruff_cache
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@ -1,5 +1,10 @@
 version: 2

+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
 python:
   install:
   - requirements: docs/requirements.txt
--- a/.ruff.toml
+++ b/.ruff.toml
@ -0,0 +1,18 @@
+select = [
+    # Pyflakes checks
+    "F",
+    # Prohibit tabs:
+    "W191",
+    # No trailing whitespace:
+    "W291",
+    "W293",
+    # Make sure we bind closure variables in a loop (equivalent to pylint
+    # cell-var-from-loop):
+    "B023",
+    # Don't silence exceptions in finally by accident:
+    "B012",
+    # Don't use mutable default arguments:
+    "B006",
+    # Errors from PyLint:
+    "PLE",
+]
--- a/default.nix
+++ b/default.nix
@ -1,8 +1,7 @@
 let
  # sources.nix contains information about which versions of some of our
-  # dependencies we should use.  since we use it to pin nixpkgs and the PyPI
-  # package database, roughly all the rest of our dependencies are *also*
-  # pinned - indirectly.
+  # dependencies we should use.  since we use it to pin nixpkgs, all the rest
+  # of our dependencies are *also* pinned - indirectly.
  #
  # sources.nix is managed using a tool called `niv`.  as an example, to
  # update to the most recent version of nixpkgs from the 21.11 maintenance
@ -10,93 +9,41 @@ let
  #
  #   niv update nixpkgs-21.11
  #
-  # or, to update the PyPI package database -- which is necessary to make any
-  # newly released packages visible -- you likewise run:
-  #
-  #   niv update pypi-deps-db
-  #
  # niv also supports chosing a specific revision, following a different
  # branch, etc.  find complete documentation for the tool at
  # https://github.com/nmattia/niv
  sources = import nix/sources.nix;
 in
 {
-  pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
+  pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
                                # niv-managed sources data

 , pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself

-, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
-                                  # for dependency resolution
+, pythonVersion ? "python310" # a string choosing the python derivation from
+                              # nixpkgs to target

-, pythonVersion ? "python39" # a string choosing the python derivation from
-                             # nixpkgs to target
+, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
+                                # the dependencies of which the resulting
+                                # package will also depend on.  Include all of the
+                                # runtime extras by default because the incremental
+                                # cost of including them is a lot smaller than the
+                                # cost of re-building the whole thing to add them.

-, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
-                           # the dependencies of which the resulting package
-                           # will also depend on.  Include all of the runtime
-                           # extras by default because the incremental cost of
-                           # including them is a lot smaller than the cost of
-                           # re-building the whole thing to add them.
-
-, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
-                                       # the tahoe-lafs package
-    inherit pkgs pypiData;
-    python = pythonVersion;
-}
 }:
-# The project name, version, and most other metadata are automatically
-# extracted from the source.  Some requirements are not properly extracted
-# and those cases are handled below.  The version can only be extracted if
-# `setup.py update_version` has been run (this is not at all ideal but it
-# seems difficult to fix) - so for now just be sure to run that first.
-mach-nix.buildPythonPackage rec {
-  # Define the location of the Tahoe-LAFS source to be packaged.  Clean up all
-  # as many of the non-source files (eg the `.git` directory, `~` backup
-  # files, nix's own `result` symlink, etc) as possible to avoid needing to
-  # re-build when files that make no difference to the package have changed.
-  src = pkgs.lib.cleanSource ./.;
-
+with (pkgs.${pythonVersion}.override {
+  packageOverrides = import ./nix/python-overrides.nix;
+}).pkgs;
+callPackage ./nix/tahoe-lafs.nix {
  # Select whichever package extras were requested.
-  inherit extras;
+  inherit extrasNames;

-  # Define some extra requirements that mach-nix does not automatically detect
-  # from inspection of the source.  We typically don't need to put version
-  # constraints on any of these requirements.  The pypi-deps-db we're
-  # operating with makes dependency resolution deterministic so as long as it
-  # works once it will always work.  It could be that in the future we update
-  # pypi-deps-db and an incompatibility arises - in which case it would make
-  # sense to apply some version constraints here.
-  requirementsExtra = ''
-    # mach-nix does not yet support pyproject.toml which means it misses any
-    # build-time requirements of our dependencies which are declared in such a
-    # file.  Tell it about them here.
-    setuptools_rust
+  # Define the location of the Tahoe-LAFS source to be packaged (the same
+  # directory as contains this file).  Clean up as many of the non-source
+  # files (eg the `.git` directory, `~` backup files, nix's own `result`
+  # symlink, etc) as possible to avoid needing to re-build when files that
+  # make no difference to the package have changed.
+  tahoe-lafs-src = pkgs.lib.cleanSource ./.;

-    # mach-nix does not yet parse environment markers (e.g. "python > '3.0'")
-    # correctly. It misses all of our requirements which have an environment marker.
-    # Duplicate them here.
-    foolscap
-    eliot
-    pyrsistent
-    collections-extended
-  '';
-
-  # Specify where mach-nix should find packages for our Python dependencies.
-  # There are some reasonable defaults so we only need to specify certain
-  # packages where the default configuration runs into some issue.
-  providers = {
-  };
-
-  # Define certain overrides to the way Python dependencies are built.
-  _ = {
-    # Remove a click-default-group patch for a test suite problem which no
-    # longer applies because the project apparently no longer has a test suite
-    # in its source distribution.
-    click-default-group.patches = [];
-  };
-
-  passthru.meta.mach-nix = {
-    inherit providers _;
-  };
+  doCheck = false;
 }
--- a/docs/performance.rst
+++ b/docs/performance.rst
@ -82,8 +82,9 @@ network: A

 memory footprint: N/K*A

-notes: Tahoe-LAFS generates a new RSA keypair for each mutable file that it
-publishes to a grid. This takes up to 1 or 2 seconds on a typical desktop PC.
+notes:
+Tahoe-LAFS generates a new RSA keypair for each mutable file that it publishes to a grid.
+This takes around 100 milliseconds on a relatively high-end laptop from 2021.

 Part of the process of encrypting, encoding, and uploading a mutable file to a
 Tahoe-LAFS grid requires that the entire file be in memory at once. For larger
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@ -3,7 +3,7 @@
 Storage Node Protocol ("Great Black Swamp", "GBS")
 ==================================================

-The target audience for this document is Tahoe-LAFS developers.
+The target audience for this document is developers working on Tahoe-LAFS or on an alternate implementation intended to be interoperable.
 After reading this document,
 one should expect to understand how Tahoe-LAFS clients interact over the network with Tahoe-LAFS storage nodes.

@ -64,6 +64,10 @@ Glossary
   lease renew secret
     a short secret string which storage servers required to be presented before allowing a particular lease to be renewed

+The key words
+"MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED",  "MAY", and "OPTIONAL"
+in this document are to be interpreted as described in RFC 2119.
+
 Motivation
 ----------

@ -119,8 +123,8 @@ An HTTP-based protocol can make use of TLS in largely the same way to provide th
 Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation
 (rather than the standard "web" rules for validation).

-Requirements
------------
+Design Requirements
+-------------------

 Security
 ~~~~~~~~
@ -189,6 +193,9 @@ Solutions
 An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described below.
 This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol.

+Summary (Non-normative)
+~~~~~~~~~~~~~~~~~~~~~~~
+
 Communication with the storage node will take place using TLS.
 The TLS version and configuration will be dictated by an ongoing understanding of best practices.
 The storage node will present an x509 certificate during the TLS handshake.
@ -237,10 +244,10 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the *
 .. note::

   Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate).
-   They are encoded with Base32 for a length of 32 bytes.
+   They are encoded with `Base32`_ for a length of 32 bytes.
   SPKI information discussed here is 32 bytes (SHA256 digest).
-   They would be encoded in Base32 for a length of 52 bytes.
-   `base64url`_ provides a more compact encoding of the information while remaining URL-compatible.
+   They would be encoded in `Base32`_ for a length of 52 bytes.
+   `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible.
   This would encode the SPKI information for a length of merely 43 bytes.
   SHA1,
   the current Foolscap hash function,
@ -329,15 +336,117 @@ and shares.
 A particular resource is addressed by the HTTP request path.
 Details about the interface are encoded in the HTTP message body.

+String Encoding
+~~~~~~~~~~~~~~~
+
+.. _Base32:
+
+Base32
+!!!!!!
+
+Where the specification refers to Base32 the meaning is *unpadded* Base32 encoding as specified by `RFC 4648`_ using a *lowercase variation* of the alphabet from Section 6.
+
+That is, the alphabet is:
+
+.. list-table:: Base32 Alphabet
+   :header-rows: 1
+
+   * - Value
+     - Encoding
+     - Value
+     - Encoding
+     - Value
+     - Encoding
+     - Value
+     - Encoding
+
+   * - 0
+     - a
+     - 9
+     - j
+     - 18
+     - s
+     - 27
+     - 3
+   * - 1
+     - b
+     - 10
+     - k
+     - 19
+     - t
+     - 28
+     - 4
+   * - 2
+     - c
+     - 11
+     - l
+     - 20
+     - u
+     - 29
+     - 5
+   * - 3
+     - d
+     - 12
+     - m
+     - 21
+     - v
+     - 30
+     - 6
+   * - 4
+     - e
+     - 13
+     - n
+     - 22
+     - w
+     - 31
+     - 7
+   * - 5
+     - f
+     - 14
+     - o
+     - 23
+     - x
+     -
+     -
+   * - 6
+     - g
+     - 15
+     - p
+     - 24
+     - y
+     -
+     -
+   * - 7
+     - h
+     - 16
+     - q
+     - 25
+     - z
+     -
+     -
+   * - 8
+     - i
+     - 17
+     - r
+     - 26
+     - 2
+     -
+     -
+
 Message Encoding
 ~~~~~~~~~~~~~~~~

-The preferred encoding for HTTP message bodies is `CBOR`_.
-A request may be submitted using an alternate encoding by declaring this in the ``Content-Type`` header.
-A request may indicate its preference for an alternate encoding in the response using the ``Accept`` header.
-These two headers are used in the typical way for an HTTP application.
+Clients and servers MUST use the ``Content-Type`` and ``Accept`` header fields as specified in `RFC 9110`_ for message body negotiation.

-The only other encoding support for which is currently recommended is JSON.
+The encoding for HTTP message bodies SHOULD be `CBOR`_.
+Clients submitting requests using this encoding MUST include a ``Content-Type: application/cbor`` request header field.
+A request MAY be submitted using an alternate encoding by declaring this in the ``Content-Type`` header field.
+A request MAY indicate its preference for an alternate encoding in the response using the ``Accept`` header field.
+A request which includes no ``Accept`` header field MUST be interpreted in the same way as a request including a ``Accept: application/cbor`` header field.
+
+Clients and servers MAY support additional request and response message body encodings.
+
+Clients and servers SHOULD support ``application/json`` request and response message body encoding.
 For HTTP messages carrying binary share data,
 this is expected to be a particularly poor encoding.
 However,
@ -350,10 +459,23 @@ Because of the simple types used throughout
 and the equivalence described in `RFC 7049`_
 these examples should be representative regardless of which of these two encodings is chosen.

-The one exception is sets.
-For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
-Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml>`_ for more details.
-Sets will be represented as JSON lists in examples because JSON doesn't support sets.
+There are two exceptions to this rule.
+
+1. Sets
+!!!!!!!
+
+For CBOR messages,
+any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
+Tag 6.258 is used to indicate sets in CBOR;
+see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml>`_ for more details.
+The JSON encoding does not support sets.
+Sets MUST be represented as arrays in JSON-encoded messages.
+
+2. Bytes
+!!!!!!!!
+
+The CBOR encoding natively supports a bytes type while the JSON encoding does not.
+Bytes MUST be represented as strings giving the `Base64`_ representation of the original bytes value.

 HTTP Design
 ~~~~~~~~~~~
@ -368,29 +490,50 @@ one branch contains all of the share data;
 another branch contains all of the lease data;
 etc.

-An ``Authorization`` header in requests is required for all endpoints.
-The standard HTTP authorization protocol is used.
-The authentication *type* used is ``Tahoe-LAFS``.
-The swissnum from the NURL used to locate the storage service is used as the *credentials*.
-If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response.
+Clients and servers MUST use the ``Authorization`` header field,
+as specified in `RFC 9110`_,
+for authorization of all requests to all endpoints specified here.
+The authentication *type* MUST be ``Tahoe-LAFS``.
+Clients MUST present the `Base64`_-encoded representation of the swissnum from the NURL used to locate the storage service as the *credentials*.

-There are also, for some endpoints, secrets sent via ``X-Tahoe-Authorization`` headers.
-If these are:
+If credentials are not presented or the swissnum is not associated with a storage service then the server MUST issue a ``401 UNAUTHORIZED`` response and perform no other processing of the message.
+
+Requests to certain endpoints MUST include additional secrets in the ``X-Tahoe-Authorization`` headers field.
+The endpoints which require these secrets are:
+
+* ``PUT /storage/v1/lease/:storage_index``:
+  The secrets included MUST be ``lease-renew-secret`` and ``lease-cancel-secret``.
+
+* ``POST /storage/v1/immutable/:storage_index``:
+  The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``upload-secret``.
+
+* ``PATCH /storage/v1/immutable/:storage_index/:share_number``:
+  The secrets included MUST be ``upload-secret``.
+
+* ``PUT /storage/v1/immutable/:storage_index/:share_number/abort``:
+  The secrets included MUST be ``upload-secret``.
+
+* ``POST /storage/v1/mutable/:storage_index/read-test-write``:
+  The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``write-enabler``.
+
+If these secrets are:

 1. Missing.
 2. The wrong length.
 3. Not the expected kind of secret.
 4. They are otherwise unparseable before they are actually semantically used.

-the server will respond with ``400 BAD REQUEST``.
+the server MUST respond with ``400 BAD REQUEST`` and perform no other processing of the message.
 401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug.

-If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent.
+If authorization using the secret fails,
+then the server MUST send a ``401 UNAUTHORIZED`` response and perform no other processing of the message.

 Encoding
 ~~~~~~~~

-* ``storage_index`` should be base32 encoded (RFC3548) in URLs.
+* ``storage_index`` MUST be `Base32`_ encoded in URLs.
+* ``share_number`` MUST be a decimal representation

 General
 ~~~~~~~
@ -398,21 +541,27 @@ General
 ``GET /storage/v1/version``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!

-Retrieve information about the version of the storage server.
-Information is returned as an encoded mapping.
-For example::
+This endpoint allows clients to retrieve some basic metadata about a storage server from the storage service.
+The response MUST validate against this CDDL schema::
+
+  {'http://allmydata.org/tahoe/protocols/storage/v1' => {
+      'maximum-immutable-share-size' => uint
+      'maximum-mutable-share-size' => uint
+      'available-space' => uint
+      }
+   'application-version' => bstr
+  }
+
+The server SHOULD populate as many fields as possible with accurate information about its behavior.
+
+For fields which relate to a specific API
+the semantics are documented below in the section for that API.
+For fields that are more general than a single API the semantics are as follows:
+
+* available-space:
+  The server SHOULD use this field to advertise the amount of space that it currently considers unused and is willing to allocate for client requests.
+  The value is a number of bytes.

-  { "http://allmydata.org/tahoe/protocols/storage/v1" :
-    { "maximum-immutable-share-size": 1234,
-      "maximum-mutable-share-size": 1235,
-      "available-space": 123456,
-      "tolerates-immutable-read-overrun": true,
-      "delete-mutable-shares-with-zero-length-writev": true,
-      "fills-holes-with-zero-bytes": true,
-      "prevents-read-past-end-of-share-data": true
-      },
-    "application-version": "1.13.0"
-    }

 ``PUT /storage/v1/lease/:storage_index``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -471,21 +620,37 @@ Writing
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

 Initialize an immutable storage index with some buckets.
-The buckets may have share data written to them once.
-A lease is also created for the shares.
+The server MUST allow share data to be written to the buckets at most one time.
+The server MAY create a lease for the buckets.
 Details of the buckets to create are encoded in the request body.
+The request body MUST validate against this CDDL schema::
+
+  {
+    share-numbers: #6.258([0*256 uint])
+    allocated-size: uint
+  }
+
 For example::

  {"share-numbers": [1, 7, ...], "allocated-size": 12345}

-The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations.
+The server SHOULD accept a value for **allocated-size** that is less than or equal to the lesser of the values of the server's version message's **maximum-immutable-share-size** or **available-space** values.
+
+The request MUST include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations.
 For example::

   X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>
   X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret>
   X-Tahoe-Authorization: upload-secret <base64-upload-secret>

-The response body includes encoded information about the created buckets.
+The response body MUST include encoded information about the created buckets.
+The response body MUST validate against this CDDL schema::
+
+  {
+    already-have: #6.258([0*256 uint])
+    allocated: #6.258([0*256 uint])
+  }
+
 For example::

  {"already-have": [1, ...], "allocated": [7, ...]}
@ -542,27 +707,35 @@ Rejected designs for upload secrets:
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

 Write data for the indicated share.
-The share number must belong to the storage index.
-The request body is the raw share data (i.e., ``application/octet-stream``).
-*Content-Range* requests are required; for large transfers this allows partially complete uploads to be resumed.
+The share number MUST belong to the storage index.
+The request body MUST be the raw share data (i.e., ``application/octet-stream``).
+The request MUST include a *Content-Range* header field;
+for large transfers this allows partially complete uploads to be resumed.
+
 For example,
 a 1MiB share can be divided in to eight separate 128KiB chunks.
 Each chunk can be uploaded in a separate request.
 Each request can include a *Content-Range* value indicating its placement within the complete share.
 If any one of these requests fails then at most 128KiB of upload work needs to be retried.

-The server must recognize when all of the data has been received and mark the share as complete
+The server MUST recognize when all of the data has been received and mark the share as complete
 (which it can do because it was informed of the size when the storage index was initialized).

-The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
+The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret::

    X-Tahoe-Authorization: upload-secret <base64-upload-secret>

 Responses:

-* When a chunk that does not complete the share is successfully uploaded the response is ``OK``.
-  The response body indicates the range of share data that has yet to be uploaded.
-  That is::
+* When a chunk that does not complete the share is successfully uploaded the response MUST be ``OK``.
+  The response body MUST indicate the range of share data that has yet to be uploaded.
+  The response body MUST validate against this CDDL schema::
+
+    {
+      required: [0* {begin: uint, end: uint}]
+    }
+
+  For example::

    { "required":
      [ { "begin": <byte position, inclusive>
@ -573,11 +746,12 @@ Responses:
      ]
    }

-* When the chunk that completes the share is successfully uploaded the response is ``CREATED``.
+* When the chunk that completes the share is successfully uploaded the response MUST be ``CREATED``.
 * If the *Content-Range* for a request covers part of the share that has already,
  and the data does not match already written data,
-  the response is ``CONFLICT``.
-  At this point the only thing to do is abort the upload and start from scratch (see below).
+  the response MUST be ``CONFLICT``.
+  In this case the client MUST abort the upload.
+  The client MAY then restart the upload from scratch.

 Discussion
 ``````````
@ -603,34 +777,42 @@ From RFC 7231::

 This cancels an *in-progress* upload.

-The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
+The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret::

    X-Tahoe-Authorization: upload-secret <base64-upload-secret>

-The response code:
-
-* When the upload is still in progress and therefore the abort has succeeded,
-  the response is ``OK``.
-  Future uploads can start from scratch with no pre-existing upload state stored on the server.
-* If the uploaded has already finished, the response is 405 (Method Not Allowed)
-  and no change is made.
+If there is an incomplete upload with a matching upload-secret then the server MUST consider the abort to have succeeded.
+In this case the response MUST be ``OK``.
+The server MUST respond to all future requests as if the operations related to this upload did not take place.

+If there is no incomplete upload with a matching upload-secret then the server MUST respond with ``Method Not Allowed`` (405).
+The server MUST make no client-visible changes to its state in this case.

 ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

-Advise the server the data read from the indicated share was corrupt. The
-request body includes an human-meaningful text string with details about the
-corruption. It also includes potentially important details about the share.
+Advise the server the data read from the indicated share was corrupt.
+The request body includes an human-meaningful text string with details about the corruption.
+It also includes potentially important details about the share.
+The request body MUST validate against this CDDL schema::
+
+  {
+    reason: tstr .size (1..32765)
+  }

 For example::

  {"reason": "expected hash abcd, got hash efgh"}

-.. share-type, storage-index, and share-number are inferred from the URL
+The report pertains to the immutable share with a **storage index** and **share number** given in the request path.
+If the identified **storage index** and **share number** are known to the server then the response SHOULD be accepted and made available to server administrators.
+In this case the response SHOULD be ``OK``.
+If the response is not accepted then the response SHOULD be ``Not Found`` (404).

-The response code is OK (200) by default, or NOT FOUND (404) if the share
-couldn't be found.
+Discussion
+``````````
+
+The seemingly odd length limit on ``reason`` is chosen so that the *encoded* representation of the message is limited to 32768.

 Reading
 ~~~~~~~
@ -638,26 +820,36 @@ Reading
 ``GET /storage/v1/immutable/:storage_index/shares``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

-Retrieve a list (semantically, a set) indicating all shares available for the
-indicated storage index. For example::
+Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index.
+The response body MUST validate against this CDDL schema::
+
+  #6.258([0*256 uint])
+
+For example::

  [1, 5]

-An unknown storage index results in an empty list.
+If the **storage index** in the request path is not known to the server then the response MUST include an empty list.

 ``GET /storage/v1/immutable/:storage_index/:share_number``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

 Read a contiguous sequence of bytes from one share in one bucket.
-The response body is the raw share data (i.e., ``application/octet-stream``).
-The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content).
-Interpretation and response behavior is as specified in RFC 7233 § 4.1.
-Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported.
+The response body MUST be the raw share data (i.e., ``application/octet-stream``).
+The ``Range`` header MAY be used to request exactly one ``bytes`` range,
+in which case the response code MUST be ``Partial Content`` (206).
+Interpretation and response behavior MUST be as specified in RFC 7233 § 4.1.
+Multiple ranges in a single request are *not* supported;
+open-ended ranges are also not supported.
+Clients MUST NOT send requests using these features.

-If the response reads beyond the end of the data, the response may be shorter than the requested range.
-The resulting ``Content-Range`` header will be consistent with the returned data.
+If the response reads beyond the end of the data,
+the response MUST be shorter than the requested range.
+It MUST contain all data up to the end of the share and then end.
+The resulting ``Content-Range`` header MUST be consistent with the returned data.

-If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used.
+If the response to a query is an empty range,
+the server MUST send a ``No Content`` (204) response.

 Discussion
 ``````````
@ -696,13 +888,27 @@ The first write operation on a mutable storage index creates it
 (that is,
 there is no separate "create this storage index" operation as there is for the immutable storage index type).

-The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets::
+The request MUST include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets::

    X-Tahoe-Authorization: write-enabler <base64-write-enabler-secret>
    X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret>
    X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>

-The request body includes test, read, and write vectors for the operation.
+The request body MUST include test, read, and write vectors for the operation.
+The request body MUST validate against this CDDL schema::
+
+  {
+    "test-write-vectors": {
+      0*256 share_number : {
+        "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}]
+        "write": [* {"offset": uint, "data": bstr}]
+        "new-length": uint / null
+      }
+    }
+    "read-vector": [0*30 {"offset": uint, "size": uint}]
+  }
+  share_number = uint
+
 For example::

   {
@ -725,6 +931,14 @@ For example::

 The response body contains a boolean indicating whether the tests all succeed
 (and writes were applied) and a mapping giving read data (pre-write).
+The response body MUST validate against this CDDL schema::
+
+  {
+    "success": bool,
+    "data": {0*256 share_number: [0* bstr]}
+  }
+  share_number = uint
+
 For example::

  {
@ -736,8 +950,17 @@ For example::
      }
  }

-A test vector or read vector that read beyond the boundaries of existing data will return nothing for any bytes past the end.
-As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length.
+A client MAY send a test vector or read vector to bytes beyond the end of existing data.
+In this case a server MUST behave as if the test or read vector referred to exactly as much data exists.
+
+For example,
+consider the case where the server has 5 bytes of data for a particular share.
+If a client sends a read vector with an ``offset`` of 1 and a ``size`` of 4 then the server MUST respond with all of the data except the first byte.
+If a client sends a read vector with the same ``offset`` and a ``size`` of 5 (or any larger value) then the server MUST respond in the same way.
+
+Similarly,
+if there is no data at all,
+an empty byte string is returned no matter what the offset or length.

 Reading
 ~~~~~~~
@ -746,23 +969,34 @@ Reading
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

 Retrieve a set indicating all shares available for the indicated storage index.
-For example (this is shown as list, since it will be list for JSON, but will be set for CBOR)::
+The response body MUST validate against this CDDL schema::
+
+  #6.258([0*256 uint])
+
+For example::

  [1, 5]

 ``GET /storage/v1/mutable/:storage_index/:share_number``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

-Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``
+Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``.

-The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content).
-Interpretation and response behavior is as specified in RFC 7233 § 4.1.
-Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported.
+The response body MUST be the raw share data (i.e., ``application/octet-stream``).
+The ``Range`` header MAY be used to request exactly one ``bytes`` range,
+in which case the response code MUST be ``Partial Content`` (206).
+Interpretation and response behavior MUST be specified in RFC 7233 § 4.1.
+Multiple ranges in a single request are *not* supported;
+open-ended ranges are also not supported.
+Clients MUST NOT send requests using these features.

-If the response reads beyond the end of the data, the response may be shorter than the requested range.
-The resulting ``Content-Range`` header will be consistent with the returned data.
+If the response reads beyond the end of the data,
+the response MUST be shorter than the requested range.
+It MUST contain all data up to the end of the share and then end.
+The resulting ``Content-Range`` header MUST be consistent with the returned data.

-If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used.
+If the response to a query is an empty range,
+the server MUST send a ``No Content`` (204) response.


 ``POST /storage/v1/mutable/:storage_index/:share_number/corrupt``
@ -774,6 +1008,9 @@ Just like the immutable version.
 Sample Interactions
 -------------------

+This section contains examples of client/server interactions to help illuminate the above specification.
+This section is non-normative.
+
 Immutable Data
 ~~~~~~~~~~~~~~

@ -926,10 +1163,16 @@ otherwise it will read a byte which won't match `b""`::

     204 NO CONTENT

+.. _Base64: https://www.rfc-editor.org/rfc/rfc4648#section-4
+
+.. _RFC 4648: https://tools.ietf.org/html/rfc4648
+
 .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4

 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4

+.. _RFC 9110: https://tools.ietf.org/html/rfc9110
+
 .. _CBOR: http://cbor.io/

 .. [#]
@ -974,7 +1217,7 @@ otherwise it will read a byte which won't match `b""`::
        spki_encoded = urlsafe_b64encode(spki_sha256)
        assert spki_encoded == tub_id

-   Note we use `base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32.
+   Note we use `unpadded base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32.

 .. [#]
   https://www.cvedetails.com/cve/CVE-2017-5638/
@ -985,6 +1228,6 @@ otherwise it will read a byte which won't match `b""`::
 .. [#]
   https://efail.de/

-.. _base64url: https://tools.ietf.org/html/rfc7515#appendix-C
+.. _unpadded base64url: https://tools.ietf.org/html/rfc7515#appendix-C

 .. _attacking SHA1: https://en.wikipedia.org/wiki/SHA-1#Attacks
--- a/docs/specifications/dirnodes.rst
+++ b/docs/specifications/dirnodes.rst
@ -267,7 +267,7 @@ How well does this design meet the goals?
   value, so there are no opportunities for staleness
 9. monotonicity: VERY: the single point of access also protects against
   retrograde motion
-     
+


 Confidentiality leaks in the storage servers
@ -332,8 +332,9 @@ MDMF design rules allow for efficient random-access reads from the middle of
 the file, which would give the index something useful to point at.

 The current SDMF design generates a new RSA public/private keypair for each
-directory. This takes considerable time and CPU effort, generally one or two
-seconds per directory. We have designed (but not yet built) a DSA-based
+directory. This takes some time and CPU effort (around 100 milliseconds on a
+relatively high-end 2021 laptop) per directory.
+We have designed (but not yet built) a DSA-based
 mutable file scheme which will use shared parameters to reduce the
 directory-creation effort to a bare minimum (picking a random number instead
 of generating two random primes).
@ -363,7 +364,7 @@ single child, looking up a single child) would require pulling or pushing a
 lot of unrelated data, increasing network overhead (and necessitating
 test-and-set semantics for the modification side, which increases the chances
 that a user operation will fail, making it more challenging to provide
-promises of atomicity to the user). 
+promises of atomicity to the user).

 It would also make it much more difficult to enable the delegation
 ("sharing") of specific directories. Since each aggregate "realm" provides
@ -469,4 +470,3 @@ Preventing delegation between communication parties is just as pointless as
 asking Bob to forget previously accessed files. However, there may be value
 to configuring the UI to ask Carol to not share files with Bob, or to
 removing all files from Bob's view at the same time his access is revoked.
-
--- a/integration/conftest.py
+++ b/integration/conftest.py
@ -1,6 +1,10 @@
 """
 Ported to Python 3.
 """
+
+from __future__ import annotations
+
+import os
 import sys
 import shutil
 from time import sleep
@ -19,6 +23,7 @@ from eliot import (
    log_call,
 )

+from twisted.python.filepath import FilePath
 from twisted.python.procutils import which
 from twisted.internet.defer import DeferredList
 from twisted.internet.error import (
@ -45,7 +50,16 @@ from .util import (
    generate_ssh_key,
    block_with_timeout,
 )
+from allmydata.node import read_config

+# No reason for HTTP requests to take longer than four minutes in the
+# integration tests. See allmydata/scripts/common_http.py for usage.
+os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "240"
+
+# Make Foolscap logging go into Twisted logging, so that integration test logs
+# include extra information
+# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst):
+os.environ["FLOGTOTWISTED"] = "1"

 # pytest customization hooks

@ -106,7 +120,7 @@ def reactor():

@pytest.fixture(scope='session')
@log_call(action_type=u"integration:temp_dir", include_args=[])
-def temp_dir(request):
+def temp_dir(request) -> str:
    """
    Invoke like 'py.test --keep-tempdir ...' to avoid deleting the temp-dir
    """
@ -148,11 +162,12 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
            '--location', 'tcp:localhost:3117',
            '--port', '3117',
            gather_dir,
-        )
+        ),
+        env=environ,
    )
    pytest_twisted.blockon(out_protocol.done)

-    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
+    twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer")
    twistd_process = reactor.spawnProcess(
        twistd_protocol,
        which('twistd')[0],
@ -161,6 +176,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
            join(gather_dir, 'gatherer.tac'),
        ),
        path=gather_dir,
+        env=environ,
    )
    pytest_twisted.blockon(twistd_protocol.magic_seen)

@ -179,6 +195,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
            (
                'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
            ),
+            env=environ,
        )
        print("Waiting for flogtool to complete")
        try:
@ -201,13 +218,6 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
    include_result=False,
 )
 def introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer0
-web.port = 4560
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
    intro_dir = join(temp_dir, 'introducer')
    print("making introducer", intro_dir)

@ -227,13 +237,14 @@ log_gatherer.furl = {log_furl}
        )
        pytest_twisted.blockon(done_proto.done)

-    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4562")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)

    # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
    # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "introducer")
    transport = _tahoe_runner_optional_coverage(
        protocol,
        reactor,
@ -270,22 +281,16 @@ def introducer_furl(introducer, temp_dir):
    return furl


-@pytest.fixture(scope='session')
+@pytest.fixture
@log_call(
    action_type=u"integration:tor:introducer",
    include_args=["temp_dir", "flog_gatherer"],
    include_result=False,
 )
 def tor_introducer(reactor, temp_dir, flog_gatherer, request, tor_network):
-    config = '''
-[node]
-nickname = introducer_tor
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
    intro_dir = join(temp_dir, 'introducer_tor')
-    print("making introducer", intro_dir)
+    print("making Tor introducer in {}".format(intro_dir))
+    print("(this can take tens of seconds to allocate Onion address)")

    if not exists(intro_dir):
        mkdir(intro_dir)
@ -296,20 +301,25 @@ log_gatherer.furl = {log_furl}
            request,
            (
                'create-introducer',
-                '--tor-control-port', 'tcp:localhost:8010',
+                # The control port should agree with the configuration of the
+                # Tor network we bootstrap with chutney.
+                '--tor-control-port', 'tcp:localhost:8007',
+                '--hide-ip',
                '--listen=tor',
                intro_dir,
            ),
        )
        pytest_twisted.blockon(done_proto.done)

-    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    # adjust a few settings
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)

    # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
    # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "tor_introducer")
    transport = _tahoe_runner_optional_coverage(
        protocol,
        reactor,
@ -328,17 +338,20 @@ log_gatherer.furl = {log_furl}
            pass
    request.addfinalizer(cleanup)

+    print("Waiting for introducer to be ready...")
    pytest_twisted.blockon(protocol.magic_seen)
+    print("Introducer ready.")
    return transport


-@pytest.fixture(scope='session')
+@pytest.fixture
 def tor_introducer_furl(tor_introducer, temp_dir):
    furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl')
    while not exists(furl_fname):
        print("Don't see {} yet".format(furl_fname))
        sleep(.1)
    furl = open(furl_fname, 'r').read()
+    print(f"Found Tor introducer furl: {furl} in {furl_fname}")
    return furl


@ -390,12 +403,9 @@ def alice(
            reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
            web_port="tcp:9980:interface=localhost",
            storage=False,
-            # We're going to kill this ourselves, so no need for finalizer to
-            # do it:
-            finalize=False,
        )
    )
-    await_client_ready(process)
+    pytest_twisted.blockon(await_client_ready(process))

    # 1. Create a new RW directory cap:
    cli(process, "create-alias", "test")
@ -426,7 +436,7 @@ alice-key ssh-rsa {ssh_public_key} {rwcap}

    # 4. Restart the node with new SFTP config.
    pytest_twisted.blockon(process.restart_async(reactor, request))
-    await_client_ready(process)
+    pytest_twisted.blockon(await_client_ready(process))
    print(f"Alice pid: {process.transport.pid}")
    return process

@ -441,22 +451,37 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques
            storage=False,
        )
    )
-    await_client_ready(process)
+    pytest_twisted.blockon(await_client_ready(process))
    return process


@pytest.fixture(scope='session')
@pytest.mark.skipif(sys.platform.startswith('win'),
                    'Tor tests are unstable on Windows')
-def chutney(reactor, temp_dir):
+def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
+    # Try to find Chutney already installed in the environment.
+    try:
+        import chutney
+    except ImportError:
+        # Nope, we'll get our own in a moment.
+        pass
+    else:
+        # We already have one, just use it.
+        return (
+            # from `checkout/lib/chutney/__init__.py` we want to get back to
+            # `checkout` because that's the parent of the directory with all
+            # of the network definitions.  So, great-grand-parent.
+            FilePath(chutney.__file__).parent().parent().parent().path,
+            # There's nothing to add to the environment.
+            {},
+        )

    chutney_dir = join(temp_dir, 'chutney')
    mkdir(chutney_dir)

-    # TODO:
-
-    # check for 'tor' binary explicitly and emit a "skip" if we can't
-    # find it
+    missing = [exe for exe in ["tor", "tor-gencert"] if not which(exe)]
+    if missing:
+        pytest.skip(f"Some command-line tools not found: {missing}")

    # XXX yuck! should add a setup.py to chutney so we can at least
    # "pip install <path to tarball>" and/or depend on chutney in "pip
@ -468,7 +493,7 @@ def chutney(reactor, temp_dir):
        executable='git',
        argv=(
            'git', 'clone',
-            'https://git.torproject.org/chutney.git',
+            'https://gitlab.torproject.org/tpo/core/chutney.git',
            chutney_dir,
        ),
        env=environ,
@ -483,79 +508,68 @@ def chutney(reactor, temp_dir):
        argv=(
            'git', '-C', chutney_dir,
            'reset', '--hard',
-            'c825cba0bcd813c644c6ac069deeb7347d3200ee'
+            'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2'
        ),
        env=environ,
        path=".",
    ))

-    return chutney_dir
+    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})


@pytest.fixture(scope='session')
@pytest.mark.skipif(sys.platform.startswith('win'),
                    reason='Tor tests are unstable on Windows')
 def tor_network(reactor, temp_dir, chutney, request):
+    """
+    Build a basic Tor network.

-    # this is the actual "chutney" script at the root of a chutney checkout
-    chutney_dir = chutney
-    chut = join(chutney_dir, 'chutney')
+    :param chutney: The root directory of a Chutney checkout and a dict of
+        additional environment variables to set so a Python process can use
+        it.

-    # now, as per Chutney's README, we have to create the network
-    # ./chutney configure networks/basic
-    # ./chutney start networks/basic
+    :return: None
+    """
+    chutney_root, chutney_env = chutney
+    basic_network = join(chutney_root, 'networks', 'basic')

    env = environ.copy()
-    env.update({"PYTHONPATH": join(chutney_dir, "lib")})
-    pytest_twisted.blockon(dump_python_output(
-        reactor,
-        argv=[
-            '-m', 'chutney.TorNet', 'configure',
-            join(chutney_dir, 'networks', 'basic'),
-        ],
-        path=join(chutney_dir),
-    ))
+    env.update(chutney_env)
+    env.update({
+        # default is 60, probably too short for reliable automated use.
+        "CHUTNEY_START_TIME": "600",
+    })
+    chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
+    def chutney(argv):
+        proto = _DumpOutputProtocol(None)
+        reactor.spawnProcess(
+            proto,
+            sys.executable,
+            chutney_argv + argv,
+            path=join(chutney_root),
+            env=env,
+        )
+        return proto.done

-    pytest_twisted.blockon(dump_python_output(
-        reactor,
-        argv=[
-            '-m', 'chutney.TorNet', 'start',
-            join(chutney_dir, 'networks', 'basic'),
-        ],
-        path=join(chutney_dir),
-    ))
-
-    # print some useful stuff
-    d = dump_python_output(
-        reactor,
-        argv=[
-            '-m', 'chutney.TorNet', 'status',
-            join(chutney_dir, 'networks', 'basic'),
-        ],
-        path=join(chutney_dir),
-    )
-    try:
-        pytest_twisted.blockon(d)
-    except ProcessTerminated:
-        print("Chutney.TorNet status failed (continuing)")
+    # now, as per Chutney's README, we have to create the network
+    pytest_twisted.blockon(chutney(("configure", basic_network)))

+    # before we start the network, ensure we will tear down at the end
    def cleanup():
        print("Tearing down Chutney Tor network")
-        d = dump_python_output(
-            reactor,
-            argv=[
-                '-m', 'chutney.TorNet', 'stop',
-                join(chutney_dir, 'networks', 'basic'),
-            ],
-            path=join(chutney_dir),
-        )
        try:
-            block_with_timeout(d, reactor)
+            block_with_timeout(chutney(("stop", basic_network)), reactor)
        except ProcessTerminated:
            # If this doesn't exit cleanly, that's fine, that shouldn't fail
            # the test suite.
            pass
-
    request.addfinalizer(cleanup)

-    return chut
+    pytest_twisted.blockon(chutney(("start", basic_network)))
+    pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network)))
+
+    # print some useful stuff
+    try:
+        pytest_twisted.blockon(chutney(("status", basic_network)))
+    except ProcessTerminated:
+        print("Chutney.TorNet status failed (continuing)")
--- a/integration/test_get_put.py
+++ b/integration/test_get_put.py
@ -4,11 +4,11 @@ and stdout.
 """

 from subprocess import Popen, PIPE, check_output, check_call
-import sys

 import pytest
-from pytest_twisted import ensureDeferred
 from twisted.internet import reactor
+from twisted.internet.threads import blockingCallFromThread
+from twisted.internet.defer import Deferred

 from .util import run_in_thread, cli, reconfigure

@ -50,6 +50,7 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir):
    assert read_bytes(tempfile) == DATA


+@run_in_thread
 def test_get_to_stdout(alice, get_put_alias, tmpdir):
    """
    It's possible to upload a file, and then download it to stdout.
@ -67,6 +68,7 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir):
    assert p.wait() == 0


+@run_in_thread
 def test_large_file(alice, get_put_alias, tmp_path):
    """
    It's possible to upload and download a larger file.
@ -85,12 +87,8 @@ def test_large_file(alice, get_put_alias, tmp_path):
    assert outfile.read_bytes() == tempfile.read_bytes()


-@pytest.mark.skipif(
-    sys.platform.startswith("win"),
-    reason="reconfigure() has issues on Windows"
-)
-@ensureDeferred
-async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
+@run_in_thread
+def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
    """
    Tahoe-LAFS used to have a default max segment size of 128KB, and is now
    1MB.  Test that an upload created when 128KB was the default can be
@ -103,22 +101,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
    with tempfile.open("wb") as f:
        f.write(large_data)

-    async def set_segment_size(segment_size):
-        await reconfigure(
+    def set_segment_size(segment_size):
+        return blockingCallFromThread(
            reactor,
-            request,
-            alice,
-            (1, 1, 1),
-            None,
-            max_segment_size=segment_size
-    )
+            lambda: Deferred.fromCoroutine(reconfigure(
+                reactor,
+                request,
+                alice,
+                (1, 1, 1),
+                None,
+                max_segment_size=segment_size
+            ))
+        )

    # 1. Upload file 1 with default segment size set to 1MB
-    await set_segment_size(1024 * 1024)
+    set_segment_size(1024 * 1024)
    cli(alice, "put", str(tempfile), "getput:seg1024kb")

    # 2. Download file 1 with default segment size set to 128KB
-    await set_segment_size(128 * 1024)
+    set_segment_size(128 * 1024)
    assert large_data == check_output(
        ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
    )
@ -127,7 +128,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
    cli(alice, "put", str(tempfile), "getput:seg128kb")

    # 4. Download file 2 with default segment size set to 1MB
-    await set_segment_size(1024 * 1024)
+    set_segment_size(1024 * 1024)
    assert large_data == check_output(
        ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
    )
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@ -2,26 +2,11 @@
 Integration tests for I2P support.
 """

-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 import sys
 from os.path import join, exists
-from os import mkdir
+from os import mkdir, environ
 from time import sleep
-
-if PY2:
-    def which(path):
-        # This will result in skipping I2P tests on Python 2. Oh well.
-        return None
-else:
-    from shutil import which
+from shutil import which

 from eliot import log_call

@ -38,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready
 from allmydata.test.common import (
    write_introducer,
 )
+from allmydata.node import read_config
+

 if which("docker") is None:
    pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True)
@ -50,7 +37,7 @@ if sys.platform.startswith('win'):
@pytest.fixture
 def i2p_network(reactor, temp_dir, request):
    """Fixture to start up local i2pd."""
-    proto = util._MagicTextProtocol("ephemeral keys")
+    proto = util._MagicTextProtocol("ephemeral keys", "i2pd")
    reactor.spawnProcess(
        proto,
        which("docker"),
@ -62,6 +49,7 @@ def i2p_network(reactor, temp_dir, request):
            "--log=stdout",
            "--loglevel=info"
        ),
+        env=environ,
    )

    def cleanup():
@ -82,13 +70,6 @@ def i2p_network(reactor, temp_dir, request):
    include_result=False,
 )
 def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_i2p
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
    intro_dir = join(temp_dir, 'introducer_i2p')
    print("making introducer", intro_dir)

@ -108,12 +89,14 @@ log_gatherer.furl = {log_furl}
        pytest_twisted.blockon(done_proto.done)

    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer_i2p")
+    config.set_config("node", "web.port", "4563")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)

    # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
    # "start" command.
-    protocol = util._MagicTextProtocol('introducer running')
+    protocol = util._MagicTextProtocol('introducer running', "introducer")
    transport = util._tahoe_runner_optional_coverage(
        protocol,
        reactor,
@ -147,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):


@pytest_twisted.inlineCallbacks
+@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons")
 def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
    yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
    yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
@ -170,7 +154,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw
            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
            '-d', join(temp_dir, 'carol_i2p'),
            'put', gold_path,
-        )
+        ),
+        env=environ,
    )
    yield proto.done
    cap = proto.output.getvalue().strip().split()[-1]
@ -184,7 +169,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw
            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
            '-d', join(temp_dir, 'dave_i2p'),
            'get', cap,
-        )
+        ),
+        env=environ,
    )
    yield proto.done

@ -211,7 +197,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
            '--hide-ip',
            '--listen', 'i2p',
            node_dir.path,
-        )
+        ),
+        env=environ,
    )
    yield proto.done

--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@ -1,17 +1,10 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401

 import sys
 from os.path import join
+from os import environ

 from twisted.internet.error import ProcessTerminated

@ -31,7 +24,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
        happy=7,
        total=10,
    )
-    util.await_client_ready(edna)
+    yield util.await_client_ready(edna)

    node_dir = join(temp_dir, 'edna')

@ -45,7 +38,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
            '-d', node_dir,
            'put', __file__,
-        ]
+        ],
+        env=environ,
    )
    try:
        yield proto.done
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@ -1,14 +1,6 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401

 import sys
 from os.path import join
@ -26,6 +18,8 @@ from twisted.python.filepath import (
 from allmydata.test.common import (
    write_introducer,
 )
+from allmydata.client import read_config
+from allmydata.util.deferredutil import async_to_deferred

 # see "conftest.py" for the fixtures (e.g. "tor_network")

@ -36,18 +30,28 @@ from allmydata.test.common import (
 if sys.platform.startswith('win'):
    pytest.skip('Skipping Tor tests on Windows', allow_module_level=True)

-if PY2:
-    pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True)
-
@pytest_twisted.inlineCallbacks
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
-    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    util.await_client_ready(carol, minimum_number_of_servers=2)
-    util.await_client_ready(dave, minimum_number_of_servers=2)
+    """
+    Two nodes and an introducer all configured to use Tahoe.
+
+    The two nodes can talk to the introducer and each other: we upload to one
+    node, read from the other.
+    """
+    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
+    dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave)
+
+
+@async_to_deferred
+async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: util.TahoeProcess, download_from: util.TahoeProcess):
+    """
+    Ensure both nodes are connected to "a grid" by uploading something via one
+    node, and retrieve it using the other.
+    """

-    # ensure both nodes are connected to "a grid" by uploading
-    # something via carol, and retrieve it using dave.
    gold_path = join(temp_dir, "gold")
    with open(gold_path, "w") as f:
        f.write(
@ -64,14 +68,14 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
        sys.executable,
        (
            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
-            '-d', join(temp_dir, 'carol'),
+            '-d', upload_to.node_dir,
            'put', gold_path,
        ),
        env=environ,
    )
-    yield proto.done
+    await proto.done
    cap = proto.output.getvalue().strip().split()[-1]
-    print("TEH CAP!", cap)
+    print("capability: {}".format(cap))

    proto = util._CollectOutputProtocol(capture_stderr=False)
    reactor.spawnProcess(
@ -79,23 +83,23 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
        sys.executable,
        (
            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
-            '-d', join(temp_dir, 'dave'),
+            '-d', download_from.node_dir,
            'get', cap,
-        )
+        ),
+        env=environ,
    )
-    yield proto.done
-
-    dave_got = proto.output.getvalue().strip()
-    assert dave_got == open(gold_path, 'rb').read().strip()
+    await proto.done
+    download_got = proto.output.getvalue().strip()
+    assert download_got == open(gold_path, 'rb').read().strip()


@pytest_twisted.inlineCallbacks
-def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
+def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess:
    node_dir = FilePath(temp_dir).child(name)
    web_port = "tcp:{}:interface=localhost".format(control_port + 2000)

    if True:
-        print("creating", node_dir.path)
+        print(f"creating {node_dir.path} with introducer {introducer_furl}")
        node_dir.makedirs()
        proto = util._DumpOutputProtocol(None)
        reactor.spawnProcess(
@ -105,49 +109,55 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                sys.executable, '-b', '-m', 'allmydata.scripts.runner',
                'create-node',
                '--nickname', name,
+                '--webport', web_port,
                '--introducer', introducer_furl,
                '--hide-ip',
                '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
                '--listen', 'tor',
+                '--shares-needed', '1',
+                '--shares-happy', '1',
+                '--shares-total', str(shares_total),
                node_dir.path,
-            )
+            ),
+            env=environ,
        )
        yield proto.done


    # Which services should this client connect to?
    write_introducer(node_dir, "default", introducer_furl)
-    with node_dir.child('tahoe.cfg').open('w') as f:
-        node_config = '''
-[node]
-nickname = %(name)s
-web.port = %(web_port)s
-web.static = public_html
-log_gatherer.furl = %(log_furl)s
+    util.basic_node_configuration(request, flog_gatherer, node_dir.path)

-[tor]
-control.port = tcp:localhost:%(control_port)d
-onion.external_port = 3457
-onion.local_port = %(local_port)d
-onion = true
-onion.private_key_file = private/tor_onion.privkey
-
-[client]
-shares.needed = 1
-shares.happy = 1
-shares.total = 2
-
-''' % {
-    'name': name,
-    'web_port': web_port,
-    'log_furl': flog_gatherer,
-    'control_port': control_port,
-    'local_port': control_port + 1000,
-}
-        node_config = node_config.encode("utf-8")
-        f.write(node_config)
+    config = read_config(node_dir.path, "tub.port")
+    config.set_config("tor", "onion", "true")
+    config.set_config("tor", "onion.external_port", "3457")
+    config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
+    config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")

    print("running")
    result = yield util._run_node(reactor, node_dir.path, request, None)
    print("okay, launched")
    return result
+
+@pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS')
+@pytest_twisted.inlineCallbacks
+def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
+    """
+    A normal node (normie) and a normal introducer are configured, and one node
+    (anonymoose) which is configured to be anonymous by talking via Tor.
+
+    Anonymoose should be able to communicate with normie.
+
+    TODO how to ensure that anonymoose is actually using Tor?
+    """
+    normie = yield util._create_node(
+        reactor, request, temp_dir, introducer_furl, flog_gatherer, "normie",
+        web_port="tcp:9989:interface=localhost",
+        storage=True, needed=1, happy=1, total=1,
+    )
+    yield util.await_client_ready(normie)
+
+    anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1)
+    yield util.await_client_ready(anonymoose, minimum_number_of_servers=1, timeout=600)
+
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, normie, anonymoose)
--- a/integration/test_web.py
+++ b/integration/test_web.py
@ -14,17 +14,21 @@ from __future__ import annotations
 import time
 from urllib.parse import unquote as url_unquote, quote as url_quote

+from twisted.internet.threads import deferToThread
+
 import allmydata.uri
 from allmydata.util import jsonbytes as json

 from . import util
+from .util import run_in_thread

 import requests
 import html5lib
 from bs4 import BeautifulSoup

-from pytest_twisted import ensureDeferred
+import pytest_twisted

+@run_in_thread
 def test_index(alice):
    """
    we can download the index file
@ -32,6 +36,7 @@ def test_index(alice):
    util.web_get(alice, u"")


+@run_in_thread
 def test_index_json(alice):
    """
    we can download the index file as json
@ -41,6 +46,7 @@ def test_index_json(alice):
    json.loads(data)


+@run_in_thread
 def test_upload_download(alice):
    """
    upload a file, then download it via readcap
@ -70,6 +76,7 @@ def test_upload_download(alice):
    assert str(data, "utf-8") == FILE_CONTENTS


+@run_in_thread
 def test_put(alice):
    """
    use PUT to create a file
@ -89,6 +96,7 @@ def test_put(alice):
    assert cap.needed_shares == int(cfg.get_config("client", "shares.needed"))


+@run_in_thread
 def test_helper_status(storage_nodes):
    """
    successfully GET the /helper_status page
@ -101,6 +109,7 @@ def test_helper_status(storage_nodes):
    assert str(dom.h1.string) == u"Helper Status"


+@run_in_thread
 def test_deep_stats(alice):
    """
    create a directory, do deep-stats on it and prove the /operations/
@ -178,7 +187,7 @@ def test_deep_stats(alice):
        time.sleep(.5)


-@util.run_in_thread
+@run_in_thread
 def test_status(alice):
    """
    confirm we get something sensible from /status and the various sub-types
@ -244,7 +253,7 @@ def test_status(alice):
    assert found_download, "Failed to find the file we downloaded in the status-page"


-@ensureDeferred
+@pytest_twisted.ensureDeferred
 async def test_directory_deep_check(reactor, request, alice):
    """
    use deep-check and confirm the result pages work
@ -256,7 +265,10 @@ async def test_directory_deep_check(reactor, request, alice):
    total = 4

    await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
+    await deferToThread(_test_directory_deep_check_blocking, alice)

+
+def _test_directory_deep_check_blocking(alice):
    # create a directory
    resp = requests.post(
        util.node_url(alice.node_dir, u"uri"),
@ -417,6 +429,7 @@ async def test_directory_deep_check(reactor, request, alice):
    assert dom is not None, "Operation never completed"


+@run_in_thread
 def test_storage_info(storage_nodes):
    """
    retrieve and confirm /storage URI for one storage node
@ -428,6 +441,7 @@ def test_storage_info(storage_nodes):
    )


+@run_in_thread
 def test_storage_info_json(storage_nodes):
    """
    retrieve and confirm /storage?t=json URI for one storage node
@ -442,6 +456,7 @@ def test_storage_info_json(storage_nodes):
    assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000


+@run_in_thread
 def test_introducer_info(introducer):
    """
    retrieve and confirm /introducer URI for the introducer
@ -460,6 +475,7 @@ def test_introducer_info(introducer):
    assert "subscription_summary" in data


+@run_in_thread
 def test_mkdir_with_children(alice):
    """
    create a directory using ?t=mkdir-with-children
--- a/integration/util.py
+++ b/integration/util.py
@ -12,7 +12,7 @@ import sys
 import time
 import json
 from os import mkdir, environ
-from os.path import exists, join
+from os.path import exists, join, basename
 from io import StringIO, BytesIO
 from subprocess import check_output

@ -127,7 +127,6 @@ class _CollectOutputProtocol(ProcessProtocol):
        self.output.write(data)

    def errReceived(self, data):
-        print("ERR: {!r}".format(data))
        if self.capture_stderr:
            self.output.write(data)

@ -163,8 +162,9 @@ class _MagicTextProtocol(ProcessProtocol):
    and then .callback()s on self.done and .errback's if the process exits
    """

-    def __init__(self, magic_text):
+    def __init__(self, magic_text: str, name: str) -> None:
        self.magic_seen = Deferred()
+        self.name = f"{name}: "
        self.exited = Deferred()
        self._magic_text = magic_text
        self._output = StringIO()
@ -174,7 +174,8 @@ class _MagicTextProtocol(ProcessProtocol):

    def outReceived(self, data):
        data = str(data, sys.stdout.encoding)
-        sys.stdout.write(data)
+        for line in data.splitlines():
+            sys.stdout.write(self.name + line + "\n")
        self._output.write(data)
        if not self.magic_seen.called and self._magic_text in self._output.getvalue():
            print("Saw '{}' in the logs".format(self._magic_text))
@ -182,7 +183,8 @@ class _MagicTextProtocol(ProcessProtocol):

    def errReceived(self, data):
        data = str(data, sys.stderr.encoding)
-        sys.stdout.write(data)
+        for line in data.splitlines():
+            sys.stdout.write(self.name + line + "\n")


 def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
@ -317,7 +319,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
    """
    if magic_text is None:
        magic_text = "client running"
-    protocol = _MagicTextProtocol(magic_text)
+    protocol = _MagicTextProtocol(magic_text, basename(node_dir))

    # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
    # "start" command.
@ -346,6 +348,36 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
    return d


+def basic_node_configuration(request, flog_gatherer, node_dir: str):
+    """
+    Setup common configuration options for a node, given a ``pytest`` request
+    fixture.
+    """
+    config_path = join(node_dir, 'tahoe.cfg')
+    config = get_config(config_path)
+    set_config(
+        config,
+        u'node',
+        u'log_gatherer.furl',
+        flog_gatherer,
+    )
+    force_foolscap = request.config.getoption("force_foolscap")
+    assert force_foolscap in (True, False)
+    set_config(
+        config,
+        'storage',
+        'force_foolscap',
+        str(force_foolscap),
+    )
+    set_config(
+        config,
+        'client',
+        'force_foolscap',
+        str(force_foolscap),
+    )
+    write_config(FilePath(config_path), config)
+
+
 def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port,
                 storage=True,
                 magic_text=None,
@ -386,29 +418,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
        created_d = done_proto.done

        def created(_):
-            config_path = join(node_dir, 'tahoe.cfg')
-            config = get_config(config_path)
-            set_config(
-                config,
-                u'node',
-                u'log_gatherer.furl',
-                flog_gatherer,
-            )
-            force_foolscap = request.config.getoption("force_foolscap")
-            assert force_foolscap in (True, False)
-            set_config(
-                config,
-                'storage',
-                'force_foolscap',
-                str(force_foolscap),
-            )
-            set_config(
-                config,
-                'client',
-                'force_foolscap',
-                str(force_foolscap),
-            )
-            write_config(FilePath(config_path), config)
+            basic_node_configuration(request, flog_gatherer, node_dir)
        created_d.addCallback(created)

    d = Deferred()
@ -465,6 +475,31 @@ class FileShouldVanishException(Exception):
        )


+def run_in_thread(f):
+    """Decorator for integration tests that runs code in a thread.
+
+    Because we're using pytest_twisted, tests that rely on the reactor are
+    expected to return a Deferred and use async APIs so the reactor can run.
+
+    In the case of the integration test suite, it launches nodes in the
+    background using Twisted APIs.  The nodes stdout and stderr is read via
+    Twisted code.  If the reactor doesn't run, reads don't happen, and
+    eventually the buffers fill up, and the nodes block when they try to flush
+    logs.
+
+    We can switch to Twisted APIs (treq instead of requests etc.), but
+    sometimes it's easier or expedient to just have a blocking test.  So this
+    decorator allows you to run the test in a thread, and the reactor can keep
+    running in the main thread.
+
+    See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
+    """
+    @wraps(f)
+    def test(*args, **kwargs):
+        return deferToThread(lambda: f(*args, **kwargs))
+    return test
+
+
 def await_file_contents(path, contents, timeout=15, error_if=None):
    """
    wait up to `timeout` seconds for the file at `path` (any path-like
@ -590,6 +625,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
    return resp.content


+@run_in_thread
 def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
    """
    Uses the status API to wait for a client-type node (in `tahoe`, a
@ -614,24 +650,25 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
            print("waiting because '{}'".format(e))
            time.sleep(1)
            continue
+        servers = js['servers']

-        if len(js['servers']) < minimum_number_of_servers:
-            print("waiting because insufficient servers")
+        if len(servers) < minimum_number_of_servers:
+            print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})")
            time.sleep(1)
            continue
+
+        print(
+            f"Now: {time.ctime()}\n"
+            f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}"
+        )
+
        server_times = [
            server['last_received_data']
-            for server in js['servers']
+            for server in servers
        ]
-        # if any times are null/None that server has never been
-        # contacted (so it's down still, probably)
-        if any(t is None for t in server_times):
-            print("waiting because at least one server not contacted")
-            time.sleep(1)
-            continue
-
-        # check that all times are 'recent enough'
-        if any([time.time() - t > liveness for t in server_times]):
+        # check that all times are 'recent enough' (it's OK if _some_ servers
+        # are down, we just want to make sure a sufficient number are up)
+        if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers:
            print("waiting because at least one server too old")
            time.sleep(1)
            continue
@ -657,30 +694,6 @@ def generate_ssh_key(path):
        f.write(s.encode("ascii"))


-def run_in_thread(f):
-    """Decorator for integration tests that runs code in a thread.
-
-    Because we're using pytest_twisted, tests that rely on the reactor are
-    expected to return a Deferred and use async APIs so the reactor can run.
-
-    In the case of the integration test suite, it launches nodes in the
-    background using Twisted APIs.  The nodes stdout and stderr is read via
-    Twisted code.  If the reactor doesn't run, reads don't happen, and
-    eventually the buffers fill up, and the nodes block when they try to flush
-    logs.
-
-    We can switch to Twisted APIs (treq instead of requests etc.), but
-    sometimes it's easier or expedient to just have a blocking test.  So this
-    decorator allows you to run the test in a thread, and the reactor can keep
-    running in the main thread.
-
-    See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
-    """
-    @wraps(f)
-    def test(*args, **kwargs):
-        return deferToThread(lambda: f(*args, **kwargs))
-    return test
-
@frozen
 class CHK:
    """
@ -827,16 +840,11 @@ async def reconfigure(reactor, request, node: TahoeProcess,
            )

    if changed:
-        # TODO reconfigure() seems to have issues on Windows. If you need to
-        # use it there, delete this assert and try to figure out what's going
-        # on...
-        assert not sys.platform.startswith("win")
-
        # restart the node
        print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
        await node.restart_async(reactor, request)
        print("Restarted.  Waiting for ready state.")
-        await_client_ready(node)
+        await await_client_ready(node)
        print("Ready.")
    else:
        print("Config unchanged, not restarting.")
--- a/misc/checkers/check_load.py
+++ b/misc/checkers/check_load.py
@ -1,5 +1,3 @@
-from __future__ import print_function
-
 """
 this is a load-generating client program. It does all of its work through a
 given tahoe node (specified by URL), and performs random reads and writes
@ -33,20 +31,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)


 """
+from __future__ import annotations

 import os, sys, httplib, binascii
 import urllib, json, random, time, urlparse

-try:
-    from typing import Dict
-except ImportError:
-    pass
-
-# Python 2 compatibility
-from future.utils import PY2
-if PY2:
-    from future.builtins import str  # noqa: F401
-
 if sys.argv[1] == "--stats":
    statsfiles = sys.argv[2:]
    # gather stats every 10 seconds, do a moving-window average of the last
@ -54,9 +43,9 @@ if sys.argv[1] == "--stats":
    DELAY = 10
    MAXSAMPLES = 6
    totals = []
-    last_stats = {}  # type: Dict[str, float]
+    last_stats : dict[str, float] = {}
    while True:
-        stats = {}  # type: Dict[str, float]
+        stats : dict[str, float] = {}
        for sf in statsfiles:
            for line in open(sf, "r").readlines():
                name, str_value = line.split(":")
--- a/misc/coding_tools/find-trailing-spaces.py
+++ b/misc/coding_tools/find-trailing-spaces.py
@ -1,44 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-
-import os, sys
-
-from twisted.python import usage
-
-class Options(usage.Options):
-    optFlags = [
-        ("recursive", "r", "Search for .py files recursively"),
-        ]
-    def parseArgs(self, *starting_points):
-        self.starting_points = starting_points
-
-found = [False]
-
-def check(fn):
-    f = open(fn, "r")
-    for i,line in enumerate(f.readlines()):
-        if line == "\n":
-            continue
-        if line[-1] == "\n":
-            line = line[:-1]
-        if line.rstrip() != line:
-            # the %s:%d:%d: lets emacs' compile-mode jump to those locations
-            print("%s:%d:%d: trailing whitespace" % (fn, i+1, len(line)+1))
-            found[0] = True
-    f.close()
-
-o = Options()
-o.parseOptions()
-if o['recursive']:
-    for starting_point in o.starting_points:
-        for root, dirs, files in os.walk(starting_point):
-            for fn in [f for f in files if f.endswith(".py")]:
-                fn = os.path.join(root, fn)
-                check(fn)
-else:
-    for fn in o.starting_points:
-        check(fn)
-if found[0]:
-    sys.exit(1)
-sys.exit(0)
--- a/mypy.ini
+++ b/mypy.ini
@ -7,4 +7,18 @@ show_error_codes = True
 warn_unused_configs =True
 no_implicit_optional = True
 warn_redundant_casts = True
-strict_equality = True
+strict_equality = True
+
+[mypy-allmydata.test.cli.wormholetesting,allmydata.listeners,allmydata.test.test_connection_status]
+disallow_any_generics = True
+disallow_subclassing_any = True
+disallow_untyped_calls = True
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+check_untyped_defs = True
+disallow_untyped_decorators = True
+warn_unused_ignores = True
+warn_return_any = True
+no_implicit_reexport = True
+strict_equality = True
+strict_concatenate = True
--- a/newsfragments/3622.minor
+++ b/newsfragments/3622.minor
--- a/newsfragments/3880.minor
+++ b/newsfragments/3880.minor
--- a/newsfragments/3910.minor
+++ b/newsfragments/3910.minor
--- a/newsfragments/3935.minor
+++ b/newsfragments/3935.minor
--- a/newsfragments/3970.minor
+++ b/newsfragments/3970.minor
--- a/newsfragments/3978.minor
+++ b/newsfragments/3978.minor
--- a/newsfragments/3987.minor
+++ b/newsfragments/3987.minor
--- a/newsfragments/3988.minor
+++ b/newsfragments/3988.minor
--- a/newsfragments/3989.installation
+++ b/newsfragments/3989.installation
@ -0,0 +1 @@
+tenacity is no longer a dependency.
--- a/newsfragments/3991.minor
+++ b/newsfragments/3991.minor
--- a/newsfragments/3993.minor
+++ b/newsfragments/3993.minor
--- a/newsfragments/3994.minor
+++ b/newsfragments/3994.minor
--- a/newsfragments/3996.minor
+++ b/newsfragments/3996.minor
--- a/newsfragments/3997.installation
+++ b/newsfragments/3997.installation
@ -0,0 +1 @@
+Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version.
--- a/newsfragments/3998.minor
+++ b/newsfragments/3998.minor
--- a/newsfragments/3999.bugfix
+++ b/newsfragments/3999.bugfix
@ -0,0 +1 @@
+A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed.
--- a/newsfragments/4000.minor
+++ b/newsfragments/4000.minor
--- a/newsfragments/4001.minor
+++ b/newsfragments/4001.minor
--- a/newsfragments/4002.minor
+++ b/newsfragments/4002.minor
--- a/newsfragments/4003.minor
+++ b/newsfragments/4003.minor
--- a/newsfragments/4004.minor
+++ b/newsfragments/4004.minor
--- a/newsfragments/4005.minor
+++ b/newsfragments/4005.minor
--- a/newsfragments/4006.minor
+++ b/newsfragments/4006.minor
--- a/newsfragments/4009.minor
+++ b/newsfragments/4009.minor
--- a/newsfragments/4010.minor
+++ b/newsfragments/4010.minor
--- a/newsfragments/4012.minor
+++ b/newsfragments/4012.minor
--- a/newsfragments/4014.minor
+++ b/newsfragments/4014.minor
--- a/newsfragments/4015.minor
+++ b/newsfragments/4015.minor
--- a/newsfragments/4016.minor
+++ b/newsfragments/4016.minor
--- a/newsfragments/4018.minor
+++ b/newsfragments/4018.minor
--- a/newsfragments/4019.minor
+++ b/newsfragments/4019.minor
--- a/newsfragments/4020.minor
+++ b/newsfragments/4020.minor
@ -0,0 +1 @@
+
--- a/newsfragments/4022.minor
+++ b/newsfragments/4022.minor
--- a/newsfragments/4023.minor
+++ b/newsfragments/4023.minor
--- a/newsfragments/4024.minor
+++ b/newsfragments/4024.minor
--- a/newsfragments/4026.minor
+++ b/newsfragments/4026.minor
--- a/newsfragments/4027.minor
+++ b/newsfragments/4027.minor
--- a/newsfragments/4028.minor
+++ b/newsfragments/4028.minor
--- a/newsfragments/4029.bugfix
+++ b/newsfragments/4029.bugfix
@ -0,0 +1,2 @@
+The (still off-by-default) HTTP storage client will now use Tor when Tor-based client-side anonymity was requested.
+Previously it would use normal TCP connections and not be anonymous.
--- a/newsfragments/4035.minor
+++ b/newsfragments/4035.minor
--- a/newsfragments/4036.feature
+++ b/newsfragments/4036.feature
@ -0,0 +1 @@
+tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes"
--- a/newsfragments/4038.minor
+++ b/newsfragments/4038.minor
--- a/newsfragments/4040.minor
+++ b/newsfragments/4040.minor
--- a/newsfragments/4044.minor
+++ b/newsfragments/4044.minor
--- a/newsfragments/4046.minor
+++ b/newsfragments/4046.minor
--- a/newsfragments/4049.minor
+++ b/newsfragments/4049.minor
--- a/nix/collections-extended.nix
+++ b/nix/collections-extended.nix
@ -0,0 +1,12 @@
+# Package a version that's compatible with Python 3.11.  This can go away once
+# https://github.com/mlenzen/collections-extended/pull/199 is merged and
+# included in a version of nixpkgs we depend on.
+{ fetchFromGitHub, collections-extended }:
+collections-extended.overrideAttrs (old: {
+  src = fetchFromGitHub {
+    owner = "mlenzen";
+    repo = "collections-extended";
+    rev = "8b93390636d58d28012b8e9d22334ee64ca37d73";
+    hash = "sha256-e7RCpNsqyS1d3q0E+uaE4UOEQziueYsRkKEvy3gCHt0=";
+  };
+})
--- a/nix/klein.nix
+++ b/nix/klein.nix
@ -0,0 +1,9 @@
+{ klein, fetchPypi }:
+klein.overrideAttrs (old: rec {
+  pname = "klein";
+  version = "23.5.0";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc=";
+  };
+})
--- a/nix/pycddl.nix
+++ b/nix/pycddl.nix
@ -0,0 +1,57 @@
+# package https://gitlab.com/tahoe-lafs/pycddl
+#
+# also in the process of being pushed upstream
+# https://github.com/NixOS/nixpkgs/pull/221220
+#
+# we should switch to the upstream package when it is available from our
+# minimum version of nixpkgs.
+#
+# if you need to update this package to a new pycddl release then
+#
+# 1. change value given to `buildPythonPackage` for `version` to match the new
+#    release
+#
+# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
+#
+# 3. run `nix-build`
+#
+# 4. there will be an error about a hash mismatch.  change the value given to
+#    `fetchPypi` for `sha256` to the "actual" hash value report.
+#
+# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`.
+#
+# 6. run `nix-build`
+#
+# 7. there will be an error about a hash mismatch.  change the value given to
+#    `cargoDeps` for `hash` to the "actual" hash value report.
+#
+# 8. run `nix-build`.  it should succeed.  if it does not, seek assistance.
+#
+{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }:
+buildPythonPackage rec {
+  pname = "pycddl";
+  version = "0.4.0";
+  format = "pyproject";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
+  };
+
+  # Without this, when building for PyPy, `maturin build` seems to fail to
+  # find the interpreter at all and then fails early in the build process with
+  # an error saying "unsupported Python interpreter".  We can easily point
+  # directly at the relevant interpreter, so do that.
+  maturinBuildFlags = [ "--interpreter" python.executable ];
+
+  nativeBuildInputs = with rustPlatform; [
+    maturinBuildHook
+    cargoSetupHook
+  ];
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    name = "${pname}-${version}";
+    hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs=";
+  };
+}
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@ -0,0 +1,10 @@
+{ pyopenssl, fetchPypi, isPyPy }:
+pyopenssl.overrideAttrs (old: rec {
+  pname = "pyOpenSSL";
+  version = "23.2.0";
+  name = "${pname}-${version}";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w=";
+  };
+})
--- a/nix/python-overrides.nix
+++ b/nix/python-overrides.nix
@ -0,0 +1,148 @@
+# Override various Python packages to create a package set that works for
+# Tahoe-LAFS on CPython and PyPy.
+self: super:
+let
+
+  # Run a function on a derivation if and only if we're building for PyPy.
+  onPyPy = f: drv: if super.isPyPy then f drv else drv;
+
+  # Disable a Python package's test suite.
+  dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; });
+
+  # Disable building a Python package's documentation.
+  dontBuildDocs = alsoDisable: drv: (drv.override ({
+    sphinxHook = null;
+  } // alsoDisable)).overrideAttrs ({ outputs, ... }: {
+    outputs = builtins.filter (x: "doc" != x) outputs;
+  });
+
+in {
+  # Some dependencies aren't packaged in nixpkgs so supply our own packages.
+  pycddl = self.callPackage ./pycddl.nix { };
+  txi2p = self.callPackage ./txi2p.nix { };
+
+  # Some packages are of somewhat too-old versions - update them.
+  klein = self.callPackage ./klein.nix {
+    # Avoid infinite recursion.
+    inherit (super) klein;
+  };
+  txtorcon = self.callPackage ./txtorcon.nix {
+    inherit (super) txtorcon;
+  };
+
+  # Update the version of pyopenssl.
+  pyopenssl = self.callPackage ./pyopenssl.nix {
+    pyopenssl =
+      # Building the docs requires sphinx which brings in a dependency on babel,
+      # the test suite of which fails.
+      onPyPy (dontBuildDocs { sphinx-rtd-theme = null; })
+        # Avoid infinite recursion.
+        super.pyopenssl;
+  };
+
+  # collections-extended is currently broken for Python 3.11 in nixpkgs but
+  # we know where a working version lives.
+  collections-extended = self.callPackage ./collections-extended.nix {
+    inherit (super) collections-extended;
+  };
+
+  # greenlet is incompatible with PyPy but PyPy has a builtin equivalent.
+  # Fixed in nixpkgs in a5f8184fb816a4fd5ae87136838c9981e0d22c67.
+  greenlet = onPyPy (drv: null) super.greenlet;
+
+  # tornado and tk pull in a huge dependency trees for functionality we don't
+  # care about, also tkinter doesn't work on PyPy.
+  matplotlib = super.matplotlib.override { tornado = null; enableTk = false; };
+
+  tqdm = super.tqdm.override {
+    # ibid.
+    tkinter = null;
+    # pandas is only required by the part of the test suite covering
+    # integration with pandas that we don't care about.  pandas is a huge
+    # dependency.
+    pandas = null;
+  };
+
+  # The treq test suite depends on httpbin.  httpbin pulls in babel (flask ->
+  # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow).  babel fails
+  # its test suite and arrow segfaults.
+  treq = onPyPy dontCheck super.treq;
+
+  # the six test suite fails on PyPy because it depends on dbm which the
+  # nixpkgs PyPy build appears to be missing.  Maybe fixed in nixpkgs in
+  # a5f8184fb816a4fd5ae87136838c9981e0d22c67.
+  six = onPyPy dontCheck super.six;
+
+  # Likewise for beautifulsoup4.
+  beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4;
+
+  # The autobahn test suite pulls in a vast number of dependencies for
+  # functionality we don't care about.  It might be nice to *selectively*
+  # disable just some of it but this is easier.
+  autobahn = onPyPy dontCheck super.autobahn;
+
+  # and python-dotenv tests pulls in a lot of dependencies, including jedi,
+  # which does not work on PyPy.
+  python-dotenv = onPyPy dontCheck super.python-dotenv;
+
+  # Upstream package unaccountably includes a sqlalchemy dependency ... but
+  # the project has no such dependency.  Fixed in nixpkgs in
+  # da10e809fff70fbe1d86303b133b779f09f56503.
+  aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; };
+
+  # By default, the sphinx docs are built, which pulls in a lot of
+  # dependencies - including jedi, which does not work on PyPy.
+  hypothesis =
+    (let h = super.hypothesis;
+     in
+       if (h.override.__functionArgs.enableDocumentation or false)
+       then h.override { enableDocumentation = false; }
+       else h).overrideAttrs ({ nativeBuildInputs, ... }: {
+         # The nixpkgs expression is missing the tzdata check input.
+         nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ];
+       });
+
+  # flaky's test suite depends on nose and nose appears to have Python 3
+  # incompatibilities (it includes `print` statements, for example).
+  flaky = onPyPy dontCheck super.flaky;
+
+  # Replace the deprecated way of running the test suite with the modern way.
+  # This also drops a bunch of unnecessary build-time dependencies, some of
+  # which are broken on PyPy.  Fixed in nixpkgs in
+  # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea.
+  zfec = (super.zfec.override {
+    setuptoolsTrial = null;
+  }).overrideAttrs (old: {
+    checkPhase = "trial zfec";
+  });
+
+  # collections-extended is packaged with poetry-core.  poetry-core test suite
+  # uses virtualenv and virtualenv test suite fails on PyPy.
+  poetry-core = onPyPy dontCheck super.poetry-core;
+
+  # The test suite fails with some rather irrelevant (to us) string comparison
+  # failure on PyPy.  Probably a PyPy bug but doesn't seem like we should
+  # care.
+  rich = onPyPy dontCheck super.rich;
+
+  # The pyutil test suite fails in some ... test ... for some deprecation
+  # functionality we don't care about.
+  pyutil = onPyPy dontCheck super.pyutil;
+
+  # testCall1 fails fairly inscrutibly on PyPy.  Perhaps someone can fix that,
+  # or we could at least just skip that one test.  Probably better to fix it
+  # since we actually depend directly and significantly on Foolscap.
+  foolscap = onPyPy dontCheck super.foolscap;
+
+  # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246
+  psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: {
+    # Upstream already disables some tests but there are even more that have
+    # build impurities that come from build system hardware configuration.
+    # Skip them too.
+    pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray;
+    disabledTests = disabledTests ++ [ "sensors_temperatures" ];
+  });
+
+  # CircleCI build systems don't have enough memory to run this test suite.
+  lz4 = dontCheck super.lz4;
+}
--- a/nix/sources.json
+++ b/nix/sources.json
@ -1,16 +1,4 @@
 {
-    "mach-nix": {
-        "branch": "switch-to-nix-pypi-fetcher-2",
-        "description": "Create highly reproducible python environments",
-        "homepage": "",
-        "owner": "PrivateStorageio",
-        "repo": "mach-nix",
-        "rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4",
-        "sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls",
-        "type": "tarball",
-        "url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
    "niv": {
        "branch": "master",
        "description": "Easy dependency management for Nix projects",
@ -23,40 +11,28 @@
        "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    },
-    "nixpkgs-21.05": {
-        "branch": "nixos-21.05",
+    "nixpkgs-22.11": {
+        "branch": "nixos-22.11",
        "description": "Nix Packages collection",
        "homepage": "",
-        "owner": "NixOS",
+        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
-        "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
+        "rev": "970402e6147c49603f4d06defe44d27fe51884ce",
+        "sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh",
        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
+        "url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    },
-    "nixpkgs-21.11": {
-        "branch": "nixos-21.11",
-        "description": "Nix Packages collection",
-        "homepage": "",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "838eefb4f93f2306d4614aafb9b2375f315d917f",
-        "sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
-    "pypi-deps-db": {
+    "nixpkgs-unstable": {
        "branch": "master",
-        "description": "Probably the most complete python dependency database",
+        "description": "Nix Packages collection",
        "homepage": "",
-        "owner": "DavHau",
-        "repo": "pypi-deps-db",
-        "rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6",
-        "sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "d0c9a536331227ab883b4f6964be638fa436d81f",
+        "sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1",
        "type": "tarball",
-        "url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz",
+        "url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    }
 }
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@ -0,0 +1,80 @@
+{ lib
+, pythonPackages
+, buildPythonPackage
+, tahoe-lafs-src
+, extrasNames
+
+# control how the test suite is run
+, doCheck
+}:
+let
+  pname = "tahoe-lafs";
+  version = "1.18.0.post1";
+
+  pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum  ++ deps.${extra}) [] extras;
+
+  pythonExtraDependencies = with pythonPackages; {
+    tor = [ txtorcon ];
+    i2p = [ txi2p ];
+  };
+
+  pythonPackageDependencies = with pythonPackages; [
+    attrs
+    autobahn
+    cbor2
+    click
+    collections-extended
+    cryptography
+    distro
+    eliot
+    filelock
+    foolscap
+    future
+    klein
+    magic-wormhole
+    netifaces
+    psutil
+    pyyaml
+    pycddl
+    pyrsistent
+    pyutil
+    six
+    treq
+    twisted
+    # Get the dependencies for the Twisted extras we depend on, too.
+    twisted.passthru.optional-dependencies.tls
+    twisted.passthru.optional-dependencies.conch
+    werkzeug
+    zfec
+    zope_interface
+  ] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
+
+  unitTestDependencies = with pythonPackages; [
+    beautifulsoup4
+    fixtures
+    hypothesis
+    mock
+    prometheus-client
+    testtools
+  ];
+
+in
+buildPythonPackage {
+  inherit pname version;
+  src = tahoe-lafs-src;
+  propagatedBuildInputs = pythonPackageDependencies;
+
+  inherit doCheck;
+  checkInputs = unitTestDependencies;
+  checkPhase = ''
+    export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
+    python -m twisted.trial -j $NIX_BUILD_CORES allmydata
+  '';
+
+  meta = with lib; {
+    homepage = "https://tahoe-lafs.org/";
+    description = "secure, decentralized, fault-tolerant file store";
+    # Also TGPPL
+    license = licenses.gpl2Plus;
+  };
+}
--- a/nix/tests.nix
+++ b/nix/tests.nix
@ -0,0 +1,4 @@
+# Build the package with the test suite enabled.
+args@{...}: (import ../. args).override {
+  doCheck = true;
+}
--- a/nix/txi2p.nix
+++ b/nix/txi2p.nix
@ -0,0 +1,39 @@
+# package https://github.com/tahoe-lafs/txi2p
+#
+# if you need to update this package to a new txi2p release then
+#
+# 1. change value given to `buildPythonPackage` for `version` to match the new
+#    release
+#
+# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
+#
+# 3. run `nix-build`
+#
+# 4. there will be an error about a hash mismatch.  change the value given to
+#    `fetchPypi` for `sha256` to the "actual" hash value report.
+#
+# 5. if there are new runtime dependencies then add them to the argument list
+#    at the top.  if there are new test dependencies add them to the
+#    `checkInputs` list.
+#
+# 6. run `nix-build`.  it should succeed.  if it does not, seek assistance.
+#
+{ fetchPypi
+, buildPythonPackage
+, parsley
+, twisted
+, unittestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "txi2p-tahoe";
+  version = "0.3.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY=";
+  };
+
+  propagatedBuildInputs = [ twisted parsley ];
+  checkInputs = [ unittestCheckHook ];
+  pythonImportsCheck = [ "parsley" "ometa"];
+}
--- a/nix/txtorcon.nix
+++ b/nix/txtorcon.nix
@ -0,0 +1,9 @@
+{ txtorcon, fetchPypi }:
+txtorcon.overrideAttrs (old: rec {
+  pname = "txtorcon";
+  version = "23.5.0";
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw=";
+  };
+})
--- a/setup.cfg
+++ b/setup.cfg
@ -6,6 +6,9 @@ develop = update_version develop
 bdist_egg = update_version bdist_egg
 bdist_wheel = update_version bdist_wheel

+# This has been replaced by ruff (see .ruff.toml), which has same checks as
+# flake8 plus many more, and is also faster. However, we're keeping this config
+# in case people still use flake8 in IDEs, etc..
 [flake8]
 # Enforce all pyflakes constraints, and also prohibit tabs for indentation.
 # Reference:
--- a/setup.py
+++ b/setup.py
@ -65,6 +65,9 @@ install_requires = [
    #   version of cryptography will *really* be installed.
    "cryptography >= 2.6",

+    # * Used for custom HTTPS validation
+    "pyOpenSSL >= 23.2.0",
+
    # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
    #   rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
    # * The SFTP frontend and manhole depend on the conch extra. However, we
@ -118,7 +121,7 @@ install_requires = [
    "attrs >= 18.2.0",

    # WebSocket library for twisted and asyncio
-    "autobahn < 22.4.1",  # remove this when 22.4.3 is released
+    "autobahn",

    # Support for Python 3 transition
    "future >= 0.18.2",
@ -136,7 +139,8 @@ install_requires = [
    "collections-extended >= 2.0.2",

    # HTTP server and client
-    "klein",
+    # Latest version is necessary to work with latest werkzeug:
+    "klein >= 23.5.0",
    # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
    "werkzeug != 2.2.0",
    "treq",
@ -159,10 +163,9 @@ setup_requires = [
 ]

 tor_requires = [
-    # This is exactly what `foolscap[tor]` means but pip resolves the pair of
-    # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose
-    # this if we don't declare it ourselves!
-    "txtorcon >= 0.17.0",
+    # 23.5 added support for custom TLS contexts in web_agent(), which is
+    # needed for the HTTP storage client to run over Tor.
+    "txtorcon >= 23.5.0",
 ]

 i2p_requires = [
@ -394,16 +397,31 @@ setup(name="tahoe-lafs", # also set in __init__.py
              "dulwich",
              "gpg",
          ],
+
+          # Here are the dependencies required to set up a reproducible test
+          # environment.  This could be for CI or local development.  These
+          # are *not* library dependencies of the test suite itself.  They are
+          # the tools we use to run the test suite at all.
+          "testenv": [
+              # Pin all of these versions for the same reason you ever want to
+              # pin anything: to prevent new releases with regressions from
+              # introducing spurious failures into CI runs for whatever
+              # development work is happening at the time.  The versions
+              # selected here are just the current versions at the time.
+              # Bumping them to keep up with future releases is fine as long
+              # as those releases are known to actually work.
+              "pip==22.0.3",
+              "wheel==0.37.1",
+              "setuptools==60.9.1",
+              "subunitreporter==22.2.0",
+              "python-subunit==1.4.2",
+              "junitxml==0.7",
+              "coverage==7.2.5",
+          ],
+
+          # Here are the library dependencies of the test suite.
          "test": [
-              "flake8",
-              # Pin a specific pyflakes so we don't have different folks
-              # disagreeing on what is or is not a lint issue.  We can bump
-              # this version from time to time, but we will do it
-              # intentionally.
-              "pyflakes == 2.2.0",
-              "coverage ~= 5.0",
              "mock",
-              "tox ~= 3.0",
              "pytest",
              "pytest-twisted",
              "hypothesis >= 3.6.1",
@ -412,8 +430,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
              "fixtures",
              "beautifulsoup4",
              "html5lib",
-              "junitxml",
-              "tenacity",
              # Pin old version until
              # https://github.com/paramiko/paramiko/issues/1961 is fixed.
              "paramiko < 2.9",
--- a/src/allmydata/cli/grid_manager.py
+++ b/src/allmydata/cli/grid_manager.py
@ -28,7 +28,7 @@ from allmydata.grid_manager import (
 from allmydata.util import jsonbytes as json


-@click.group()
+@click.group() # type: ignore[arg-type]
@click.option(
    '--config', '-c',
    type=click.Path(),
@ -71,7 +71,7 @@ def grid_manager(ctx, config):
    ctx.obj = Config()


-@grid_manager.command()
+@grid_manager.command() # type: ignore[attr-defined]
@click.pass_context
 def create(ctx):
    """
@ -91,7 +91,7 @@ def create(ctx):
        )


-@grid_manager.command()
+@grid_manager.command() # type: ignore[attr-defined]
@click.pass_obj
 def public_identity(config):
    """
@ -103,7 +103,7 @@ def public_identity(config):
    click.echo(config.grid_manager.public_identity())


-@grid_manager.command()
+@grid_manager.command() # type: ignore[arg-type, attr-defined]
@click.argument("name")
@click.argument("public_key", type=click.STRING)
@click.pass_context
@ -132,7 +132,7 @@ def add(ctx, name, public_key):
    return 0


-@grid_manager.command()
+@grid_manager.command() # type: ignore[arg-type, attr-defined]
@click.argument("name")
@click.pass_context
 def remove(ctx, name):
@ -155,7 +155,8 @@ def remove(ctx, name):
    save_grid_manager(fp, ctx.obj.grid_manager, create=False)


-@grid_manager.command()  # noqa: F811
+@grid_manager.command() # type: ignore[attr-defined]
+                        # noqa: F811
@click.pass_context
 def list(ctx):
    """
@ -175,7 +176,7 @@ def list(ctx):
                click.echo("expired {} ({})".format(cert.expires, abbreviate_time(delta)))


-@grid_manager.command()
+@grid_manager.command() # type: ignore[arg-type, attr-defined]
@click.argument("name")
@click.argument(
    "expiry_days",
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@ -1,5 +1,5 @@
 """
-Ported to Python 3.
+Functionality related to operating a Tahoe-LAFS node (client _or_ server).
 """
 from __future__ import annotations

@ -7,10 +7,9 @@ import os
 import stat
 import time
 import weakref
-from typing import Optional
+from typing import Optional, Iterable
 from base64 import urlsafe_b64encode
 from functools import partial
-# On Python 2 this will be the backported package:
 from configparser import NoSectionError

 from foolscap.furl import (
@ -47,7 +46,7 @@ from allmydata.util.encodingutil import get_filesystem_encoding
 from allmydata.util.abbreviate import parse_abbreviated_size
 from allmydata.util.time_format import parse_duration, parse_date
 from allmydata.util.i2p_provider import create as create_i2p_provider
-from allmydata.util.tor_provider import create as create_tor_provider
+from allmydata.util.tor_provider import create as create_tor_provider, _Provider as TorProvider
 from allmydata.stats import StatsProvider
 from allmydata.history import History
 from allmydata.interfaces import (
@ -175,8 +174,6 @@ class KeyGenerator(object):
        """I return a Deferred that fires with a (verifyingkey, signingkey)
        pair. The returned key will be 2048 bit"""
        keysize = 2048
-        # RSA key generation for a 2048 bit key takes between 0.8 and 3.2
-        # secs
        signer, verifier = rsa.create_signing_keypair(keysize)
        return defer.succeed( (verifier, signer) )

@ -191,7 +188,7 @@ class Terminator(service.Service):
        return service.Service.stopService(self)


-def read_config(basedir, portnumfile, generated_files=[]):
+def read_config(basedir, portnumfile, generated_files: Iterable=()):
    """
    Read and validate configuration for a client-style Node. See
    :method:`allmydata.node.read_config` for parameter meanings (the
@ -270,7 +267,7 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory=
    introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory)
    storage_broker = create_storage_farm_broker(
        config, default_connection_handlers, foolscap_connection_handlers,
-        tub_options, introducer_clients
+        tub_options, introducer_clients, tor_provider
    )

    client = _client_factory(
@ -466,7 +463,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
    return introducer_clients


-def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients):
+def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients, tor_provider: Optional[TorProvider]):
    """
    Create a StorageFarmBroker object, for use by Uploader/Downloader
    (and everybody else who wants to use storage servers)
@ -502,6 +499,8 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo
        tub_maker=tub_creator,
        node_config=config,
        storage_client_config=storage_client_config,
+        default_connection_handlers=default_connection_handlers,
+        tor_provider=tor_provider,
    )
    for ic in introducer_clients:
        sb.use_introducer(ic)
@ -838,7 +837,11 @@ class _Client(node.Node, pollmixin.PollMixin):
            if hasattr(self.tub.negotiationClass, "add_storage_server"):
                nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii"))
                self.storage_nurls = nurls
-                announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls]
+                # There is code in e.g. storage_client.py that checks if an
+                # announcement has changed. Since NURL order isn't meaningful,
+                # we don't want a change in the order to count as a change, so we
+                # send the NURLs as a set. CBOR supports sets, as does Foolscap.
+                announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = {n.to_text() for n in nurls}
            announcement["anonymous-storage-FURL"] = furl

        enabled_storage_servers = self._enable_storage_servers(
@ -1030,14 +1033,14 @@ class _Client(node.Node, pollmixin.PollMixin):
    def init_web(self, webport):
        self.log("init_web(webport=%s)", args=(webport,))

-        from allmydata.webish import WebishServer
+        from allmydata.webish import WebishServer, anonymous_tempfile_factory
        nodeurl_path = self.config.get_config_path("node.url")
        staticdir_config = self.config.get_config("node", "web.static", "public_html")
        staticdir = self.config.get_config_path(staticdir_config)
        ws = WebishServer(
            self,
            webport,
-            self._get_tempdir(),
+            anonymous_tempfile_factory(self._get_tempdir()),
            nodeurl_path,
            staticdir,
        )
@ -1105,7 +1108,7 @@ class _Client(node.Node, pollmixin.PollMixin):
        # may get an opaque node if there were any problems.
        return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)

-    def create_dirnode(self, initial_children={}, version=None):
+    def create_dirnode(self, initial_children=None, version=None):
        d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
        return d

--- a/src/allmydata/dirnode.py
+++ b/src/allmydata/dirnode.py
@ -678,8 +678,10 @@ class DirectoryNode(object):
        return d

    # XXX: Too many arguments? Worthwhile to break into mutable/immutable?
-    def create_subdirectory(self, namex, initial_children={}, overwrite=True,
+    def create_subdirectory(self, namex, initial_children=None, overwrite=True,
                            mutable=True, mutable_version=None, metadata=None):
+        if initial_children is None:
+            initial_children = {}
        name = normalize(namex)
        if self.is_readonly():
            return defer.fail(NotWriteableError())
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@ -1925,7 +1925,11 @@ class FakeTransport(object):
    def loseConnection(self):
        logmsg("FakeTransport.loseConnection()", level=NOISY)

-    # getPeer and getHost can just raise errors, since we don't know what to return
+    def getHost(self):
+        raise NotImplementedError()
+
+    def getPeer(self):
+        raise NotImplementedError()


@implementer(ISession)
@ -1990,15 +1994,18 @@ class Dispatcher(object):
    def __init__(self, client):
        self._client = client

-    def requestAvatar(self, avatarID, mind, interface):
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        [interface] = interfaces
        _assert(interface == IConchUser, interface=interface)
-        rootnode = self._client.create_node_from_uri(avatarID.rootcap)
-        handler = SFTPUserHandler(self._client, rootnode, avatarID.username)
+        rootnode = self._client.create_node_from_uri(avatarId.rootcap)
+        handler = SFTPUserHandler(self._client, rootnode, avatarId.username)
        return (interface, handler, handler.logout)


 class SFTPServer(service.MultiService):
-    name = "frontend:sftp"
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "frontend:sftp"  # type: ignore[assignment]

    def __init__(self, client, accountfile,
                 sftp_portstr, pubkey_file, privkey_file):
--- a/src/allmydata/hashtree.py
+++ b/src/allmydata/hashtree.py
@ -332,7 +332,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
            name += " (leaf [%d] of %d)" % (leafnum, numleaves)
        return name

-    def set_hashes(self, hashes={}, leaves={}):
+    def set_hashes(self, hashes=None, leaves=None):
        """Add a bunch of hashes to the tree.

        I will validate these to the best of my ability. If I already have a
@ -382,7 +382,10 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
        corrupted or one of the received hashes was corrupted. If it raises
        NotEnoughHashesError, then the otherhashes dictionary was incomplete.
        """
-
+        if hashes is None:
+            hashes = {}
+        if leaves is None:
+            leaves = {}
        assert isinstance(hashes, dict)
        for h in hashes.values():
            assert isinstance(h, bytes)
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@ -2,22 +2,12 @@
 Ported to Python 3.
 """

-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations

-from future.utils import PY2, native_str
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from future.utils import native_str
 from past.builtins import long, unicode
 from six import ensure_str

-try:
-    from typing import List
-except ImportError:
-    pass
-
 import os, time, weakref, itertools

 import attr
@ -915,12 +905,12 @@ class _Accum(object):
    :ivar remaining: The number of bytes still expected.
    :ivar ciphertext: The bytes accumulated so far.
    """
-    remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int
-    ciphertext = attr.ib(default=attr.Factory(list))                # type: List[bytes]
+    remaining : int = attr.ib(validator=attr.validators.instance_of(int))
+    ciphertext : list[bytes] = attr.ib(default=attr.Factory(list))

    def extend(self,
               size,           # type: int
-               ciphertext,     # type: List[bytes]
+               ciphertext,     # type: list[bytes]
    ):
        """
        Accumulate some more ciphertext.
@ -1401,7 +1391,9 @@ class CHKUploader(object):
    def get_upload_status(self):
        return self._upload_status

-def read_this_many_bytes(uploadable, size, prepend_data=[]):
+def read_this_many_bytes(uploadable, size, prepend_data=None):
+    if prepend_data is None:
+        prepend_data = []
    if size == 0:
        return defer.succeed([])
    d = uploadable.read(size)
@ -1851,7 +1843,9 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
    """I am a service that allows file uploading. I am a service-child of the
    Client.
    """
-    name = "uploader"
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "uploader"  # type: ignore[assignment]
    URI_LIT_SIZE_THRESHOLD = 55

    def __init__(self, helper_furl=None, stats_provider=None, history=None):
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@ -17,11 +17,13 @@ if PY2:
    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min  # noqa: F401

 from past.builtins import long
+from typing import Dict

 from zope.interface import Interface, Attribute
 from twisted.plugin import (
    IPlugin,
 )
+from twisted.internet.defer import Deferred
 from foolscap.api import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
     ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable

@ -307,12 +309,15 @@ class RIStorageServer(RemoteInterface):
        store that on disk.
        """

+# The result of IStorageServer.get_version():
+VersionMessage = Dict[bytes, object]
+

 class IStorageServer(Interface):
    """
    An object capable of storing shares for a storage client.
    """
-    def get_version():
+    def get_version() -> Deferred[VersionMessage]:
        """
        :see: ``RIStorageServer.get_version``
        """
@ -493,47 +498,6 @@ class IStorageBroker(Interface):
        @return: unicode nickname, or None
        """

-    # methods moved from IntroducerClient, need review
-    def get_all_connections():
-        """Return a frozenset of (nodeid, service_name, rref) tuples, one for
-        each active connection we've established to a remote service. This is
-        mostly useful for unit tests that need to wait until a certain number
-        of connections have been made."""
-
-    def get_all_connectors():
-        """Return a dict that maps from (nodeid, service_name) to a
-        RemoteServiceConnector instance for all services that we are actively
-        trying to connect to. Each RemoteServiceConnector has the following
-        public attributes::
-
-          service_name: the type of service provided, like 'storage'
-          last_connect_time: when we last established a connection
-          last_loss_time: when we last lost a connection
-
-          version: the peer's version, from the most recent connection
-          oldest_supported: the peer's oldest supported version, same
-
-          rref: the RemoteReference, if connected, otherwise None
-
-        This method is intended for monitoring interfaces, such as a web page
-        that describes connecting and connected peers.
-        """
-
-    def get_all_peerids():
-        """Return a frozenset of all peerids to whom we have a connection (to
-        one or more services) established. Mostly useful for unit tests."""
-
-    def get_all_connections_for(service_name):
-        """Return a frozenset of (nodeid, service_name, rref) tuples, one
-        for each active connection that provides the given SERVICE_NAME."""
-
-    def get_permuted_peers(service_name, key):
-        """Returns an ordered list of (peerid, rref) tuples, selecting from
-        the connections that provide SERVICE_NAME, using a hash-based
-        permutation keyed by KEY. This randomizes the service list in a
-        repeatable way, to distribute load over many peers.
-        """
-

 class IDisplayableServer(Interface):
    def get_nickname():
@ -551,16 +515,6 @@ class IServer(IDisplayableServer):
    def start_connecting(trigger_cb):
        pass

-    def get_rref():
-        """Obsolete.  Use ``get_storage_server`` instead.
-
-        Once a server is connected, I return a RemoteReference.
-        Before a server is connected for the first time, I return None.
-
-        Note that the rref I return will start producing DeadReferenceErrors
-        once the connection is lost.
-        """
-
    def upload_permitted():
        """
        :return: True if we should use this server for uploads, False
@ -1447,7 +1401,7 @@ class IDirectoryNode(IFilesystemNode):
        is a file, or if must_be_file is True and the child is a directory,
        I raise ChildOfWrongTypeError."""

-    def create_subdirectory(name, initial_children={}, overwrite=True,
+    def create_subdirectory(name, initial_children=None, overwrite=True,
                            mutable=True, mutable_version=None, metadata=None):
        """I create and attach a directory at the given name. The new
        directory can be empty, or it can be populated with children
@ -2586,7 +2540,7 @@ class IClient(Interface):
        @return: a Deferred that fires with an IMutableFileNode instance.
        """

-    def create_dirnode(initial_children={}):
+    def create_dirnode(initial_children=None):
        """Create a new unattached dirnode, possibly with initial children.

        @param initial_children: dict with keys that are unicode child names,
@ -2641,7 +2595,7 @@ class INodeMaker(Interface):
        for use by unit tests, to create mutable files that are smaller than
        usual."""

-    def create_new_mutable_directory(initial_children={}):
+    def create_new_mutable_directory(initial_children=None):
        """I create a new mutable directory, and return a Deferred that will
        fire with the IDirectoryNode instance when it is ready. If
        initial_children= is provided (a dict mapping unicode child name to
--- a/src/allmydata/introducer/client.py
+++ b/src/allmydata/introducer/client.py
@ -35,7 +35,7 @@ class InvalidCacheError(Exception):

 V2 = b"http://allmydata.org/tahoe/protocols/introducer/v2"

-@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient)
+@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient)  # type: ignore[misc]
 class IntroducerClient(service.Service, Referenceable):

    def __init__(self, tub, introducer_furl,
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@ -2,24 +2,13 @@
 Ported to Python 3.
 """

-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations

-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long
 from six import ensure_text

 import time, os.path, textwrap
-
-try:
-    from typing import Any, Dict, Union
-except ImportError:
-    pass
+from typing import Any, Union

 from zope.interface import implementer
 from twisted.application import service
@ -79,10 +68,6 @@ def create_introducer(basedir=u"."):
        default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider)
        tub_options = create_tub_options(config)

-        # we don't remember these because the Introducer doesn't make
-        # outbound connections.
-        i2p_provider = None
-        tor_provider = None
        main_tub = create_main_tub(
            config, tub_options, default_connection_handlers,
            foolscap_connection_handlers, i2p_provider, tor_provider,
@ -94,6 +79,8 @@ def create_introducer(basedir=u"."):
            i2p_provider,
            tor_provider,
        )
+        i2p_provider.setServiceParent(node)
+        tor_provider.setServiceParent(node)
        return defer.succeed(node)
    except Exception:
        return Failure()
@ -155,17 +142,20 @@ def stringify_remote_address(rref):
    return str(remote)


+# MyPy doesn't work well with remote interfaces...
@implementer(RIIntroducerPublisherAndSubscriberService_v2)
-class IntroducerService(service.MultiService, Referenceable):
-    name = "introducer"
+class IntroducerService(service.MultiService, Referenceable):  # type: ignore[misc]
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "introducer"  # type: ignore[assignment]
    # v1 is the original protocol, added in 1.0 (but only advertised starting
    # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
    # TODO: reconcile bytes/str for keys
-    VERSION = {
+    VERSION : dict[Union[bytes, str], Any]= {
                #"http://allmydata.org/tahoe/protocols/introducer/v1": { },
                b"http://allmydata.org/tahoe/protocols/introducer/v2": { },
                b"application-version": allmydata.__full_version__.encode("utf-8"),
-                }  # type: Dict[Union[bytes, str], Any]
+                }

    def __init__(self):
        service.MultiService.__init__(self)
--- a/src/allmydata/listeners.py
+++ b/src/allmydata/listeners.py
@ -0,0 +1,121 @@
+"""
+Define a protocol for listening on a transport such that Tahoe-LAFS can
+communicate over it, manage configuration for it in its configuration file,
+detect when it is possible to use it, etc.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, Sequence, Mapping, Optional, Union, Awaitable
+from typing_extensions import Literal
+
+from attrs import frozen
+from twisted.python.usage import Options
+
+from .interfaces import IAddressFamily
+from .util.iputil import allocate_tcp_port
+from .node import _Config
+
+@frozen
+class ListenerConfig:
+    """
+    :ivar tub_ports: Entries to merge into ``[node]tub.port``.
+
+    :ivar tub_locations: Entries to merge into ``[node]tub.location``.
+
+    :ivar node_config: Entries to add into the overall Tahoe-LAFS
+        configuration beneath a section named after this listener.
+    """
+    tub_ports: Sequence[str]
+    tub_locations: Sequence[str]
+    node_config: Mapping[str, Sequence[tuple[str, str]]]
+
+class Listener(Protocol):
+    """
+    An object which can listen on a transport and allow Tahoe-LAFS
+    communication to happen over it.
+    """
+    def is_available(self) -> bool:
+        """
+        Can this type of listener actually be used in this runtime
+        environment?
+        """
+
+    def can_hide_ip(self) -> bool:
+        """
+        Can the transport supported by this type of listener conceal the
+        node's public internet address from peers?
+        """
+
+    async def create_config(self, reactor: Any, cli_config: Options) -> Optional[ListenerConfig]:
+        """
+        Set up an instance of this listener according to the given
+        configuration parameters.
+
+        This may also allocate ephemeral resources if necessary.
+
+        :return: The created configuration which can be merged into the
+            overall *tahoe.cfg* configuration file.
+        """
+
+    def create(self, reactor: Any, config: _Config) -> IAddressFamily:
+        """
+        Instantiate this listener according to the given
+        previously-generated configuration.
+
+        :return: A handle on the listener which can be used to integrate it
+            into the Tahoe-LAFS node.
+        """
+
+class TCPProvider:
+    """
+    Support plain TCP connections.
+    """
+    def is_available(self) -> Literal[True]:
+        return True
+
+    def can_hide_ip(self) -> Literal[False]:
+        return False
+
+    async def create_config(self, reactor: Any, cli_config: Options) -> ListenerConfig:
+        tub_ports = []
+        tub_locations = []
+        if cli_config["port"]: # --port/--location are a pair
+            tub_ports.append(cli_config["port"])
+            tub_locations.append(cli_config["location"])
+        else:
+            assert "hostname" in cli_config
+            hostname = cli_config["hostname"]
+            new_port = allocate_tcp_port()
+            tub_ports.append(f"tcp:{new_port}")
+            tub_locations.append(f"tcp:{hostname}:{new_port}")
+
+        return ListenerConfig(tub_ports, tub_locations, {})
+
+    def create(self, reactor: Any, config: _Config) -> IAddressFamily:
+        raise NotImplementedError()
+
+
+@frozen
+class StaticProvider:
+    """
+    A provider that uses all pre-computed values.
+    """
+    _available: bool
+    _hide_ip: bool
+    _config: Union[Awaitable[Optional[ListenerConfig]], Optional[ListenerConfig]]
+    _address: IAddressFamily
+
+    def is_available(self) -> bool:
+        return self._available
+
+    def can_hide_ip(self) -> bool:
+        return self._hide_ip
+
+    async def create_config(self, reactor: Any, cli_config: Options) -> Optional[ListenerConfig]:
+        if self._config is None or isinstance(self._config, ListenerConfig):
+            return self._config
+        return await self._config
+
+    def create(self, reactor: Any, config: _Config) -> IAddressFamily:
+        return self._address
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@ -4,14 +4,8 @@ a node for Tahoe-LAFS.

 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations

-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from six import ensure_str, ensure_text

 import json
@ -23,11 +17,7 @@ import errno
 from base64 import b32decode, b32encode
 from errno import ENOENT, EPERM
 from warnings import warn
-
-try:
-    from typing import Union
-except ImportError:
-    pass
+from typing import Union, Iterable

 import attr

@ -182,7 +172,7 @@ def create_node_dir(basedir, readme_text):
            f.write(readme_text)


-def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
+def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_config=None):
    """
    Read and validate configuration.

@ -281,8 +271,7 @@ def _error_about_old_config_files(basedir, generated_files):
        raise e


-def ensure_text_and_abspath_expanduser_unicode(basedir):
-    # type: (Union[bytes, str]) -> str
+def ensure_text_and_abspath_expanduser_unicode(basedir: Union[bytes, str]) -> str:
    return abspath_expanduser_unicode(ensure_text(basedir))


@ -752,7 +741,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider):


 def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers,
-               handler_overrides={}, force_foolscap=False, **kwargs):
+               handler_overrides=None, force_foolscap=False, **kwargs):
    """
    Create a Tub with the right options and handlers. It will be
    ephemeral unless the caller provides certFile= in kwargs
@ -766,6 +755,8 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han
    :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS
        storage protocol.
    """
+    if handler_overrides is None:
+        handler_overrides = {}
    # We listen simultaneously for both Foolscap and HTTPS on the same port,
    # so we have to create a special Foolscap Tub for that to work:
    if force_foolscap:
@ -933,7 +924,7 @@ def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location):
 def create_main_tub(config, tub_options,
                    default_connection_handlers, foolscap_connection_handlers,
                    i2p_provider, tor_provider,
-                    handler_overrides={}, cert_filename="node.pem"):
+                    handler_overrides=None, cert_filename="node.pem"):
    """
    Creates a 'main' Foolscap Tub, typically for use as the top-level
    access point for a running Node.
@ -954,6 +945,8 @@ def create_main_tub(config, tub_options,
    :param tor_provider: None, or a _Provider instance if txtorcon +
        Tor are installed.
    """
+    if handler_overrides is None:
+        handler_overrides = {}
    portlocation = _tub_portlocation(
        config,
        iputil.get_local_addresses_sync,
--- a/src/allmydata/nodemaker.py
+++ b/src/allmydata/nodemaker.py
@ -135,8 +135,9 @@ class NodeMaker(object):
        d.addCallback(lambda res: n)
        return d

-    def create_new_mutable_directory(self, initial_children={}, version=None):
-        # initial_children must have metadata (i.e. {} instead of None)
+    def create_new_mutable_directory(self, initial_children=None, version=None):
+        if initial_children is None:
+            initial_children = {}
        for (name, (node, metadata)) in initial_children.items():
            precondition(isinstance(metadata, dict),
                         "create_new_mutable_directory requires metadata to be a dict, not None", metadata)
--- a/src/allmydata/protocol_switch.py
+++ b/src/allmydata/protocol_switch.py
@ -16,9 +16,10 @@ later in the configuration process.
 from __future__ import annotations

 from itertools import chain
+from typing import cast

 from twisted.internet.protocol import Protocol
-from twisted.internet.interfaces import IDelayedCall
+from twisted.internet.interfaces import IDelayedCall, IReactorFromThreads
 from twisted.internet.ssl import CertificateOptions
 from twisted.web.server import Site
 from twisted.protocols.tls import TLSMemoryBIOFactory
@ -89,7 +90,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
            certificate=cls.tub.myCertificate.original,
        )

-        http_storage_server = HTTPServer(reactor, storage_server, swissnum)
+        http_storage_server = HTTPServer(cast(IReactorFromThreads, reactor), storage_server, swissnum)
        cls.https_factory = TLSMemoryBIOFactory(
            certificate_options,
            False,
@ -102,8 +103,18 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
        for location_hint in chain.from_iterable(
            hints.split(",") for hints in cls.tub.locationHints
        ):
-            if location_hint.startswith("tcp:"):
-                _, hostname, port = location_hint.split(":")
+            if location_hint.startswith("tcp:") or location_hint.startswith("tor:"):
+                scheme, hostname, port = location_hint.split(":")
+                if scheme == "tcp":
+                    subscheme = None
+                else:
+                    subscheme = "tor"
+                    # If we're listening on Tor, the hostname needs to have an
+                    # .onion TLD.
+                    assert hostname.endswith(".onion")
+                # The I2P scheme is yet not supported by the HTTP client, so we
+                # don't want generate a NURL that won't work. This will be
+                # fixed in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4037
                port = int(port)
                storage_nurls.add(
                    build_nurl(
@ -111,12 +122,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
                        port,
                        str(swissnum, "ascii"),
                        cls.tub.myCertificate.original.to_cryptography(),
+                        subscheme
                    )
                )
-            # TODO this is probably where we'll have to support Tor and I2P?
-            # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9
-            # for discussion (there will be separate tickets added for those at
-            # some point.)
+
        return storage_nurls

    def __init__(self, *args, **kwargs):
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@ -112,6 +112,9 @@ class AddGridManagerCertOptions(BaseOptions):
        return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]"

    def postOptions(self) -> None:
+        assert self.parent is not None
+        assert self.parent.parent is not None
+
        if self['name'] is None:
            raise usage.UsageError(
                "Must provide --name option"
@ -123,8 +126,8 @@ class AddGridManagerCertOptions(BaseOptions):

        data: str
        if self['filename'] == '-':
-            print("reading certificate from stdin", file=self.parent.parent.stderr)
-            data = self.parent.parent.stdin.read()
+            print("reading certificate from stdin", file=self.parent.parent.stderr)  # type: ignore[attr-defined]
+            data = self.parent.parent.stdin.read()  # type: ignore[attr-defined]
            if len(data) == 0:
                raise usage.UsageError(
                    "Reading certificate from stdin failed"
@ -255,9 +258,9 @@ def do_admin(options):
    return f(so)


-subCommands = [
+subCommands : SubCommands = [
    ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"),
-    ]  # type: SubCommands
+    ]

 dispatch = {
    "admin": do_admin,
--- a/src/allmydata/scripts/cli.py
+++ b/src/allmydata/scripts/cli.py
@ -1,22 +1,10 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-

 import os.path, re, fnmatch

-try:
-    from allmydata.scripts.types_ import SubCommands, Parameters
-except ImportError:
-    pass
+from allmydata.scripts.types_ import SubCommands, Parameters

 from twisted.python import usage
 from allmydata.scripts.common import get_aliases, get_default_nodedir, \
@ -29,14 +17,14 @@ NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
 _default_nodedir = get_default_nodedir()

 class FileStoreOptions(BaseOptions):
-    optParameters = [
+    optParameters : Parameters = [
        ["node-url", "u", None,
         "Specify the URL of the Tahoe gateway node, such as "
         "'http://127.0.0.1:3456'. "
         "This overrides the URL found in the --node-directory ."],
        ["dir-cap", None, None,
         "Specify which dirnode URI should be used as the 'tahoe' alias."]
-        ]  # type: Parameters
+        ]

    def postOptions(self):
        self["quiet"] = self.parent["quiet"]
@ -484,7 +472,7 @@ class DeepCheckOptions(FileStoreOptions):
    (which must be a directory), like 'tahoe check' but for multiple files.
    Optionally repair any problems found."""

-subCommands = [
+subCommands : SubCommands = [
    ("mkdir", None, MakeDirectoryOptions, "Create a new directory."),
    ("add-alias", None, AddAliasOptions, "Add a new alias cap."),
    ("create-alias", None, CreateAliasOptions, "Create a new alias cap."),
@ -503,7 +491,7 @@ subCommands = [
    ("check", None, CheckOptions, "Check a single file or directory."),
    ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."),
    ("status", None, TahoeStatusCommand, "Various status information."),
-    ]  # type: SubCommands
+    ]

 def mkdir(options):
    from allmydata.scripts import tahoe_mkdir
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@ -4,29 +4,13 @@
 Ported to Python 3.
 """

-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-else:
-    from typing import Union
-
+from typing import Union, Optional

 import os, sys, textwrap
 import codecs
 from os.path import join
 import urllib.parse

-try:
-    from typing import Optional
-    from .types_ import Parameters
-except ImportError:
-    pass
-
 from yaml import (
    safe_dump,
 )
@ -37,6 +21,8 @@ from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import quote_output, \
    quote_local_unicode_path, argv_to_abspath
 from allmydata.scripts.default_nodedir import _default_nodedir
+from .types_ import Parameters
+

 def get_default_nodedir():
    return _default_nodedir
@ -59,7 +45,7 @@ class BaseOptions(usage.Options):
    def opt_version(self):
        raise usage.UsageError("--version not allowed on subcommands")

-    description = None  # type: Optional[str]
+    description : Optional[str] = None
    description_unwrapped = None  # type: Optional[str]

    def __str__(self):
@ -80,10 +66,10 @@ class BaseOptions(usage.Options):
 class BasedirOptions(BaseOptions):
    default_nodedir = _default_nodedir

-    optParameters = [
+    optParameters : Parameters = [
        ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]"
         % quote_local_unicode_path(_default_nodedir)],
-    ]  # type: Parameters
+    ]

    def parseArgs(self, basedir=None):
        # This finds the node-directory option correctly even if we are in a subcommand.
@ -283,9 +269,8 @@ def get_alias(aliases, path_unicode, default):
                                quote_output(alias))
    return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]

-def escape_path(path):
-    # type: (Union[str,bytes]) -> str
-    u"""
+def escape_path(path: Union[str, bytes]) -> str:
+    """
    Return path quoted to US-ASCII, valid URL characters.

    >>> path = u'/føö/bar/☃'
@ -302,9 +287,4 @@ def escape_path(path):
        ]),
        "ascii"
    )
-    # Eventually (i.e. as part of Python 3 port) we want this to always return
-    # Unicode strings. However, to reduce diff sizes in the short term it'll
-    # return native string (i.e. bytes) on Python 2.
-    if PY2:
-        result = result.encode("ascii").__native__()
    return result
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@ -1,19 +1,11 @@
 """
-Ported to Python 3.
+Blocking HTTP client APIs.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401

 import os
 from io import BytesIO
-from six.moves import urllib, http_client
-import six
+from http import client as http_client
+import urllib
 import allmydata # for __full_version__

 from allmydata.util.encodingutil import quote_output
@ -51,7 +43,7 @@ class BadResponse(object):
 def do_http(method, url, body=b""):
    if isinstance(body, bytes):
        body = BytesIO(body)
-    elif isinstance(body, six.text_type):
+    elif isinstance(body, str):
        raise TypeError("do_http body must be a bytestring, not unicode")
    else:
        # We must give a Content-Length header to twisted.web, otherwise it
@ -61,10 +53,17 @@ def do_http(method, url, body=b""):
        assert body.seek
        assert body.read
    scheme, host, port, path = parse_url(url)
+
+    # For testing purposes, allow setting a timeout on HTTP requests. If this
+    # ever become a user-facing feature, this should probably be a CLI option?
+    timeout = os.environ.get("__TAHOE_CLI_HTTP_TIMEOUT", None)
+    if timeout is not None:
+        timeout = float(timeout)
+
    if scheme == "http":
-        c = http_client.HTTPConnection(host, port)
+        c = http_client.HTTPConnection(host, port, timeout=timeout, blocksize=65536)
    elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port)
+        c = http_client.HTTPSConnection(host, port, timeout=timeout, blocksize=65536)
    else:
        raise ValueError("unknown scheme '%s', need http or https" % scheme)
    c.putrequest(method, path)
@ -85,7 +84,7 @@ def do_http(method, url, body=b""):
        return BadResponse(url, err)

    while True:
-        data = body.read(8192)
+        data = body.read(65536)
        if not data:
            break
        c.send(data)
@ -94,16 +93,14 @@ def do_http(method, url, body=b""):


 def format_http_success(resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
    return quote_output(
-        "%s %s" % (resp.status, six.ensure_text(resp.reason)),
+        "%s %s" % (resp.status, resp.reason),
        quotemarks=False)

 def format_http_error(msg, resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
    return quote_output(
-        "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason),
-                           six.ensure_text(resp.read())),
+        "%s: %s %s\n%r" % (msg, resp.status, resp.reason,
+                           resp.read()),
        quotemarks=False)

 def check_http_error(resp, stderr):
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@ -1,25 +1,16 @@
-# Ported to Python 3

-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
+from __future__ import annotations

-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from typing import Optional

 import io
 import os

-try:
-    from allmydata.scripts.types_ import (
-        SubCommands,
-        Parameters,
-        Flags,
-    )
-except ImportError:
-    pass
+from allmydata.scripts.types_ import (
+    SubCommands,
+    Parameters,
+    Flags,
+)

 from twisted.internet import reactor, defer
 from twisted.python.usage import UsageError
@ -33,9 +24,40 @@ from allmydata.scripts.common import (
    write_introducer,
 )
 from allmydata.scripts.default_nodedir import _default_nodedir
+from allmydata.util import dictutil
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding
-from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json
+
+i2p_provider: Listener
+tor_provider: Listener
+
+from allmydata.util import fileutil, i2p_provider, tor_provider, jsonbytes as json
+
+from ..listeners import ListenerConfig, Listener, TCPProvider, StaticProvider
+
+def _get_listeners() -> dict[str, Listener]:
+    """
+    Get all of the kinds of listeners we might be able to use.
+    """
+    return {
+        "tor": tor_provider,
+        "i2p": i2p_provider,
+        "tcp": TCPProvider(),
+        "none": StaticProvider(
+            available=True,
+            hide_ip=False,
+            config=defer.succeed(None),
+            # This is supposed to be an IAddressFamily but we have none for
+            # this kind of provider.  We could implement new client and server
+            # endpoint types that always fail and pass an IAddressFamily here
+            # that uses those.  Nothing would ever even ask for them (at
+            # least, yet), let alone try to use them, so that's a lot of extra
+            # work for no practical result so I'm not doing it now.
+            address=None, # type: ignore[arg-type]
+        ),
+    }
+
+_LISTENERS = _get_listeners()

 dummy_tac = """
 import sys
@ -48,7 +70,7 @@ def write_tac(basedir, nodetype):
    fileutil.write(os.path.join(basedir, "tahoe-%s.tac" % (nodetype,)), dummy_tac)


-WHERE_OPTS = [
+WHERE_OPTS : Parameters = [
    ("location", None, None,
     "Server location to advertise (e.g. tcp:example.org:12345)"),
    ("port", None, None,
@ -57,29 +79,29 @@ WHERE_OPTS = [
     "Hostname to automatically set --location/--port when --listen=tcp"),
    ("listen", None, "tcp",
     "Comma-separated list of listener types (tcp,tor,i2p,none)."),
-] # type: Parameters
+]

-TOR_OPTS = [
+TOR_OPTS : Parameters = [
    ("tor-control-port", None, None,
     "Tor's control port endpoint descriptor string (e.g. tcp:127.0.0.1:9051 or unix:/var/run/tor/control)"),
    ("tor-executable", None, None,
     "The 'tor' executable to run (default is to search $PATH)."),
-] # type: Parameters
+]

-TOR_FLAGS = [
+TOR_FLAGS : Flags = [
    ("tor-launch", None, "Launch a tor instead of connecting to a tor control port."),
-] # type: Flags
+]

-I2P_OPTS = [
+I2P_OPTS : Parameters = [
    ("i2p-sam-port", None, None,
     "I2P's SAM API port endpoint descriptor string (e.g. tcp:127.0.0.1:7656)"),
    ("i2p-executable", None, None,
     "(future) The 'i2prouter' executable to run (default is to search $PATH)."),
-] # type: Parameters
+]

-I2P_FLAGS = [
+I2P_FLAGS : Flags = [
    ("i2p-launch", None, "(future) Launch an I2P router instead of connecting to a SAM API port."),
-] # type: Flags
+]

 def validate_where_options(o):
    if o['listen'] == "none":
@ -112,8 +134,11 @@ def validate_where_options(o):
        if o['listen'] != "none" and o.get('join', None) is None:
            listeners = o['listen'].split(",")
            for l in listeners:
-                if l not in ["tcp", "tor", "i2p"]:
-                    raise UsageError("--listen= must be none, or one/some of: tcp, tor, i2p")
+                if l not in _LISTENERS:
+                    raise UsageError(
+                        "--listen= must be one/some of: "
+                        f"{', '.join(sorted(_LISTENERS))}",
+                    )
            if 'tcp' in listeners and not o['hostname']:
                raise UsageError("--listen=tcp requires --hostname=")
            if 'tcp' not in listeners and o['hostname']:
@ -122,7 +147,7 @@ def validate_where_options(o):
 def validate_tor_options(o):
    use_tor = "tor" in o["listen"].split(",")
    if use_tor or any((o["tor-launch"], o["tor-control-port"])):
-        if tor_provider._import_txtorcon() is None:
+        if not _LISTENERS["tor"].is_available():
            raise UsageError(
                "Specifying any Tor options requires the 'txtorcon' module"
            )
@ -137,7 +162,7 @@ def validate_tor_options(o):
 def validate_i2p_options(o):
    use_i2p = "i2p" in o["listen"].split(",")
    if use_i2p or any((o["i2p-launch"], o["i2p-sam-port"])):
-        if i2p_provider._import_txi2p() is None:
+        if not _LISTENERS["i2p"].is_available():
            raise UsageError(
                "Specifying any I2P options requires the 'txi2p' module"
            )
@ -159,11 +184,17 @@ class _CreateBaseOptions(BasedirOptions):
    def postOptions(self):
        super(_CreateBaseOptions, self).postOptions()
        if self['hide-ip']:
-            if tor_provider._import_txtorcon() is None and i2p_provider._import_txi2p() is None:
+            ip_hiders = dictutil.filter(lambda v: v.can_hide_ip(), _LISTENERS)
+            available = dictutil.filter(lambda v: v.is_available(), ip_hiders)
+            if not available:
                raise UsageError(
-                    "--hide-ip was specified but neither 'txtorcon' nor 'txi2p' "
-                    "are installed.\nTo do so:\n   pip install tahoe-lafs[tor]\nor\n"
-                    "   pip install tahoe-lafs[i2p]"
+                    "--hide-ip was specified but no IP-hiding listener is installed.\n"
+                    "Try one of these:\n" +
+                    "".join([
+                        f"\tpip install tahoe-lafs[{name}]\n"
+                        for name
+                        in ip_hiders
+                    ])
                )

 class CreateClientOptions(_CreateBaseOptions):
@ -232,8 +263,34 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions):
        validate_i2p_options(self)


-@defer.inlineCallbacks
-def write_node_config(c, config):
+def merge_config(
+        left: Optional[ListenerConfig],
+        right: Optional[ListenerConfig],
+) -> Optional[ListenerConfig]:
+    """
+    Merge two listener configurations into one configuration representing
+    both of them.
+
+    If either is ``None`` then the result is ``None``.  This supports the
+    "disable listeners" functionality.
+
+    :raise ValueError: If the keys in the node configs overlap.
+    """
+    if left is None or right is None:
+        return None
+
+    overlap = set(left.node_config) & set(right.node_config)
+    if overlap:
+        raise ValueError(f"Node configs overlap: {overlap}")
+
+    return ListenerConfig(
+        list(left.tub_ports) + list(right.tub_ports),
+        list(left.tub_locations) + list(right.tub_locations),
+        dict(list(left.node_config.items()) + list(right.node_config.items())),
+    )
+
+
+async def write_node_config(c, config):
    # this is shared between clients and introducers
    c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c))
    c.write("\n")
@ -246,9 +303,10 @@ def write_node_config(c, config):

    if config["hide-ip"]:
        c.write("[connections]\n")
-        if tor_provider._import_txtorcon():
+        if _LISTENERS["tor"].is_available():
            c.write("tcp = tor\n")
        else:
+            # XXX What about i2p?
            c.write("tcp = disabled\n")
        c.write("\n")

@ -267,38 +325,23 @@ def write_node_config(c, config):
    c.write("web.port = %s\n" % (webport,))
    c.write("web.static = public_html\n")

-    listeners = config['listen'].split(",")
+    listener_config = ListenerConfig([], [], {})
+    for listener_name in config['listen'].split(","):
+        listener = _LISTENERS[listener_name]
+        listener_config = merge_config(
+            (await listener.create_config(reactor, config)),
+            listener_config,
+        )

-    tor_config = {}
-    i2p_config = {}
-    tub_ports = []
-    tub_locations = []
-    if listeners == ["none"]:
-        c.write("tub.port = disabled\n")
-        c.write("tub.location = disabled\n")
+    if listener_config is None:
+        tub_ports = ["disabled"]
+        tub_locations = ["disabled"]
    else:
-        if "tor" in listeners:
-            (tor_config, tor_port, tor_location) = \
-                         yield tor_provider.create_config(reactor, config)
-            tub_ports.append(tor_port)
-            tub_locations.append(tor_location)
-        if "i2p" in listeners:
-            (i2p_config, i2p_port, i2p_location) = \
-                         yield i2p_provider.create_config(reactor, config)
-            tub_ports.append(i2p_port)
-            tub_locations.append(i2p_location)
-        if "tcp" in listeners:
-            if config["port"]: # --port/--location are a pair
-                tub_ports.append(config["port"])
-                tub_locations.append(config["location"])
-            else:
-                assert "hostname" in config
-                hostname = config["hostname"]
-                new_port = iputil.allocate_tcp_port()
-                tub_ports.append("tcp:%s" % new_port)
-                tub_locations.append("tcp:%s:%s" % (hostname, new_port))
-        c.write("tub.port = %s\n" % ",".join(tub_ports))
-        c.write("tub.location = %s\n" % ",".join(tub_locations))
+        tub_ports = listener_config.tub_ports
+        tub_locations = listener_config.tub_locations
+
+    c.write("tub.port = %s\n" % ",".join(tub_ports))
+    c.write("tub.location = %s\n" % ",".join(tub_locations))
    c.write("\n")

    c.write("#log_gatherer.furl =\n")
@ -308,17 +351,12 @@ def write_node_config(c, config):
    c.write("#ssh.authorized_keys_file = ~/.ssh/authorized_keys\n")
    c.write("\n")

-    if tor_config:
-        c.write("[tor]\n")
-        for key, value in list(tor_config.items()):
-            c.write("%s = %s\n" % (key, value))
-        c.write("\n")
-
-    if i2p_config:
-        c.write("[i2p]\n")
-        for key, value in list(i2p_config.items()):
-            c.write("%s = %s\n" % (key, value))
-        c.write("\n")
+    if listener_config is not None:
+        for section, items in listener_config.node_config.items():
+            c.write(f"[{section}]\n")
+            for k, v in items:
+                c.write(f"{k} = {v}\n")
+            c.write("\n")


 def write_client_config(c, config):
@ -459,7 +497,7 @@ def create_node(config):
    fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
    cfg_name = os.path.join(basedir, "tahoe.cfg")
    with io.open(cfg_name, "w", encoding='utf-8') as c:
-        yield write_node_config(c, config)
+        yield defer.Deferred.fromCoroutine(write_node_config(c, config))
        write_client_config(c, config)

    print("Node created in %s" % quote_local_unicode_path(basedir), file=out)
@ -502,17 +540,17 @@ def create_introducer(config):
    fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
    cfg_name = os.path.join(basedir, "tahoe.cfg")
    with io.open(cfg_name, "w", encoding='utf-8') as c:
-        yield write_node_config(c, config)
+        yield defer.Deferred.fromCoroutine(write_node_config(c, config))

    print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)
    defer.returnValue(0)


-subCommands = [
+subCommands : SubCommands = [
    ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."),
    ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."),
    ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."),
-]  # type: SubCommands
+]

 dispatch = {
    "create-node": create_node,
--- a/src/allmydata/scripts/debug.py
+++ b/src/allmydata/scripts/debug.py
@ -1,19 +1,8 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function

-from future.utils import PY2, bchr
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
-try:
-    from allmydata.scripts.types_ import SubCommands
-except ImportError:
-    pass
+from future.utils import bchr

 import struct, time, os, sys

@ -31,6 +20,7 @@ from allmydata.mutable.common import NeedMoreDataError
 from allmydata.immutable.layout import ReadBucketProxy
 from allmydata.util import base32
 from allmydata.util.encodingutil import quote_output
+from allmydata.scripts.types_ import SubCommands

 class DumpOptions(BaseOptions):
    def getSynopsis(self):
@ -1076,9 +1066,9 @@ def do_debug(options):
    return f(so)


-subCommands = [
+subCommands : SubCommands = [
    ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
-    ]  # type: SubCommands
+    ]

 dispatch = {
    "debug": do_debug,
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version.`
				`@ -0,0 +1 @@`
				`A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed.`
				`@ -0,0 +1 @@`
				`tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes"`