Start hooking up end-to-end tests with TLS, fixing bugs along the way.

At this point the issue is that the client fails certificate validation (which
is expected lacking the pinning validation logic, which should be added next).

src/allmydata/storage/http_client.py

@@ -3,7 +3,6 @@ HTTP client that talks to the HTTP storage server.
 """
 
 from typing import Union, Set, Optional
-from treq.testing import StubTreq
 
 from base64 import b64encode
 
@@ -77,7 +76,7 @@ class StorageClient(object):
         self._treq = treq
 
     @classmethod
-    def from_furl(cls, furl: DecodedURL) -> "StorageClient":
+    def from_furl(cls, furl: DecodedURL, treq=treq) -> "StorageClient":
         """
         Create a ``StorageClient`` for the given furl.
         """
@@ -86,6 +85,9 @@ class StorageClient(object):
         swissnum = furl.path[0].encode("ascii")
         certificate_hash = furl.user.encode("ascii")
 
+        https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port)
+        return cls(https_url, swissnum, treq)
+
     def relative_url(self, path):
         """Get a URL relative to the base URL."""
         return self._base_url.click(path)

src/allmydata/storage/http_common.py

@@ -48,4 +48,4 @@ def get_spki_hash(certificate: Certificate) -> bytes:
     public_key_bytes = certificate.public_key().public_bytes(
         Encoding.DER, PublicFormat.SubjectPublicKeyInfo
     )
-    return b64encode(sha256(public_key_bytes).digest()).strip()
+    return b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=")

src/allmydata/storage/http_server.py

@@ -534,6 +534,8 @@ def listen_tls(
     The hostname is the external IP or hostname clients will connect to; it
     does not modify what interfaces the server listens on.  To set the
     listening interface, use the ``interface`` argument.
+
+    Port can be 0 to choose a random port.
     """
     endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format(
         quoteStringArgument(str(private_key_path)),
@@ -545,13 +547,19 @@ def listen_tls(
     endpoint = serverFromString(reactor, endpoint_string)
 
     def build_furl(listening_port: IListeningPort) -> DecodedURL:
-        furl = DecodedURL()
-        furl.fragment = "v=1"  # HTTP-based
-        furl.host = hostname
-        furl.port = listening_port.getHost().port
-        furl.path = (server._swissnum,)
-        furl.user = get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes()))
-        furl.scheme = "pb"
+        furl = DecodedURL().replace(
+            fragment="v=1",  # HTTP-based
+            host=hostname,
+            port=listening_port.getHost().port,
+            path=(str(server._swissnum, "ascii"),),
+            userinfo=[
+                str(
+                    get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())),
+                    "ascii",
+                )
+            ],
+            scheme="pb",
+        )
         return furl
 
     return endpoint.listen(Site(server.get_resource())).addCallback(

src/allmydata/test/certs/domain.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDCzCCAfMCFHrs4pMBs35SlU3ZGMnVY5qp5MfZMA0GCSqGSIb3DQEBCwUAMEIx
+CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl
+ZmF1bHQgQ29tcGFueSBMdGQwHhcNMjIwMzIzMjAwNTM0WhcNMjIwNDIyMjAwNTM0
+WjBCMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQK
+DBNEZWZhdWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiESF/Lvnrc
+VcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsxx89u/W5B
+VgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4XtqlCKNFGYO
+7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVmr7UgatuA
+LrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8ACif+ZQJc
+AukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCv
+BzVSinez5lcSWWCRve/TlMePEJK5d7OFcc90n7kmn+rkEYrel3a7Q+ctJxY9AKYG
+A9X1AMDSH9z3243KFNaRJ1xKg0Mg8J/BLN9iphM5AnAuiAkCqs8VbD4hF4hZHLMZ
+BVNyuLSdo+lBzbS57/Lz+lUWcxrXR5qgsEWSbjP+SrsDQKODfyoxKuU0XrxmNLd2
+dTswbZKsqXBs80/T1jHwJjJXLp6YUsZqN1TGYtk8hEcE7bGaC3n7WhRjBP1WghNl
+OG7FFRPte2w5seQRgkrBodLb9OCkhU4xfdyLnFICqkQHQAqIdXksEvir9uGY/yjC
+zxAh60LEEQz6Kz29jYog
+-----END CERTIFICATE-----

src/allmydata/test/certs/private.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiE
+SF/LvnrcVcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsx
+x89u/W5BVgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4Xtq
+lCKNFGYO7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVm
+r7UgatuALrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8A
+Cif+ZQJcAukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABAoIBAG71rGDuIazif+Bq
+PGDDs/c5q3BQ9LLdF6Zywonp3J0CFqbbc/BsefYVrA4I6mnqECd2TWsO+cfyKxeb
+aRrDne75l9YZcaXU2ZKqtIKShHQgqlV2giX6mMCJXWWlenfRMglooLaGslxYZR6e
+/GG9iVbXLI0m52EhYjH21W6MVgXUhsrDoI16pB87zk7jFZzyNsjRU5+bwr4L2jed
+A1PseE6AI2kIpJCl8IIu6hRhVwjr8MIkaAtI3G8WmSAru99apHNttf6sgB2kcq2r
+Qp1uXEXNVFQiJqcwMPOlWZ5X0kMIBxmFe10MkJbmCUoE/jPqO90XN2jyZPSONZU8
+4yqd9GECgYEA5qQ+tfgOJFJ86t103pwkPm0+PxuQUTXry5ETnez+wxqqwbDxrEHi
+MQoPZuVXPkbQ6g80KSpdI7AkFvu6BzcNlgpOI3gLZ30PHTF4HJTP01fKfbbVhg8N
+WJS0yUh+kQDrGVcZbIbB5Q1vS5hu8ftk5ukns5BdFf/NS7fBfU8b3K0CgYEA5Onm
+V2D1D9kdhTjta8f0d0s6+TdHoV86SRbkAEgnqzwlHWV17LlpXQic6iwokfS4TQSl
++1Z23Dt+OhLm/N0N3LgCxBhzTMnWGdy+w9co4GifwqR6T72JAxGOVoqIWk2cVpa5
+8qJx0eAFXqcvpIASEoxYrdoKFUh60mAiQE6JQ08CgYB8wCoLUviTPOrEPrSQE/Sm
+r4ATsl0FEB1SJk5uBVpnPW1PBt4xRhGKZN6f0Ty3OqaVc1PLUFbAju12YQHmFSkM
+Ftbc6HmCqGocaD2HeBZRQhMMnHAx6sJVP1np5YRP+icvtaTSxrDpq7KfOPwJdujE
+3SfUQCmZVJs+cU3+8WMooQKBgQCdvvl2eWAm/a00IxipT2+NzY/kMU3xTFg0Ccww
+zYhYnefNrB9pdBPBgq/vR2LlwchHes6OtvTNq0m+50u6MPLeiQeO7nJ2FhiuVco3
+1staaX6+eO24iZojPTPjOy/fWuBDYzbcl0jsIf5RTdCtAXxyv7hUhY6xP/Mzif/Q
+ZM5+TQKBgQCF7pB6yLIAp6YJQ4uqOVbC2bMLr6tVWaNdEykiDd9zQkoacw/DPX1Y
+FKehY/9EKraJ4t6d2/BBpJQyuIU4/gz8QMvjqQGP3NIfVeqBAPYo/nTYKOK0PSxB
+Kl28Axxz6rjEeK4BixOES5PXuq2nNJXT8OSPYZQxQdTHstCWOP4Z6g==
+-----END RSA PRIVATE KEY-----

src/allmydata/test/test_istorageserver.py

@@ -24,12 +24,11 @@ else:
 
 from random import Random
 from unittest import SkipTest
+from pathlib import Path
 
 from twisted.internet.defer import inlineCallbacks, returnValue, succeed
 from twisted.internet.task import Clock
 from twisted.internet import reactor
-from twisted.internet.endpoints import serverFromString
-from twisted.web.server import Site
 from twisted.web.client import Agent, HTTPConnectionPool
 from hyperlink import DecodedURL
 from treq.client import HTTPClient
@@ -40,7 +39,7 @@ from allmydata.interfaces import IStorageServer  # really, IStorageClient
 from .common_system import SystemTestMixin
 from .common import AsyncTestCase, SameProcessStreamEndpointAssigner
 from allmydata.storage.server import StorageServer  # not a IStorageServer!!
-from allmydata.storage.http_server import HTTPServer
+from allmydata.storage.http_server import HTTPServer, listen_tls
 from allmydata.storage.http_client import StorageClient
 from allmydata.storage_client import _HTTPStorageServer
 
@@ -1074,27 +1073,29 @@ class _HTTPMixin(_SharedMixin):
         swissnum = b"1234"
         http_storage_server = HTTPServer(self.server, swissnum)
 
-        # Listen on randomly assigned port:
-        tcp_address, endpoint_string = self._port_assigner.assign(reactor)
-        _, host, port = tcp_address.split(":")
-        port = int(port)
-        endpoint = serverFromString(reactor, endpoint_string)
-        listening_port = yield endpoint.listen(Site(http_storage_server.get_resource()))
+        # Listen on randomly assigned port, using self-signed cert we generated
+        # manually:
+        certs_dir = Path(__file__).parent / "certs"
+        furl, listening_port = yield listen_tls(
+            reactor,
+            http_storage_server,
+            "127.0.0.1",
+            0,
+            certs_dir / "private.key",
+            certs_dir / "domain.crt",
+            interface="127.0.0.1",
+        )
         self.addCleanup(listening_port.stopListening)
 
         # Create HTTP client with non-persistent connections, so we don't leak
         # state across tests:
         treq_client = HTTPClient(
-            Agent(reactor, HTTPConnectionPool(reactor, persistent=False))
+            Agent(reactor, pool=HTTPConnectionPool(reactor, persistent=False))
         )
 
         returnValue(
             _HTTPStorageServer.from_http_client(
-                StorageClient(
-                    DecodedURL().replace(scheme="http", host=host, port=port),
-                    swissnum,
-                    treq=treq_client,
-                )
+                StorageClient.from_furl(furl, treq_client)
             )
         )
         # Eventually should also: