Merge branch '4028-more-integration' into 4027-invalid-unicode

This commit is contained in:
Itamar Turner-Trauring 2023-05-23 14:08:09 -04:00
commit b03db14d70
9 changed files with 46 additions and 26 deletions

View File

@ -53,9 +53,9 @@ jobs:
- "3.11" - "3.11"
include: include:
# On macOS don't bother with 3.8, just to get faster builds. # On macOS don't bother with 3.8, just to get faster builds.
- os: macos-latest - os: macos-12
python-version: "3.9" python-version: "3.9"
- os: macos-latest - os: macos-12
python-version: "3.11" python-version: "3.11"
# We only support PyPy on Linux at the moment. # We only support PyPy on Linux at the moment.
- os: ubuntu-latest - os: ubuntu-latest
@ -165,7 +165,7 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- os: macos-latest - os: macos-12
python-version: "3.9" python-version: "3.9"
force-foolscap: false force-foolscap: false
- os: windows-latest - os: windows-latest
@ -248,7 +248,7 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
os: os:
- macos-10.15 - macos-12
- windows-latest - windows-latest
- ubuntu-latest - ubuntu-latest
python-version: python-version:

View File

@ -401,9 +401,6 @@ def alice(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
web_port="tcp:9980:interface=localhost", web_port="tcp:9980:interface=localhost",
storage=False, storage=False,
# We're going to kill this ourselves, so no need for finalizer to
# do it:
finalize=False,
) )
) )
pytest_twisted.blockon(await_client_ready(process)) pytest_twisted.blockon(await_client_ready(process))

View File

@ -6,8 +6,9 @@ and stdout.
from subprocess import Popen, PIPE, check_output, check_call from subprocess import Popen, PIPE, check_output, check_call
import pytest import pytest
from pytest_twisted import ensureDeferred
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.threads import blockingCallFromThread
from twisted.internet.defer import Deferred
from .util import run_in_thread, cli, reconfigure from .util import run_in_thread, cli, reconfigure
@ -86,8 +87,8 @@ def test_large_file(alice, get_put_alias, tmp_path):
assert outfile.read_bytes() == tempfile.read_bytes() assert outfile.read_bytes() == tempfile.read_bytes()
@ensureDeferred @run_in_thread
async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
""" """
Tahoe-LAFS used to have a default max segment size of 128KB, and is now Tahoe-LAFS used to have a default max segment size of 128KB, and is now
1MB. Test that an upload created when 128KB was the default can be 1MB. Test that an upload created when 128KB was the default can be
@ -100,22 +101,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
with tempfile.open("wb") as f: with tempfile.open("wb") as f:
f.write(large_data) f.write(large_data)
async def set_segment_size(segment_size): def set_segment_size(segment_size):
await reconfigure( return blockingCallFromThread(
reactor,
lambda: Deferred.fromCoroutine(reconfigure(
reactor, reactor,
request, request,
alice, alice,
(1, 1, 1), (1, 1, 1),
None, None,
max_segment_size=segment_size max_segment_size=segment_size
))
) )
# 1. Upload file 1 with default segment size set to 1MB # 1. Upload file 1 with default segment size set to 1MB
await set_segment_size(1024 * 1024) set_segment_size(1024 * 1024)
cli(alice, "put", str(tempfile), "getput:seg1024kb") cli(alice, "put", str(tempfile), "getput:seg1024kb")
# 2. Download file 1 with default segment size set to 128KB # 2. Download file 1 with default segment size set to 128KB
await set_segment_size(128 * 1024) set_segment_size(128 * 1024)
assert large_data == check_output( assert large_data == check_output(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"] ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
) )
@ -124,7 +128,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
cli(alice, "put", str(tempfile), "getput:seg128kb") cli(alice, "put", str(tempfile), "getput:seg128kb")
# 4. Download file 2 with default segment size set to 1MB # 4. Download file 2 with default segment size set to 1MB
await set_segment_size(1024 * 1024) set_segment_size(1024 * 1024)
assert large_data == check_output( assert large_data == check_output(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"] ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
) )

View File

@ -14,6 +14,8 @@ from __future__ import annotations
import time import time
from urllib.parse import unquote as url_unquote, quote as url_quote from urllib.parse import unquote as url_unquote, quote as url_quote
from twisted.internet.threads import deferToThread
import allmydata.uri import allmydata.uri
from allmydata.util import jsonbytes as json from allmydata.util import jsonbytes as json
@ -24,7 +26,7 @@ import requests
import html5lib import html5lib
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from pytest_twisted import ensureDeferred import pytest_twisted
@run_in_thread @run_in_thread
def test_index(alice): def test_index(alice):
@ -185,7 +187,7 @@ def test_deep_stats(alice):
time.sleep(.5) time.sleep(.5)
@util.run_in_thread @run_in_thread
def test_status(alice): def test_status(alice):
""" """
confirm we get something sensible from /status and the various sub-types confirm we get something sensible from /status and the various sub-types
@ -251,7 +253,7 @@ def test_status(alice):
assert found_download, "Failed to find the file we downloaded in the status-page" assert found_download, "Failed to find the file we downloaded in the status-page"
@ensureDeferred @pytest_twisted.ensureDeferred
async def test_directory_deep_check(reactor, request, alice): async def test_directory_deep_check(reactor, request, alice):
""" """
use deep-check and confirm the result pages work use deep-check and confirm the result pages work
@ -263,7 +265,10 @@ async def test_directory_deep_check(reactor, request, alice):
total = 4 total = 4
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
await deferToThread(_test_directory_deep_check_blocking, alice)
def _test_directory_deep_check_blocking(alice):
# create a directory # create a directory
resp = requests.post( resp = requests.post(
util.node_url(alice.node_dir, u"uri"), util.node_url(alice.node_dir, u"uri"),

0
newsfragments/4023.minor Normal file
View File

0
newsfragments/4028.minor Normal file
View File

View File

@ -310,6 +310,7 @@ class StorageClient(object):
_base_url: DecodedURL _base_url: DecodedURL
_swissnum: bytes _swissnum: bytes
_treq: Union[treq, StubTreq, HTTPClient] _treq: Union[treq, StubTreq, HTTPClient]
_pool: Optional[HTTPConnectionPool]
_clock: IReactorTime _clock: IReactorTime
@classmethod @classmethod
@ -325,7 +326,7 @@ class StorageClient(object):
certificate_hash = nurl.user.encode("ascii") certificate_hash = nurl.user.encode("ascii")
if pool is None: if pool is None:
pool = HTTPConnectionPool(reactor) pool = HTTPConnectionPool(reactor)
pool.maxPersistentPerHost = 20 pool.maxPersistentPerHost = 10
if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
cls.TEST_MODE_REGISTER_HTTP_POOL(pool) cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
@ -339,7 +340,7 @@ class StorageClient(object):
) )
https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
return cls(https_url, swissnum, treq_client, reactor) return cls(https_url, swissnum, treq_client, pool, reactor)
def relative_url(self, path: str) -> DecodedURL: def relative_url(self, path: str) -> DecodedURL:
"""Get a URL relative to the base URL.""" """Get a URL relative to the base URL."""
@ -479,6 +480,11 @@ class StorageClient(object):
).read() ).read()
raise ClientException(response.code, response.phrase, data) raise ClientException(response.code, response.phrase, data)
def shutdown(self) -> Deferred:
"""Shutdown any connections."""
if self._pool is not None:
return self._pool.closeCachedConnections()
@define(hash=True) @define(hash=True)
class StorageClientGeneral(object): class StorageClientGeneral(object):

View File

@ -1271,6 +1271,11 @@ class HTTPNativeStorageServer(service.MultiService):
if self._lc.running: if self._lc.running:
self._lc.stop() self._lc.stop()
self._failed_to_connect("shut down") self._failed_to_connect("shut down")
if self._istorage_server is not None:
client_shutting_down = self._istorage_server._http_client.shutdown()
result.addCallback(lambda _: client_shutting_down)
return result return result

View File

@ -335,6 +335,7 @@ class CustomHTTPServerTests(SyncTestCase):
DecodedURL.from_text("http://127.0.0.1"), DecodedURL.from_text("http://127.0.0.1"),
SWISSNUM_FOR_TEST, SWISSNUM_FOR_TEST,
treq=treq, treq=treq,
pool=None,
# We're using a Treq private API to get the reactor, alas, but only # We're using a Treq private API to get the reactor, alas, but only
# in a test, so not going to worry about it too much. This would be # in a test, so not going to worry about it too much. This would be
# fixed if https://github.com/twisted/treq/issues/226 were ever # fixed if https://github.com/twisted/treq/issues/226 were ever
@ -555,6 +556,7 @@ class HttpTestFixture(Fixture):
DecodedURL.from_text("http://127.0.0.1"), DecodedURL.from_text("http://127.0.0.1"),
SWISSNUM_FOR_TEST, SWISSNUM_FOR_TEST,
treq=self.treq, treq=self.treq,
pool=None,
clock=self.clock, clock=self.clock,
) )
@ -667,6 +669,7 @@ class GenericHTTPAPITests(SyncTestCase):
DecodedURL.from_text("http://127.0.0.1"), DecodedURL.from_text("http://127.0.0.1"),
b"something wrong", b"something wrong",
treq=StubTreq(self.http.http_server.get_resource()), treq=StubTreq(self.http.http_server.get_resource()),
pool=None,
clock=self.http.clock, clock=self.http.clock,
) )
) )