Merge branch '4028-more-integration' into 4027-invalid-unicode
This commit is contained in:
commit
b03db14d70
8
.github/workflows/ci.yml
vendored
8
.github/workflows/ci.yml
vendored
@ -53,9 +53,9 @@ jobs:
|
|||||||
- "3.11"
|
- "3.11"
|
||||||
include:
|
include:
|
||||||
# On macOS don't bother with 3.8, just to get faster builds.
|
# On macOS don't bother with 3.8, just to get faster builds.
|
||||||
- os: macos-latest
|
- os: macos-12
|
||||||
python-version: "3.9"
|
python-version: "3.9"
|
||||||
- os: macos-latest
|
- os: macos-12
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
# We only support PyPy on Linux at the moment.
|
# We only support PyPy on Linux at the moment.
|
||||||
- os: ubuntu-latest
|
- os: ubuntu-latest
|
||||||
@ -165,7 +165,7 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- os: macos-latest
|
- os: macos-12
|
||||||
python-version: "3.9"
|
python-version: "3.9"
|
||||||
force-foolscap: false
|
force-foolscap: false
|
||||||
- os: windows-latest
|
- os: windows-latest
|
||||||
@ -248,7 +248,7 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
os:
|
os:
|
||||||
- macos-10.15
|
- macos-12
|
||||||
- windows-latest
|
- windows-latest
|
||||||
- ubuntu-latest
|
- ubuntu-latest
|
||||||
python-version:
|
python-version:
|
||||||
|
@ -401,9 +401,6 @@ def alice(
|
|||||||
reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
|
reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
|
||||||
web_port="tcp:9980:interface=localhost",
|
web_port="tcp:9980:interface=localhost",
|
||||||
storage=False,
|
storage=False,
|
||||||
# We're going to kill this ourselves, so no need for finalizer to
|
|
||||||
# do it:
|
|
||||||
finalize=False,
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
pytest_twisted.blockon(await_client_ready(process))
|
pytest_twisted.blockon(await_client_ready(process))
|
||||||
|
@ -6,8 +6,9 @@ and stdout.
|
|||||||
from subprocess import Popen, PIPE, check_output, check_call
|
from subprocess import Popen, PIPE, check_output, check_call
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from pytest_twisted import ensureDeferred
|
|
||||||
from twisted.internet import reactor
|
from twisted.internet import reactor
|
||||||
|
from twisted.internet.threads import blockingCallFromThread
|
||||||
|
from twisted.internet.defer import Deferred
|
||||||
|
|
||||||
from .util import run_in_thread, cli, reconfigure
|
from .util import run_in_thread, cli, reconfigure
|
||||||
|
|
||||||
@ -86,8 +87,8 @@ def test_large_file(alice, get_put_alias, tmp_path):
|
|||||||
assert outfile.read_bytes() == tempfile.read_bytes()
|
assert outfile.read_bytes() == tempfile.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
@ensureDeferred
|
@run_in_thread
|
||||||
async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
|
def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
|
||||||
"""
|
"""
|
||||||
Tahoe-LAFS used to have a default max segment size of 128KB, and is now
|
Tahoe-LAFS used to have a default max segment size of 128KB, and is now
|
||||||
1MB. Test that an upload created when 128KB was the default can be
|
1MB. Test that an upload created when 128KB was the default can be
|
||||||
@ -100,22 +101,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
|
|||||||
with tempfile.open("wb") as f:
|
with tempfile.open("wb") as f:
|
||||||
f.write(large_data)
|
f.write(large_data)
|
||||||
|
|
||||||
async def set_segment_size(segment_size):
|
def set_segment_size(segment_size):
|
||||||
await reconfigure(
|
return blockingCallFromThread(
|
||||||
|
reactor,
|
||||||
|
lambda: Deferred.fromCoroutine(reconfigure(
|
||||||
reactor,
|
reactor,
|
||||||
request,
|
request,
|
||||||
alice,
|
alice,
|
||||||
(1, 1, 1),
|
(1, 1, 1),
|
||||||
None,
|
None,
|
||||||
max_segment_size=segment_size
|
max_segment_size=segment_size
|
||||||
|
))
|
||||||
)
|
)
|
||||||
|
|
||||||
# 1. Upload file 1 with default segment size set to 1MB
|
# 1. Upload file 1 with default segment size set to 1MB
|
||||||
await set_segment_size(1024 * 1024)
|
set_segment_size(1024 * 1024)
|
||||||
cli(alice, "put", str(tempfile), "getput:seg1024kb")
|
cli(alice, "put", str(tempfile), "getput:seg1024kb")
|
||||||
|
|
||||||
# 2. Download file 1 with default segment size set to 128KB
|
# 2. Download file 1 with default segment size set to 128KB
|
||||||
await set_segment_size(128 * 1024)
|
set_segment_size(128 * 1024)
|
||||||
assert large_data == check_output(
|
assert large_data == check_output(
|
||||||
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
|
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
|
||||||
)
|
)
|
||||||
@ -124,7 +128,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
|
|||||||
cli(alice, "put", str(tempfile), "getput:seg128kb")
|
cli(alice, "put", str(tempfile), "getput:seg128kb")
|
||||||
|
|
||||||
# 4. Download file 2 with default segment size set to 1MB
|
# 4. Download file 2 with default segment size set to 1MB
|
||||||
await set_segment_size(1024 * 1024)
|
set_segment_size(1024 * 1024)
|
||||||
assert large_data == check_output(
|
assert large_data == check_output(
|
||||||
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
|
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
|
||||||
)
|
)
|
||||||
|
@ -14,6 +14,8 @@ from __future__ import annotations
|
|||||||
import time
|
import time
|
||||||
from urllib.parse import unquote as url_unquote, quote as url_quote
|
from urllib.parse import unquote as url_unquote, quote as url_quote
|
||||||
|
|
||||||
|
from twisted.internet.threads import deferToThread
|
||||||
|
|
||||||
import allmydata.uri
|
import allmydata.uri
|
||||||
from allmydata.util import jsonbytes as json
|
from allmydata.util import jsonbytes as json
|
||||||
|
|
||||||
@ -24,7 +26,7 @@ import requests
|
|||||||
import html5lib
|
import html5lib
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from pytest_twisted import ensureDeferred
|
import pytest_twisted
|
||||||
|
|
||||||
@run_in_thread
|
@run_in_thread
|
||||||
def test_index(alice):
|
def test_index(alice):
|
||||||
@ -185,7 +187,7 @@ def test_deep_stats(alice):
|
|||||||
time.sleep(.5)
|
time.sleep(.5)
|
||||||
|
|
||||||
|
|
||||||
@util.run_in_thread
|
@run_in_thread
|
||||||
def test_status(alice):
|
def test_status(alice):
|
||||||
"""
|
"""
|
||||||
confirm we get something sensible from /status and the various sub-types
|
confirm we get something sensible from /status and the various sub-types
|
||||||
@ -251,7 +253,7 @@ def test_status(alice):
|
|||||||
assert found_download, "Failed to find the file we downloaded in the status-page"
|
assert found_download, "Failed to find the file we downloaded in the status-page"
|
||||||
|
|
||||||
|
|
||||||
@ensureDeferred
|
@pytest_twisted.ensureDeferred
|
||||||
async def test_directory_deep_check(reactor, request, alice):
|
async def test_directory_deep_check(reactor, request, alice):
|
||||||
"""
|
"""
|
||||||
use deep-check and confirm the result pages work
|
use deep-check and confirm the result pages work
|
||||||
@ -263,7 +265,10 @@ async def test_directory_deep_check(reactor, request, alice):
|
|||||||
total = 4
|
total = 4
|
||||||
|
|
||||||
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
|
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
|
||||||
|
await deferToThread(_test_directory_deep_check_blocking, alice)
|
||||||
|
|
||||||
|
|
||||||
|
def _test_directory_deep_check_blocking(alice):
|
||||||
# create a directory
|
# create a directory
|
||||||
resp = requests.post(
|
resp = requests.post(
|
||||||
util.node_url(alice.node_dir, u"uri"),
|
util.node_url(alice.node_dir, u"uri"),
|
||||||
|
0
newsfragments/4023.minor
Normal file
0
newsfragments/4023.minor
Normal file
0
newsfragments/4028.minor
Normal file
0
newsfragments/4028.minor
Normal file
@ -310,6 +310,7 @@ class StorageClient(object):
|
|||||||
_base_url: DecodedURL
|
_base_url: DecodedURL
|
||||||
_swissnum: bytes
|
_swissnum: bytes
|
||||||
_treq: Union[treq, StubTreq, HTTPClient]
|
_treq: Union[treq, StubTreq, HTTPClient]
|
||||||
|
_pool: Optional[HTTPConnectionPool]
|
||||||
_clock: IReactorTime
|
_clock: IReactorTime
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -325,7 +326,7 @@ class StorageClient(object):
|
|||||||
certificate_hash = nurl.user.encode("ascii")
|
certificate_hash = nurl.user.encode("ascii")
|
||||||
if pool is None:
|
if pool is None:
|
||||||
pool = HTTPConnectionPool(reactor)
|
pool = HTTPConnectionPool(reactor)
|
||||||
pool.maxPersistentPerHost = 20
|
pool.maxPersistentPerHost = 10
|
||||||
|
|
||||||
if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
|
if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
|
||||||
cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
|
cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
|
||||||
@ -339,7 +340,7 @@ class StorageClient(object):
|
|||||||
)
|
)
|
||||||
|
|
||||||
https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
|
https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
|
||||||
return cls(https_url, swissnum, treq_client, reactor)
|
return cls(https_url, swissnum, treq_client, pool, reactor)
|
||||||
|
|
||||||
def relative_url(self, path: str) -> DecodedURL:
|
def relative_url(self, path: str) -> DecodedURL:
|
||||||
"""Get a URL relative to the base URL."""
|
"""Get a URL relative to the base URL."""
|
||||||
@ -479,6 +480,11 @@ class StorageClient(object):
|
|||||||
).read()
|
).read()
|
||||||
raise ClientException(response.code, response.phrase, data)
|
raise ClientException(response.code, response.phrase, data)
|
||||||
|
|
||||||
|
def shutdown(self) -> Deferred:
|
||||||
|
"""Shutdown any connections."""
|
||||||
|
if self._pool is not None:
|
||||||
|
return self._pool.closeCachedConnections()
|
||||||
|
|
||||||
|
|
||||||
@define(hash=True)
|
@define(hash=True)
|
||||||
class StorageClientGeneral(object):
|
class StorageClientGeneral(object):
|
||||||
|
@ -1271,6 +1271,11 @@ class HTTPNativeStorageServer(service.MultiService):
|
|||||||
if self._lc.running:
|
if self._lc.running:
|
||||||
self._lc.stop()
|
self._lc.stop()
|
||||||
self._failed_to_connect("shut down")
|
self._failed_to_connect("shut down")
|
||||||
|
|
||||||
|
if self._istorage_server is not None:
|
||||||
|
client_shutting_down = self._istorage_server._http_client.shutdown()
|
||||||
|
result.addCallback(lambda _: client_shutting_down)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@ -335,6 +335,7 @@ class CustomHTTPServerTests(SyncTestCase):
|
|||||||
DecodedURL.from_text("http://127.0.0.1"),
|
DecodedURL.from_text("http://127.0.0.1"),
|
||||||
SWISSNUM_FOR_TEST,
|
SWISSNUM_FOR_TEST,
|
||||||
treq=treq,
|
treq=treq,
|
||||||
|
pool=None,
|
||||||
# We're using a Treq private API to get the reactor, alas, but only
|
# We're using a Treq private API to get the reactor, alas, but only
|
||||||
# in a test, so not going to worry about it too much. This would be
|
# in a test, so not going to worry about it too much. This would be
|
||||||
# fixed if https://github.com/twisted/treq/issues/226 were ever
|
# fixed if https://github.com/twisted/treq/issues/226 were ever
|
||||||
@ -555,6 +556,7 @@ class HttpTestFixture(Fixture):
|
|||||||
DecodedURL.from_text("http://127.0.0.1"),
|
DecodedURL.from_text("http://127.0.0.1"),
|
||||||
SWISSNUM_FOR_TEST,
|
SWISSNUM_FOR_TEST,
|
||||||
treq=self.treq,
|
treq=self.treq,
|
||||||
|
pool=None,
|
||||||
clock=self.clock,
|
clock=self.clock,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -667,6 +669,7 @@ class GenericHTTPAPITests(SyncTestCase):
|
|||||||
DecodedURL.from_text("http://127.0.0.1"),
|
DecodedURL.from_text("http://127.0.0.1"),
|
||||||
b"something wrong",
|
b"something wrong",
|
||||||
treq=StubTreq(self.http.http_server.get_resource()),
|
treq=StubTreq(self.http.http_server.get_resource()),
|
||||||
|
pool=None,
|
||||||
clock=self.http.clock,
|
clock=self.http.clock,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user