From 492bcbbd128b5c3a3d4c4c9f9898e9ddac522713 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 13 Aug 2021 18:22:10 +0100 Subject: [PATCH 001/916] Refactored test_logs to be consistent with base testcases Signed-off-by: fenn-cs --- newsfragments/3758.other | 1 + src/allmydata/test/web/test_logs.py | 27 ++++++++++++++++----------- src/allmydata/util/eliotutil.py | 9 +++++++-- 3 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3758.other diff --git a/newsfragments/3758.other b/newsfragments/3758.other new file mode 100644 index 000000000..d0eb1d4c1 --- /dev/null +++ b/newsfragments/3758.other @@ -0,0 +1 @@ +Refactored test_logs, test_grid and test_root in web tests to use custom base test cases diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 89ec7ba42..043541690 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -17,10 +17,8 @@ if PY2: import json -from twisted.trial import unittest from twisted.internet.defer import inlineCallbacks -from eliot import log_call from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper @@ -48,6 +46,7 @@ from .matchers import ( from ..common import ( SyncTestCase, + AsyncTestCase, ) from ...web.logs import ( @@ -55,6 +54,10 @@ from ...web.logs import ( TokenAuthenticatedWebSocketServerProtocol, ) +from ...util.eliotutil import ( + log_call_deferred +) + class StreamingEliotLogsTests(SyncTestCase): """ Tests for the log streaming resources created by ``create_log_resources``. @@ -75,18 +78,20 @@ class StreamingEliotLogsTests(SyncTestCase): ) -class TestStreamingLogs(unittest.TestCase): +class TestStreamingLogs(AsyncTestCase): """ Test websocket streaming of logs """ def setUp(self): + super(TestStreamingLogs, self).setUp() self.reactor = MemoryReactorClockResolver() self.pumper = create_pumper() self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) return self.pumper.start() def tearDown(self): + super(TestStreamingLogs, self).tearDown() return self.pumper.stop() @inlineCallbacks @@ -105,7 +110,7 @@ class TestStreamingLogs(unittest.TestCase): messages.append(json.loads(msg)) proto.on("message", got_message) - @log_call(action_type=u"test:cli:some-exciting-action") + @log_call_deferred(action_type=u"test:cli:some-exciting-action") def do_a_thing(arguments): pass @@ -114,10 +119,10 @@ class TestStreamingLogs(unittest.TestCase): proto.transport.loseConnection() yield proto.is_closed - self.assertEqual(len(messages), 2) - self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action") - self.assertEqual(messages[0]["arguments"], - ["hello", "good-\\xff-day", 123, {"a": 35}, [None]]) - self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action") - self.assertEqual("started", messages[0]["action_status"]) - self.assertEqual("succeeded", messages[1]["action_status"]) + self.assertThat(len(messages), Equals(3)) + self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) + self.assertThat(messages[0]["arguments"], + Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) + self.assertThat(messages[1]["action_type"], Equals("test:cli:some-exciting-action")) + self.assertThat("started", Equals(messages[0]["action_status"])) + self.assertThat("succeeded", Equals(messages[1]["action_status"])) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 4e48fbb9f..ec4c0bf97 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,7 +87,11 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import AnyBytesJSONEncoder +from .jsonbytes import ( + AnyBytesJSONEncoder, + bytes_to_unicode +) + def validateInstanceOf(t): @@ -320,7 +324,8 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - with start_action(action_type=action_type).context(): + args = bytes_to_unicode(True, kw['arguments']) + with start_action(action_type=action_type, arguments=args).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From 27c8e62cf648a1186a91e84aa0cd84e62774c5e9 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 14 Aug 2021 00:09:34 +0100 Subject: [PATCH 002/916] Replaced fixed arg with dynamic args in log_call_deferred Signed-off-by: fenn-cs --- src/allmydata/util/eliotutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ec4c0bf97..d989c9e2a 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -324,8 +324,8 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - args = bytes_to_unicode(True, kw['arguments']) - with start_action(action_type=action_type, arguments=args).context(): + args = {k: bytes_to_unicode(True, kw[k]) for k in kw} + with start_action(action_type=action_type, **args).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From f7f08c93f9088b187bbef8de225c0e8352a6cd36 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 16 Aug 2021 12:57:24 +0100 Subject: [PATCH 003/916] Refactored test_root to be consistent with base testcases Signed-off-by: fenn-cs --- src/allmydata/test/web/test_root.py | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index ca3cc695d..1d5e45ba4 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -20,10 +20,11 @@ from bs4 import ( BeautifulSoup, ) -from twisted.trial import unittest from twisted.web.template import Tag from twisted.web.test.requesthelper import DummyRequest from twisted.application import service +from testtools.twistedsupport import succeeded +from twisted.internet.defer import inlineCallbacks from ...storage_client import ( NativeStorageServer, @@ -44,7 +45,17 @@ from ..common import ( EMPTY_CLIENT_CONFIG, ) -class RenderSlashUri(unittest.TestCase): +from ..common import ( + SyncTestCase, +) + +from testtools.matchers import ( + Equals, + Contains, + AfterPreprocessing, +) + +class RenderSlashUri(SyncTestCase): """ Ensure that URIs starting with /uri?uri= only accept valid capabilities @@ -53,7 +64,9 @@ class RenderSlashUri(unittest.TestCase): def setUp(self): self.client = object() self.res = URIHandler(self.client) + super(RenderSlashUri, self).setUp() + @inlineCallbacks def test_valid_query_redirect(self): """ A syntactically valid capability given in the ``uri`` query argument @@ -64,9 +77,7 @@ class RenderSlashUri(unittest.TestCase): b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882" ) query_args = {b"uri": [cap]} - response_body = self.successResultOf( - render(self.res, query_args), - ) + response_body = yield render(self.res, query_args) soup = BeautifulSoup(response_body, 'html5lib') tag = assert_soup_has_tag_with_attributes( self, @@ -74,9 +85,9 @@ class RenderSlashUri(unittest.TestCase): u"meta", {u"http-equiv": "refresh"}, ) - self.assertIn( - quote(cap, safe=""), + self.assertThat( tag.attrs.get(u"content"), + Contains(quote(cap, safe="")), ) def test_invalid(self): @@ -84,16 +95,14 @@ class RenderSlashUri(unittest.TestCase): A syntactically invalid capbility results in an error. """ query_args = {b"uri": [b"not a capability"]} - response_body = self.successResultOf( - render(self.res, query_args), - ) - self.assertEqual( + response_body = render(self.res, query_args) + self.assertThat( response_body, - b"Invalid capability", + succeeded(AfterPreprocessing(bytes, Equals(b"Invalid capability"))), ) -class RenderServiceRow(unittest.TestCase): +class RenderServiceRow(SyncTestCase): def test_missing(self): """ minimally-defined static servers just need anonymous-storage-FURL @@ -127,5 +136,5 @@ class RenderServiceRow(unittest.TestCase): # Coerce `items` to list and pick the first item from it. item = list(items)[0] - self.assertEqual(item.slotData.get("version"), "") - self.assertEqual(item.slotData.get("nickname"), "") + self.assertThat(item.slotData.get("version"), Equals("")) + self.assertThat(item.slotData.get("nickname"), Equals("")) From bef2413e4b9bfbfe3553be3b56c9d3a57cf4f623 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 17 Aug 2021 13:11:54 +0100 Subject: [PATCH 004/916] Refactored test_grid to be consistent with base testcases Signed-off-by: fenn-cs --- src/allmydata/test/web/test_grid.py | 204 +++++++++++++++------------- 1 file changed, 107 insertions(+), 97 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index edcf32268..54aa13941 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -18,7 +18,6 @@ from six.moves import StringIO from bs4 import BeautifulSoup from twisted.web import resource -from twisted.trial import unittest from allmydata import uri, dirnode from allmydata.util import base32 from allmydata.util.encodingutil import to_bytes @@ -43,6 +42,20 @@ from .common import ( unknown_rwcap, ) +from ..common import ( + AsyncTestCase, +) + +from testtools.matchers import ( + Equals, + Contains, + Is, + Not, +) + +from testtools.twistedsupport import flush_logged_errors + + DIR_HTML_TAG = '' class CompletelyUnhandledError(Exception): @@ -53,7 +66,7 @@ class ErrorBoom(resource.Resource, object): def render(self, req): raise CompletelyUnhandledError("whoops") -class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): +class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, AsyncTestCase): def CHECK(self, ign, which, args, clientnum=0): fileurl = self.fileurls[which] @@ -117,37 +130,37 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "good", "t=check") def _got_html_good(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy", ))) soup = BeautifulSoup(res, 'html5lib') assert_soup_has_favicon(self, soup) d.addCallback(_got_html_good) d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere") def _got_html_good_return_to(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn('Return to file', res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains('Return to file')) d.addCallback(_got_html_good_return_to) d.addCallback(self.CHECK, "good", "t=check&output=json") def _got_json_good(res): r = json.loads(res) self.failUnlessEqual(r["summary"], "Healthy") self.failUnless(r["results"]["healthy"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"], Not(Contains("needs-rebalancing",))) self.failUnless(r["results"]["recoverable"]) d.addCallback(_got_json_good) d.addCallback(self.CHECK, "small", "t=check") def _got_html_small(res): - self.failUnlessIn("Literal files are always healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Literal files are always healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) d.addCallback(_got_html_small) d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere") def _got_html_small_return_to(res): - self.failUnlessIn("Literal files are always healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn('Return to file', res) + self.assertThat(res, Contains("Literal files are always healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains('Return to file')) d.addCallback(_got_html_small_return_to) d.addCallback(self.CHECK, "small", "t=check&output=json") def _got_json_small(res): @@ -158,8 +171,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "smalldir", "t=check") def _got_html_smalldir(res): - self.failUnlessIn("Literal files are always healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Literal files are always healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) d.addCallback(_got_html_smalldir) d.addCallback(self.CHECK, "smalldir", "t=check&output=json") def _got_json_smalldir(res): @@ -170,43 +183,43 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "sick", "t=check") def _got_html_sick(res): - self.failUnlessIn("Not Healthy", res) + self.assertThat(res, Contains("Not Healthy")) d.addCallback(_got_html_sick) d.addCallback(self.CHECK, "sick", "t=check&output=json") def _got_json_sick(res): r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 9 shares (enc 3-of-10)") - self.failIf(r["results"]["healthy"]) + self.assertThat(r["results"]["healthy"], Is(False)) self.failUnless(r["results"]["recoverable"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_sick) d.addCallback(self.CHECK, "dead", "t=check") def _got_html_dead(res): - self.failUnlessIn("Not Healthy", res) + self.assertThat(res, Contains("Not Healthy")) d.addCallback(_got_html_dead) d.addCallback(self.CHECK, "dead", "t=check&output=json") def _got_json_dead(res): r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 1 shares (enc 3-of-10)") - self.failIf(r["results"]["healthy"]) - self.failIf(r["results"]["recoverable"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"]["healthy"], Is(False)) + self.assertThat(r["results"]["recoverable"], Is(False)) + self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_dead) d.addCallback(self.CHECK, "corrupt", "t=check&verify=true") def _got_html_corrupt(res): - self.failUnlessIn("Not Healthy! : Unhealthy", res) + self.assertThat(res, Contains("Not Healthy! : Unhealthy")) d.addCallback(_got_html_corrupt) d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json") def _got_json_corrupt(res): r = json.loads(res) - self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"]) - self.failIf(r["results"]["healthy"]) + self.assertThat(r["summary"], Contains("Unhealthy: 9 shares (enc 3-of-10)")) + self.assertThat(r["results"]["healthy"], Is(False)) self.failUnless(r["results"]["recoverable"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) self.failUnlessReallyEqual(r["results"]["count-happiness"], 9) self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9) self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1) @@ -261,9 +274,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "good", "t=check&repair=true") def _got_html_good(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn("No repair necessary", res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains("No repair necessary", )) soup = BeautifulSoup(res, 'html5lib') assert_soup_has_favicon(self, soup) @@ -271,9 +284,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "sick", "t=check&repair=true") def _got_html_sick(res): - self.failUnlessIn("Healthy : healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn("Repair successful", res) + self.assertThat(res, Contains("Healthy : healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains("Repair successful")) d.addCallback(_got_html_sick) # repair of a dead file will fail, of course, but it isn't yet @@ -290,9 +303,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true") def _got_html_corrupt(res): - self.failUnlessIn("Healthy : Healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn("Repair successful", res) + self.assertThat(res, Contains("Healthy : Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains("Repair successful")) d.addCallback(_got_html_corrupt) d.addErrback(self.explain_web_error) @@ -392,31 +405,31 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if expect_rw_uri: self.failUnlessReallyEqual(to_bytes(f[1]["rw_uri"]), unknown_rwcap, data) else: - self.failIfIn("rw_uri", f[1]) + self.assertThat(f[1], Not(Contains("rw_uri"))) if immutable: self.failUnlessReallyEqual(to_bytes(f[1]["ro_uri"]), unknown_immcap, data) else: self.failUnlessReallyEqual(to_bytes(f[1]["ro_uri"]), unknown_rocap, data) - self.failUnlessIn("metadata", f[1]) + self.assertThat(f[1], Contains("metadata")) d.addCallback(_check_directory_json, expect_rw_uri=not immutable) def _check_info(res, expect_rw_uri, expect_ro_uri): if expect_rw_uri: - self.failUnlessIn(unknown_rwcap, res) + self.assertThat(res, Contains(unknown_rwcap)) if expect_ro_uri: if immutable: - self.failUnlessIn(unknown_immcap, res) + self.assertThat(res, Contains(unknown_immcap)) else: - self.failUnlessIn(unknown_rocap, res) + self.assertThat(res, Contains(unknown_rocap)) else: - self.failIfIn(unknown_rocap, res) + self.assertThat(res, Not(Contains(unknown_rocap))) res = str(res, "utf-8") - self.failUnlessIn("Object Type: unknown", res) - self.failIfIn("Raw data as", res) - self.failIfIn("Directory writecap", res) - self.failIfIn("Checker Operations", res) - self.failIfIn("Mutable File Operations", res) - self.failIfIn("Directory Operations", res) + self.assertThat(res, Contains("Object Type: unknown")) + self.assertThat(res, Not(Contains("Raw data as"))) + self.assertThat(res, Not(Contains("Directory writecap"))) + self.assertThat(res, Not(Contains("Checker Operations"))) + self.assertThat(res, Not(Contains("Mutable File Operations"))) + self.assertThat(res, Not(Contains("Directory Operations"))) # FIXME: these should have expect_rw_uri=not immutable; I don't know # why they fail. Possibly related to ticket #922. @@ -432,7 +445,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if expect_rw_uri: self.failUnlessReallyEqual(to_bytes(data[1]["rw_uri"]), unknown_rwcap, data) else: - self.failIfIn("rw_uri", data[1]) + self.assertThat(data[1], Not(Contains("rw_uri"))) if immutable: self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_immcap, data) @@ -442,10 +455,10 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessReallyEqual(data[1]["mutable"], True) else: self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_rocap, data) - self.failIfIn("mutable", data[1]) + self.assertThat(data[1], Not(Contains("mutable"))) # TODO: check metadata contents - self.failUnlessIn("metadata", data[1]) + self.assertThat(data[1], Contains("metadata")) d.addCallback(lambda ign: self.GET("%s/%s?t=json" % (self.rooturl, str(name)))) d.addCallback(_check_json, expect_rw_uri=not immutable) @@ -519,14 +532,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) - self.failIf(dn.is_mutable()) + self.assertThat(dn.is_mutable(), Is(False)) self.failUnless(dn.is_readonly()) # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail. - self.failIf(hasattr(dn._node, 'get_writekey')) + self.assertThat(hasattr(dn._node, 'get_writekey'), Is(False)) rep = str(dn) - self.failUnlessIn("RO-IMM", rep) + self.assertThat(rep, Contains("RO-IMM")) cap = dn.get_cap() - self.failUnlessIn(b"CHK", cap.to_string()) + self.assertThat(cap.to_string(), Contains(b"CHK")) self.cap = cap self.rootnode = dn self.rooturl = "uri/" + url_quote(dn.get_uri()) @@ -546,7 +559,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) name = name_utf8.decode("utf-8") self.failUnlessEqual(rwcapdata, b"") - self.failUnlessIn(name, kids) + self.assertThat(kids, Contains(name)) (expected_child, ign) = kids[name] self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri()) numkids += 1 @@ -572,27 +585,27 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.GET(self.rooturl)) def _check_html(res): soup = BeautifulSoup(res, 'html5lib') - self.failIfIn(b"URI:SSK", res) + self.assertThat(res, Not(Contains(b"URI:SSK"))) found = False for td in soup.find_all(u"td"): if td.text != u"FILE": continue a = td.findNextSibling()(u"a")[0] - self.assertIn(url_quote(lonely_uri), a[u"href"]) - self.assertEqual(u"lonely", a.text) - self.assertEqual(a[u"rel"], [u"noreferrer"]) - self.assertEqual(u"{}".format(len("one")), td.findNextSibling().findNextSibling().text) + self.assertThat(a[u"href"], Contains(url_quote(lonely_uri))) + self.assertThat(a.text, Equals(u"lonely")) + self.assertThat(a[u"rel"], Equals([u"noreferrer"])) + self.assertThat(td.findNextSibling().findNextSibling().text, Equals(u"{}".format(len("one")))) found = True break - self.assertTrue(found) + self.assertThat(found, Is(True)) infos = list( a[u"href"] for a in soup.find_all(u"a") if a.text == u"More Info" ) - self.assertEqual(1, len(infos)) - self.assertTrue(infos[0].endswith(url_quote(lonely_uri) + "?t=info")) + self.assertThat(len(infos), Equals(1)) + self.assertThat(infos[0].endswith(url_quote(lonely_uri) + "?t=info"), Is(True)) d.addCallback(_check_html) # ... and in JSON. @@ -604,7 +617,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"]) ll_type, ll_data = listed_children[u"lonely"] self.failUnlessEqual(ll_type, "filenode") - self.failIfIn("rw_uri", ll_data) + self.assertThat(ll_data, Not(Contains("rw_uri"))) self.failUnlessReallyEqual(to_bytes(ll_data["ro_uri"]), lonely_uri) d.addCallback(_check_json) return d @@ -744,8 +757,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi error_line = lines[first_error] error_msg = lines[first_error+1:] error_msg_s = "\n".join(error_msg) + "\n" - self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)", - error_line) + self.assertThat(error_line, Contains("ERROR: UnrecoverableFileError(no recoverable versions)")) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [json.loads(line) for line in lines[:first_error]] self.failUnlessReallyEqual(len(units), 6) # includes subdir @@ -765,8 +777,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi error_line = lines[first_error] error_msg = lines[first_error+1:] error_msg_s = "\n".join(error_msg) + "\n" - self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)", - error_line) + self.assertThat(error_line, Contains("ERROR: UnrecoverableFileError(no recoverable versions)")) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [json.loads(line) for line in lines[:first_error]] self.failUnlessReallyEqual(len(units), 6) # includes subdir @@ -936,8 +947,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "one", "t=check") # no add-lease def _got_html_good(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) d.addCallback(_got_html_good) d.addCallback(self._count_leases, "one") @@ -1111,7 +1122,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["0shares"])) def _check_zero_shares(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) body = " ".join(body.strip().split()) exp = ("NoSharesError: no shares could be found. " "Zero shares usually indicates a corrupt URI, or that " @@ -1129,7 +1140,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["1share"])) def _check_one_share(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) body = " ".join(body.strip().split()) msgbase = ("NotEnoughSharesError: This indicates that some " "servers were unavailable, or that shares have been " @@ -1154,17 +1165,16 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["imaginary"])) def _missing_child(body): body = str(body, "utf-8") - self.failUnlessIn("No such child: imaginary", body) + self.assertThat(body, Contains("No such child: imaginary")) d.addCallback(_missing_child) d.addCallback(lambda ignored: self.GET_unicode(self.fileurls["dir-0share"])) def _check_0shares_dir_html(body): - self.failUnlessIn(DIR_HTML_TAG, body) + self.assertThat(body, Contains(DIR_HTML_TAG)) # we should see the regular page, but without the child table or # the dirops forms body = " ".join(body.strip().split()) - self.failUnlessIn('href="?t=info">More info on this directory', - body) + self.assertThat(body, Contains('href="?t=info">More info on this directory')) exp = ("UnrecoverableFileError: the directory (or mutable file) " "could not be retrieved, because there were insufficient " "good shares. This might indicate that no servers were " @@ -1172,8 +1182,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "was corrupt, or that shares have been lost due to server " "departure, hard drive failure, or disk corruption. You " "should perform a filecheck on this object to learn more.") - self.failUnlessIn(exp, body) - self.failUnlessIn("No upload forms: directory is unreadable", body) + self.assertThat(body, Contains(exp)) + self.assertThat(body, Contains("No upload forms: directory is unreadable")) d.addCallback(_check_0shares_dir_html) d.addCallback(lambda ignored: self.GET_unicode(self.fileurls["dir-1share"])) @@ -1182,10 +1192,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # and some-shares like we did for immutable files (since there # are different sorts of advice to offer in each case). For now, # they present the same way. - self.failUnlessIn(DIR_HTML_TAG, body) + self.assertThat(body, Contains(DIR_HTML_TAG)) body = " ".join(body.strip().split()) - self.failUnlessIn('href="?t=info">More info on this directory', - body) + self.assertThat(body, Contains('href="?t=info">More info on this directory')) exp = ("UnrecoverableFileError: the directory (or mutable file) " "could not be retrieved, because there were insufficient " "good shares. This might indicate that no servers were " @@ -1193,8 +1202,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "was corrupt, or that shares have been lost due to server " "departure, hard drive failure, or disk corruption. You " "should perform a filecheck on this object to learn more.") - self.failUnlessIn(exp, body) - self.failUnlessIn("No upload forms: directory is unreadable", body) + self.assertThat(body, Contains(exp)) + self.assertThat(body, Contains("No upload forms: directory is unreadable")) d.addCallback(_check_1shares_dir_html) d.addCallback(lambda ignored: @@ -1204,7 +1213,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.fileurls["dir-0share-json"])) def _check_unrecoverable_file(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) body = " ".join(body.strip().split()) exp = ("UnrecoverableFileError: the directory (or mutable file) " "could not be retrieved, because there were insufficient " @@ -1213,7 +1222,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "was corrupt, or that shares have been lost due to server " "departure, hard drive failure, or disk corruption. You " "should perform a filecheck on this object to learn more.") - self.failUnlessIn(exp, body) + self.assertThat(body, Contains(exp)) d.addCallback(_check_unrecoverable_file) d.addCallback(lambda ignored: @@ -1245,7 +1254,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi headers={"accept": "*/*"})) def _internal_error_html1(body): body = str(body, "utf-8") - self.failUnlessIn("", "expected HTML, not '%s'" % body) + self.assertThat("expected HTML, not '%s'" % body, Contains("")) d.addCallback(_internal_error_html1) d.addCallback(lambda ignored: @@ -1255,8 +1264,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi headers={"accept": "text/plain"})) def _internal_error_text2(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) self.failUnless(body.startswith("Traceback "), body) + d.addCallback(_internal_error_text2) CLI_accepts = "text/plain, application/octet-stream" @@ -1267,7 +1277,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi headers={"accept": CLI_accepts})) def _internal_error_text3(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) self.failUnless(body.startswith("Traceback "), body) d.addCallback(_internal_error_text3) @@ -1276,12 +1286,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 500, "Internal Server Error", None, self.GET, "ERRORBOOM")) def _internal_error_html4(body): - self.failUnlessIn(b"", body) + self.assertThat(body, Contains(b"")) d.addCallback(_internal_error_html4) def _flush_errors(res): # Trial: please ignore the CompletelyUnhandledError in the logs - self.flushLoggedErrors(CompletelyUnhandledError) + flush_logged_errors(CompletelyUnhandledError) return res d.addBoth(_flush_errors) @@ -1312,8 +1322,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_dir) d.addCallback(lambda ign: self.GET_unicode(self.dir_url, followRedirect=True)) def _check_dir_html(body): - self.failUnlessIn(DIR_HTML_TAG, body) - self.failUnlessIn("blacklisted.txt", body) + self.assertThat(body, Contains(DIR_HTML_TAG)) + self.assertThat(body, Contains("blacklisted.txt")) d.addCallback(_check_dir_html) d.addCallback(lambda ign: self.GET(self.url)) d.addCallback(lambda body: self.failUnlessEqual(DATA, body)) @@ -1336,8 +1346,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # We should still be able to list the parent directory, in HTML... d.addCallback(lambda ign: self.GET_unicode(self.dir_url, followRedirect=True)) def _check_dir_html2(body): - self.failUnlessIn(DIR_HTML_TAG, body) - self.failUnlessIn("blacklisted.txt", body) + self.assertThat(body, Contains(DIR_HTML_TAG)) + self.assertThat(body, Contains("blacklisted.txt")) d.addCallback(_check_dir_html2) # ... and in JSON (used by CLI). @@ -1347,8 +1357,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnless(isinstance(data, list), data) self.failUnlessEqual(data[0], "dirnode") self.failUnless(isinstance(data[1], dict), data) - self.failUnlessIn("children", data[1]) - self.failUnlessIn("blacklisted.txt", data[1]["children"]) + self.assertThat(data[1], Contains("children")) + self.assertThat(data[1]["children"], Contains("blacklisted.txt")) childdata = data[1]["children"]["blacklisted.txt"] self.failUnless(isinstance(childdata, list), data) self.failUnlessEqual(childdata[0], "filenode") @@ -1387,7 +1397,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.child_url = b"uri/"+dn.get_readonly_uri()+b"/child" d.addCallback(_get_dircap) d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True)) - d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, str(body, "utf-8"))) + d.addCallback(lambda body: self.assertThat(str(body, "utf-8"), Contains(DIR_HTML_TAG))) d.addCallback(lambda ign: self.GET(self.dir_url_json1)) d.addCallback(lambda res: json.loads(res)) # just check it decodes d.addCallback(lambda ign: self.GET(self.dir_url_json2)) From b4cdf7f96915943be85ee2b48c6954a1b2c128e7 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 8 Sep 2021 00:08:37 +0100 Subject: [PATCH 005/916] changed fragment to minor, improved test_grid.py refactor Signed-off-by: fenn-cs --- newsfragments/3758.minor | 0 newsfragments/3758.other | 1 - src/allmydata/test/web/test_grid.py | 21 +++++++++++---------- 3 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 newsfragments/3758.minor delete mode 100644 newsfragments/3758.other diff --git a/newsfragments/3758.minor b/newsfragments/3758.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3758.other b/newsfragments/3758.other deleted file mode 100644 index d0eb1d4c1..000000000 --- a/newsfragments/3758.other +++ /dev/null @@ -1 +0,0 @@ -Refactored test_logs, test_grid and test_root in web tests to use custom base test cases diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 54aa13941..1ebe3a90f 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -49,8 +49,9 @@ from ..common import ( from testtools.matchers import ( Equals, Contains, - Is, Not, + HasLength, + EndsWith, ) from testtools.twistedsupport import flush_logged_errors @@ -190,7 +191,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 9 shares (enc 3-of-10)") - self.assertThat(r["results"]["healthy"], Is(False)) + self.assertThat(r["results"]["healthy"], Equals(False)) self.failUnless(r["results"]["recoverable"]) self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_sick) @@ -204,8 +205,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 1 shares (enc 3-of-10)") - self.assertThat(r["results"]["healthy"], Is(False)) - self.assertThat(r["results"]["recoverable"], Is(False)) + self.assertThat(r["results"]["healthy"], Equals(False)) + self.assertThat(r["results"]["recoverable"], Equals(False)) self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_dead) @@ -217,7 +218,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _got_json_corrupt(res): r = json.loads(res) self.assertThat(r["summary"], Contains("Unhealthy: 9 shares (enc 3-of-10)")) - self.assertThat(r["results"]["healthy"], Is(False)) + self.assertThat(r["results"]["healthy"], Equals(False)) self.failUnless(r["results"]["recoverable"]) self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) self.failUnlessReallyEqual(r["results"]["count-happiness"], 9) @@ -532,10 +533,10 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) - self.assertThat(dn.is_mutable(), Is(False)) + self.assertThat(dn.is_mutable(), Equals(False)) self.failUnless(dn.is_readonly()) # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail. - self.assertThat(hasattr(dn._node, 'get_writekey'), Is(False)) + self.assertThat(hasattr(dn._node, 'get_writekey'), Equals(False)) rep = str(dn) self.assertThat(rep, Contains("RO-IMM")) cap = dn.get_cap() @@ -597,15 +598,15 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.assertThat(td.findNextSibling().findNextSibling().text, Equals(u"{}".format(len("one")))) found = True break - self.assertThat(found, Is(True)) + self.assertThat(found, Equals(True)) infos = list( a[u"href"] for a in soup.find_all(u"a") if a.text == u"More Info" ) - self.assertThat(len(infos), Equals(1)) - self.assertThat(infos[0].endswith(url_quote(lonely_uri) + "?t=info"), Is(True)) + self.assertThat(infos, HasLength(1)) + self.assertThat(infos[0], EndsWith(url_quote(lonely_uri) + "?t=info")) d.addCallback(_check_html) # ... and in JSON. From 55221d4532fe051e01e67c21e53507feda5f7feb Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 9 Sep 2021 01:50:21 +0100 Subject: [PATCH 006/916] replaced testools.unittest.TestCase with common base case Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_checker.py | 5 +++-- src/allmydata/test/mutable/test_datahandle.py | 6 ++++-- src/allmydata/test/mutable/test_different_encoding.py | 5 +++-- src/allmydata/test/mutable/test_exceptions.py | 5 +++-- src/allmydata/test/mutable/test_filehandle.py | 6 ++++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/mutable/test_checker.py b/src/allmydata/test/mutable/test_checker.py index 11ba776fd..6d9145d68 100644 --- a/src/allmydata/test/mutable/test_checker.py +++ b/src/allmydata/test/mutable/test_checker.py @@ -10,14 +10,15 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase from foolscap.api import flushEventualQueue from allmydata.monitor import Monitor from allmydata.mutable.common import CorruptShareError from .util import PublishMixin, corrupt, CheckerMixin -class Checker(unittest.TestCase, CheckerMixin, PublishMixin): +class Checker(AsyncTestCase, CheckerMixin, PublishMixin): def setUp(self): + super(Checker, self).setUp() return self.publish_one() diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 1819cba01..53e2983d1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -10,11 +10,13 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.publish import MutableData -class DataHandle(unittest.TestCase): + +class DataHandle(SyncTestCase): def setUp(self): + super(DataHandle, self).setUp() self.test_data = b"Test Data" * 50000 self.uploadable = MutableData(self.test_data) diff --git a/src/allmydata/test/mutable/test_different_encoding.py b/src/allmydata/test/mutable/test_different_encoding.py index a5165532c..f1796d373 100644 --- a/src/allmydata/test/mutable/test_different_encoding.py +++ b/src/allmydata/test/mutable/test_different_encoding.py @@ -10,11 +10,12 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase from .util import FakeStorage, make_nodemaker -class DifferentEncoding(unittest.TestCase): +class DifferentEncoding(AsyncTestCase): def setUp(self): + super(DifferentEncoding, self).setUp() self._storage = s = FakeStorage() self.nodemaker = make_nodemaker(s) diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index 6a9b2b575..aa2b56b86 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -11,10 +11,11 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError -class Exceptions(unittest.TestCase): + +class Exceptions(SyncTestCase): def test_repr(self): nmde = NeedMoreDataError(100, 50, 100) self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde)) diff --git a/src/allmydata/test/mutable/test_filehandle.py b/src/allmydata/test/mutable/test_filehandle.py index 8db02f3fd..795f60654 100644 --- a/src/allmydata/test/mutable/test_filehandle.py +++ b/src/allmydata/test/mutable/test_filehandle.py @@ -12,11 +12,13 @@ if PY2: import os from io import BytesIO -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.publish import MutableFileHandle -class FileHandle(unittest.TestCase): + +class FileHandle(SyncTestCase): def setUp(self): + super(FileHandle, self).setUp() self.test_data = b"Test Data" * 50000 self.sio = BytesIO(self.test_data) self.uploadable = MutableFileHandle(self.sio) From bbbc8592f09d8f7bac1f434dab6b305f6b58ea54 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 9 Sep 2021 14:41:06 +0100 Subject: [PATCH 007/916] removed deprecated methods, already refactored mutable files Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_datahandle.py | 11 +-- src/allmydata/test/mutable/test_exceptions.py | 5 +- .../test/mutable/test_interoperability.py | 9 +-- .../test/mutable/test_multiple_encodings.py | 8 ++- .../test/mutable/test_multiple_versions.py | 38 ++++++----- src/allmydata/test/mutable/test_problems.py | 19 +++--- src/allmydata/test/mutable/test_repair.py | 67 ++++++++++--------- 7 files changed, 83 insertions(+), 74 deletions(-) diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 53e2983d1..7aabcd8e1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -12,6 +12,7 @@ if PY2: from ..common import SyncTestCase from allmydata.mutable.publish import MutableData +from testtools.matchers import Equals, HasLength class DataHandle(SyncTestCase): @@ -28,13 +29,13 @@ class DataHandle(SyncTestCase): data = b"".join(data) start = i end = i + chunk_size - self.failUnlessEqual(data, self.test_data[start:end]) + self.assertThat(data, Equals(self.test_data[start:end])) def test_datahandle_get_size(self): actual_size = len(self.test_data) size = self.uploadable.get_size() - self.failUnlessEqual(size, actual_size) + self.assertThat(size, Equals(actual_size)) def test_datahandle_get_size_out_of_order(self): @@ -42,14 +43,14 @@ class DataHandle(SyncTestCase): # disturbing the location of the seek pointer. chunk_size = 100 data = self.uploadable.read(chunk_size) - self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size]) + self.assertThat(b"".join(data), Equals(self.test_data[:chunk_size])) # Now get the size. size = self.uploadable.get_size() - self.failUnlessEqual(size, len(self.test_data)) + self.assertThat(self.test_data, HasLength(size)) # Now get more data. We should be right where we left off. more_data = self.uploadable.read(chunk_size) start = chunk_size end = chunk_size * 2 - self.failUnlessEqual(b"".join(more_data), self.test_data[start:end]) + self.assertThat(b"".join(more_data), Equals(self.test_data[start:end])) diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index aa2b56b86..23674d036 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -18,6 +18,7 @@ from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError class Exceptions(SyncTestCase): def test_repr(self): nmde = NeedMoreDataError(100, 50, 100) - self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde)) + self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde)) + self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde)) ucwe = UncoordinatedWriteError() - self.failUnless("UncoordinatedWriteError" in repr(ucwe), repr(ucwe)) + self.assertTrue("UncoordinatedWriteError" in repr(ucwe), msg=repr(ucwe)) diff --git a/src/allmydata/test/mutable/test_interoperability.py b/src/allmydata/test/mutable/test_interoperability.py index 5d7414907..496da1d2a 100644 --- a/src/allmydata/test/mutable/test_interoperability.py +++ b/src/allmydata/test/mutable/test_interoperability.py @@ -11,14 +11,15 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, base64 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import HasLength from allmydata import uri from allmydata.storage.common import storage_index_to_dir from allmydata.util import fileutil from .. import common_util as testutil from ..no_network import GridTestMixin -class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Interoperability(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): sdmf_old_shares = {} sdmf_old_shares[0] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAQ/EX4eC/1+hGOQ/h4EiKUkqxdsfzdcPlDvd11SGWZ0VHsUclZChTzuBAU2zLTXm+cG8IFhO50ly6Ey/DB44NtMKVaVzO0nU8DE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" sdmf_old_shares[1] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAP7FHJWQoU87gQFNsy015vnBvCBYTudJcuhMvwweODbTD8Rfh4L/X6EY5D+HgSIpSSrF2x/N1w+UO93XVIZZnRUeePDXEwhqYDE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" @@ -53,7 +54,7 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi sharedata) # ...and verify that the shares are there. shares = self.find_uri_shares(self.sdmf_old_cap) - assert len(shares) == 10 + self.assertThat(shares, HasLength(10)) def test_new_downloader_can_read_old_shares(self): self.basedir = "mutable/Interoperability/new_downloader_can_read_old_shares" @@ -62,5 +63,5 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi nm = self.g.clients[0].nodemaker n = nm.create_from_cap(self.sdmf_old_cap) d = n.download_best_version() - d.addCallback(self.failUnlessEqual, self.sdmf_old_contents) + d.addCallback(self.assertEqual, self.sdmf_old_contents) return d diff --git a/src/allmydata/test/mutable/test_multiple_encodings.py b/src/allmydata/test/mutable/test_multiple_encodings.py index 12c5be051..2291b60d8 100644 --- a/src/allmydata/test/mutable/test_multiple_encodings.py +++ b/src/allmydata/test/mutable/test_multiple_encodings.py @@ -10,7 +10,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals from allmydata.interfaces import SDMF_VERSION from allmydata.monitor import Monitor from foolscap.logging import log @@ -20,8 +21,9 @@ from allmydata.mutable.servermap import ServerMap, ServermapUpdater from ..common_util import DevNullDictionary from .util import FakeStorage, make_nodemaker -class MultipleEncodings(unittest.TestCase): +class MultipleEncodings(AsyncTestCase): def setUp(self): + super(MultipleEncodings, self).setUp() self.CONTENTS = b"New contents go here" self.uploadable = MutableData(self.CONTENTS) self._storage = FakeStorage() @@ -159,6 +161,6 @@ class MultipleEncodings(unittest.TestCase): d.addCallback(lambda res: fn3.download_best_version()) def _retrieved(new_contents): # the current specified behavior is "first version recoverable" - self.failUnlessEqual(new_contents, contents1) + self.assertThat(new_contents, Equals(contents1)) d.addCallback(_retrieved) return d diff --git a/src/allmydata/test/mutable/test_multiple_versions.py b/src/allmydata/test/mutable/test_multiple_versions.py index 460cde4b3..c9b7e71df 100644 --- a/src/allmydata/test/mutable/test_multiple_versions.py +++ b/src/allmydata/test/mutable/test_multiple_versions.py @@ -10,15 +10,17 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength from allmydata.monitor import Monitor from allmydata.mutable.common import MODE_CHECK, MODE_READ from .util import PublishMixin, CheckerMixin -class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): +class MultipleVersions(AsyncTestCase, PublishMixin, CheckerMixin): def setUp(self): + super(MultipleVersions, self).setUp() return self.publish_multiple() def test_multiple_versions(self): @@ -26,7 +28,7 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): # should get the latest one self._set_versions(dict([(i,2) for i in (0,2,4,6,8)])) d = self._fn.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[4])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[4]))) # and the checker should report problems d.addCallback(lambda res: self._fn.check(Monitor())) d.addCallback(self.check_bad, "test_multiple_versions") @@ -35,23 +37,23 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): d.addCallback(lambda res: self._set_versions(dict([(i,2) for i in range(10)]))) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2]))) # if exactly one share is at version 3, we should still get v2 d.addCallback(lambda res: self._set_versions({0:3})) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2]))) # but the servermap should see the unrecoverable version. This # depends upon the single newer share being queried early. d.addCallback(lambda res: self._fn.get_servermap(MODE_READ)) def _check_smap(smap): - self.failUnlessEqual(len(smap.unrecoverable_versions()), 1) + self.assertThat(smap.unrecoverable_versions(), HasLength(1)) newer = smap.unrecoverable_newer_versions() - self.failUnlessEqual(len(newer), 1) + self.assertThat(newer, HasLength(1)) verinfo, health = list(newer.items())[0] - self.failUnlessEqual(verinfo[0], 4) - self.failUnlessEqual(health, (1,3)) - self.failIf(smap.needs_merge()) + self.assertThat(verinfo[0], Equals(4)) + self.assertThat(health, Equals((1,3))) + self.assertThat(smap.needs_merge(), Equals(False)) d.addCallback(_check_smap) # if we have a mix of two parallel versions (s4a and s4b), we could # recover either @@ -60,13 +62,13 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): 1:4,3:4,5:4,7:4,9:4})) d.addCallback(lambda res: self._fn.get_servermap(MODE_READ)) def _check_smap_mixed(smap): - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) newer = smap.unrecoverable_newer_versions() - self.failUnlessEqual(len(newer), 0) - self.failUnless(smap.needs_merge()) + self.assertThat(newer, HasLength(0)) + self.assertTrue(smap.needs_merge()) d.addCallback(_check_smap_mixed) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnless(res == self.CONTENTS[3] or + d.addCallback(lambda res: self.assertTrue(res == self.CONTENTS[3] or res == self.CONTENTS[4])) return d @@ -86,12 +88,12 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): d = self._fn.modify(_modify) d.addCallback(lambda res: self._fn.download_best_version()) expected = self.CONTENTS[2] + b" modified" - d.addCallback(lambda res: self.failUnlessEqual(res, expected)) + d.addCallback(lambda res: self.assertThat(res, Equals(expected))) # and the servermap should indicate that the outlier was replaced too d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(smap.highest_seqnum(), 5) - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) - self.failUnlessEqual(len(smap.recoverable_versions()), 1) + self.assertThat(smap.highest_seqnum(), Equals(5)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) + self.assertThat(smap.recoverable_versions(), HasLength(1)) d.addCallback(_check_smap) return d diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 86a367596..9abee560d 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -11,7 +11,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, base64 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import HasLength from twisted.internet import defer from foolscap.logging import log from allmydata import uri @@ -61,7 +62,7 @@ class FirstServerGetsDeleted(object): return (True, {}) return retval -class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): def do_publish_surprise(self, version): self.basedir = "mutable/Problems/test_publish_surprise_%s" % version self.set_up_grid() @@ -198,8 +199,8 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): def _overwritten_again(smap): # Make sure that all shares were updated by making sure that # there aren't any other versions in the sharemap. - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) d.addCallback(_overwritten_again) return d @@ -240,7 +241,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # that ought to work def _got_node(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) # now break the second peer def _break_peer1(res): self.g.break_server(self.server1.get_serverid()) @@ -248,7 +249,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) def _explain_error(f): print(f) if f.check(NotEnoughServersError): @@ -280,7 +281,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d = nm.create_mutable_file(MutableData(b"contents 1")) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) # now break one of the remaining servers def _break_second_server(res): self.g.break_server(peerids[1]) @@ -288,7 +289,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) return d d.addCallback(_created) return d @@ -419,7 +420,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): return self._node.download_version(servermap, ver) d.addCallback(_then) d.addCallback(lambda data: - self.failUnlessEqual(data, CONTENTS)) + self.assertTrue(data, CONTENTS)) return d def test_1654(self): diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index fb1caa974..987b21cc3 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -10,7 +10,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength from allmydata.interfaces import IRepairResults, ICheckAndRepairResults from allmydata.monitor import Monitor from allmydata.mutable.common import MODE_CHECK @@ -19,7 +20,7 @@ from allmydata.mutable.repairer import MustForceRepairError from ..common import ShouldFailMixin from .util import PublishMixin -class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): +class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin): def get_shares(self, s): all_shares = {} # maps (peerid, shnum) to share data @@ -40,8 +41,8 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda res: self._fn.check(Monitor())) d.addCallback(lambda check_results: self._fn.repair(check_results)) def _check_results(rres): - self.failUnless(IRepairResults.providedBy(rres)) - self.failUnless(rres.get_successful()) + self.assertThat(IRepairResults.providedBy(rres), Equals(True)) + self.assertThat(rres.get_successful(), Equals(True)) # TODO: examine results self.copy_shares() @@ -50,11 +51,11 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): new_shares = self.old_shares[1] # TODO: this really shouldn't change anything. When we implement # a "minimal-bandwidth" repairer", change this test to assert: - #self.failUnlessEqual(new_shares, initial_shares) + #self.assertThat(new_shares, Equals(initial_shares)) # all shares should be in the same place as before - self.failUnlessEqual(set(initial_shares.keys()), - set(new_shares.keys())) + self.assertThat(set(initial_shares.keys()), + Equals(set(new_shares.keys()))) # but they should all be at a newer seqnum. The IV will be # different, so the roothash will be too. for key in initial_shares: @@ -70,19 +71,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): IV1, k1, N1, segsize1, datalen1, o1) = unpack_header(new_shares[key]) - self.failUnlessEqual(version0, version1) - self.failUnlessEqual(seqnum0+1, seqnum1) - self.failUnlessEqual(k0, k1) - self.failUnlessEqual(N0, N1) - self.failUnlessEqual(segsize0, segsize1) - self.failUnlessEqual(datalen0, datalen1) + self.assertThat(version0, Equals(version1)) + self.assertThat(seqnum0+1, Equals(seqnum1)) + self.assertThat(k0, Equals(k1)) + self.assertThat(N0, Equals(N1)) + self.assertThat(segsize0, Equals(segsize1)) + self.assertThat(datalen0, Equals(datalen1)) d.addCallback(_check_results) return d def failIfSharesChanged(self, ignored=None): old_shares = self.old_shares[-2] current_shares = self.old_shares[-1] - self.failUnlessEqual(old_shares, current_shares) + self.assertThat(old_shares, Equals(current_shares)) def _test_whether_repairable(self, publisher, nshares, expected_result): @@ -96,12 +97,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(_delete_some_shares) d.addCallback(lambda ign: self._fn.check(Monitor())) def _check(cr): - self.failIf(cr.is_healthy()) - self.failUnlessEqual(cr.is_recoverable(), expected_result) + self.assertThat(cr.is_healthy(), Equals(False)) + self.assertThat(cr.is_recoverable(), Equals(expected_result)) return cr d.addCallback(_check) d.addCallback(lambda check_results: self._fn.repair(check_results)) - d.addCallback(lambda crr: self.failUnlessEqual(crr.get_successful(), expected_result)) + d.addCallback(lambda crr: self.assertThat(crr.get_successful(), Equals(expected_result))) return d def test_unrepairable_0shares(self): @@ -136,7 +137,7 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): del shares[peerid][shnum] d.addCallback(_delete_some_shares) d.addCallback(lambda ign: self._fn.check_and_repair(Monitor())) - d.addCallback(lambda crr: self.failUnlessEqual(crr.get_repair_successful(), expected_result)) + d.addCallback(lambda crr: self.assertThat(crr.get_repair_successful(), Equals(expected_result))) return d def test_unrepairable_0shares_checkandrepair(self): @@ -181,13 +182,13 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): self._fn.repair(check_results, force=True)) # this should give us 10 shares of the highest roothash def _check_repair_results(rres): - self.failUnless(rres.get_successful()) + self.assertThat(rres.get_successful(), Equals(True)) pass # TODO d.addCallback(_check_repair_results) d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failIf(smap.unrecoverable_versions()) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) # now, which should have won? roothash_s4a = self.get_roothash_for(3) roothash_s4b = self.get_roothash_for(4) @@ -196,9 +197,9 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): else: expected_contents = self.CONTENTS[3] new_versionid = smap.best_recoverable_version() - self.failUnlessEqual(new_versionid[0], 5) # seqnum 5 + self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.failUnlessEqual, expected_contents) + d2.addCallback(self.assertEqual, expected_contents) return d2 d.addCallback(_check_smap) return d @@ -216,19 +217,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda check_results: self._fn.repair(check_results)) # this should give us 10 shares of v3 def _check_repair_results(rres): - self.failUnless(rres.get_successful()) + self.assertThat(rres.get_successful(), Equals(True)) pass # TODO d.addCallback(_check_repair_results) d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failIf(smap.unrecoverable_versions()) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) # now, which should have won? expected_contents = self.CONTENTS[3] new_versionid = smap.best_recoverable_version() - self.failUnlessEqual(new_versionid[0], 5) # seqnum 5 + self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.failUnlessEqual, expected_contents) + d2.addCallback(self.assertTrue, expected_contents) return d2 d.addCallback(_check_smap) return d @@ -256,12 +257,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(_get_readcap) d.addCallback(lambda res: self._fn3.check_and_repair(Monitor())) def _check_results(crr): - self.failUnless(ICheckAndRepairResults.providedBy(crr)) + self.assertThat(ICheckAndRepairResults.providedBy(crr), Equals(True)) # we should detect the unhealthy, but skip over mutable-readcap # repairs until #625 is fixed - self.failIf(crr.get_pre_repair_results().is_healthy()) - self.failIf(crr.get_repair_attempted()) - self.failIf(crr.get_post_repair_results().is_healthy()) + self.assertThat(crr.get_pre_repair_results().is_healthy(), Equals(False)) + self.assertThat(crr.get_repair_attempted(), Equals(False)) + self.assertThat(crr.get_post_repair_results().is_healthy(), Equals(False)) d.addCallback(_check_results) return d @@ -281,6 +282,6 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda ign: self._fn2.check(Monitor())) d.addCallback(lambda check_results: self._fn2.repair(check_results)) def _check(crr): - self.failUnlessEqual(crr.get_successful(), True) + self.assertThat(crr.get_successful(), Equals(True)) d.addCallback(_check) return d From 61b9f15fd1f27f428c1492a14ff3e998e07ac79b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 10 Sep 2021 00:59:55 +0100 Subject: [PATCH 008/916] test.mutable : refactored roundtrip and servermap tests Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_roundtrip.py | 44 +++++++-------- src/allmydata/test/mutable/test_servermap.py | 57 ++++++++++---------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index 79292b000..96ecdf640 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -11,7 +11,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import cStringIO as StringIO -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength, Contains from twisted.internet import defer from allmydata.util import base32, consumer @@ -23,8 +24,9 @@ from allmydata.mutable.retrieve import Retrieve from .util import PublishMixin, make_storagebroker, corrupt from .. import common_util as testutil -class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): +class Roundtrip(AsyncTestCase, testutil.ShouldFailMixin, PublishMixin): def setUp(self): + super(Roundtrip, self).setUp() return self.publish_one() def make_servermap(self, mode=MODE_READ, oldmap=None, sb=None): @@ -73,11 +75,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): def _do_retrieve(servermap): self._smap = servermap #self.dump_servermap(servermap) - self.failUnlessEqual(len(servermap.recoverable_versions()), 1) + self.assertThat(servermap.recoverable_versions(), HasLength(1)) return self.do_download(servermap) d.addCallback(_do_retrieve) def _retrieved(new_contents): - self.failUnlessEqual(new_contents, self.CONTENTS) + self.assertThat(new_contents, Equals(self.CONTENTS)) d.addCallback(_retrieved) # we should be able to re-use the same servermap, both with and # without updating it. @@ -132,10 +134,10 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # back empty d = self.make_servermap(sb=sb2) def _check_servermap(servermap): - self.failUnlessEqual(servermap.best_recoverable_version(), None) - self.failIf(servermap.recoverable_versions()) - self.failIf(servermap.unrecoverable_versions()) - self.failIf(servermap.all_servers()) + self.assertThat(servermap.best_recoverable_version(), Equals(None)) + self.assertFalse(servermap.recoverable_versions()) + self.assertFalse(servermap.unrecoverable_versions()) + self.assertFalse(servermap.all_servers()) d.addCallback(_check_servermap) return d @@ -154,7 +156,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): self._fn._storage_broker = self._storage_broker return self._fn.download_best_version() def _retrieved(new_contents): - self.failUnlessEqual(new_contents, self.CONTENTS) + self.assertThat(new_contents, Equals(self.CONTENTS)) d.addCallback(_restore) d.addCallback(_retrieved) return d @@ -178,13 +180,13 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # should be noted in the servermap's list of problems. if substring: allproblems = [str(f) for f in servermap.get_problems()] - self.failUnlessIn(substring, "".join(allproblems)) + self.assertThat("".join(allproblems), Contains(substring)) return servermap if should_succeed: d1 = self._fn.download_version(servermap, ver, fetch_privkey) d1.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) else: d1 = self.shouldFail(NotEnoughSharesError, "_corrupt_all(offset=%s)" % (offset,), @@ -207,7 +209,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # and the dump should mention the problems s = StringIO() dump = servermap.dump(s).getvalue() - self.failUnless("30 PROBLEMS" in dump, dump) + self.assertTrue("30 PROBLEMS" in dump, msg=dump) d.addCallback(_check_servermap) return d @@ -299,8 +301,8 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # in NotEnoughSharesError, since each share will look invalid def _check(res): f = res[0] - self.failUnless(f.check(NotEnoughSharesError)) - self.failUnless("uncoordinated write" in str(f)) + self.assertThat(f.check(NotEnoughSharesError), HasLength(1)) + self.assertThat("uncoordinated write" in str(f), Equals(True)) return self._test_corrupt_all(1, "ran out of servers", corrupt_early=False, failure_checker=_check) @@ -309,7 +311,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): def test_corrupt_all_block_late(self): def _check(res): f = res[0] - self.failUnless(f.check(NotEnoughSharesError)) + self.assertTrue(f.check(NotEnoughSharesError)) return self._test_corrupt_all("share_data", "block hash tree failure", corrupt_early=False, failure_checker=_check) @@ -330,9 +332,9 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): shnums_to_corrupt=list(range(0, N-k))) d.addCallback(lambda res: self.make_servermap()) def _do_retrieve(servermap): - self.failUnless(servermap.get_problems()) - self.failUnless("pubkey doesn't match fingerprint" - in str(servermap.get_problems()[0])) + self.assertTrue(servermap.get_problems()) + self.assertThat("pubkey doesn't match fingerprint" + in str(servermap.get_problems()[0]), Equals(True)) ver = servermap.best_recoverable_version() r = Retrieve(self._fn, self._storage_broker, servermap, ver) c = consumer.MemoryConsumer() @@ -340,7 +342,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): d.addCallback(_do_retrieve) d.addCallback(lambda mc: b"".join(mc.chunks)) d.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) return d @@ -355,11 +357,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): self.make_servermap()) def _do_retrieve(servermap): ver = servermap.best_recoverable_version() - self.failUnless(ver) + self.assertTrue(ver) return self._fn.download_best_version() d.addCallback(_do_retrieve) d.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) return d diff --git a/src/allmydata/test/mutable/test_servermap.py b/src/allmydata/test/mutable/test_servermap.py index e8f933977..505d31e73 100644 --- a/src/allmydata/test/mutable/test_servermap.py +++ b/src/allmydata/test/mutable/test_servermap.py @@ -11,7 +11,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, NotEquals, HasLength from twisted.internet import defer from allmydata.monitor import Monitor from allmydata.mutable.common import \ @@ -20,8 +21,9 @@ from allmydata.mutable.publish import MutableData from allmydata.mutable.servermap import ServerMap, ServermapUpdater from .util import PublishMixin -class Servermap(unittest.TestCase, PublishMixin): +class Servermap(AsyncTestCase, PublishMixin): def setUp(self): + super(Servermap, self).setUp() return self.publish_one() def make_servermap(self, mode=MODE_CHECK, fn=None, sb=None, @@ -42,17 +44,17 @@ class Servermap(unittest.TestCase, PublishMixin): return d def failUnlessOneRecoverable(self, sm, num_shares): - self.failUnlessEqual(len(sm.recoverable_versions()), 1) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 0) + self.assertThat(sm.recoverable_versions(), HasLength(1)) + self.assertThat(sm.unrecoverable_versions(), HasLength(0)) best = sm.best_recoverable_version() - self.failIfEqual(best, None) - self.failUnlessEqual(sm.recoverable_versions(), set([best])) - self.failUnlessEqual(len(sm.shares_available()), 1) - self.failUnlessEqual(sm.shares_available()[best], (num_shares, 3, 10)) + self.assertThat(best, NotEquals(None)) + self.assertThat(sm.recoverable_versions(), Equals(set([best]))) + self.assertThat(sm.shares_available(), HasLength(1)) + self.assertThat(sm.shares_available()[best], Equals((num_shares, 3, 10))) shnum, servers = list(sm.make_sharemap().items())[0] server = list(servers)[0] - self.failUnlessEqual(sm.version_on_server(server, shnum), best) - self.failUnlessEqual(sm.version_on_server(server, 666), None) + self.assertThat(sm.version_on_server(server, shnum), Equals(best)) + self.assertThat(sm.version_on_server(server, 666), Equals(None)) return sm def test_basic(self): @@ -117,7 +119,7 @@ class Servermap(unittest.TestCase, PublishMixin): v = sm.best_recoverable_version() vm = sm.make_versionmap() shares = list(vm[v]) - self.failUnlessEqual(len(shares), 6) + self.assertThat(shares, HasLength(6)) self._corrupted = set() # mark the first 5 shares as corrupt, then update the servermap. # The map should not have the marked shares it in any more, and @@ -135,18 +137,17 @@ class Servermap(unittest.TestCase, PublishMixin): shares = list(vm[v]) for (server, shnum) in self._corrupted: server_shares = sm.debug_shares_on_server(server) - self.failIf(shnum in server_shares, - "%d was in %s" % (shnum, server_shares)) - self.failUnlessEqual(len(shares), 5) + self.assertFalse(shnum in server_shares, "%d was in %s" % (shnum, server_shares)) + self.assertThat(shares, HasLength(5)) d.addCallback(_check_map) return d def failUnlessNoneRecoverable(self, sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 0) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 0) + self.assertThat(sm.recoverable_versions(), HasLength(0)) + self.assertThat(sm.unrecoverable_versions(), HasLength(0)) best = sm.best_recoverable_version() - self.failUnlessEqual(best, None) - self.failUnlessEqual(len(sm.shares_available()), 0) + self.assertThat(best, Equals(None)) + self.assertThat(sm.shares_available(), HasLength(0)) def test_no_shares(self): self._storage._peers = {} # delete all shares @@ -168,12 +169,12 @@ class Servermap(unittest.TestCase, PublishMixin): return d def failUnlessNotQuiteEnough(self, sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 0) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 1) + self.assertThat(sm.recoverable_versions(), HasLength(0)) + self.assertThat(sm.unrecoverable_versions(), HasLength(1)) best = sm.best_recoverable_version() - self.failUnlessEqual(best, None) - self.failUnlessEqual(len(sm.shares_available()), 1) - self.failUnlessEqual(list(sm.shares_available().values())[0], (2,3,10) ) + self.assertThat(best, Equals(None)) + self.assertThat(sm.shares_available(), HasLength(1)) + self.assertThat(list(sm.shares_available().values())[0], Equals((2,3,10))) return sm def test_not_quite_enough_shares(self): @@ -193,7 +194,7 @@ class Servermap(unittest.TestCase, PublishMixin): d.addCallback(lambda res: ms(mode=MODE_CHECK)) d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm)) d.addCallback(lambda sm: - self.failUnlessEqual(len(sm.make_sharemap()), 2)) + self.assertThat(sm.make_sharemap(), HasLength(2))) d.addCallback(lambda res: ms(mode=MODE_ANYTHING)) d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm)) d.addCallback(lambda res: ms(mode=MODE_WRITE)) @@ -216,7 +217,7 @@ class Servermap(unittest.TestCase, PublishMixin): # Calling make_servermap also updates the servermap in the mode # that we specify, so we just need to see what it says. def _check_servermap(sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 1) + self.assertThat(sm.recoverable_versions(), HasLength(1)) d.addCallback(_check_servermap) return d @@ -229,10 +230,10 @@ class Servermap(unittest.TestCase, PublishMixin): self.make_servermap(mode=MODE_WRITE, update_range=(1, 2))) def _check_servermap(sm): # 10 shares - self.failUnlessEqual(len(sm.update_data), 10) + self.assertThat(sm.update_data, HasLength(10)) # one version for data in sm.update_data.values(): - self.failUnlessEqual(len(data), 1) + self.assertThat(data, HasLength(1)) d.addCallback(_check_servermap) return d @@ -244,5 +245,5 @@ class Servermap(unittest.TestCase, PublishMixin): d.addCallback(lambda ignored: self.make_servermap(mode=MODE_CHECK)) d.addCallback(lambda servermap: - self.failUnlessEqual(len(servermap.recoverable_versions()), 1)) + self.assertThat(servermap.recoverable_versions(), HasLength(1))) return d From 3b80b8cbe97a14e9e14e46a0a329c9e6bdcc8c12 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 10 Sep 2021 14:24:20 +0100 Subject: [PATCH 009/916] test.mutable : refactored test_version.py Signed-off-by: fenn-cs --- newsfragments/3788.minor | 0 src/allmydata/test/mutable/test_version.py | 116 +++++++++++---------- 2 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 newsfragments/3788.minor diff --git a/newsfragments/3788.minor b/newsfragments/3788.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 042305c24..d5c44f204 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -14,7 +14,13 @@ import os from six.moves import cStringIO as StringIO from twisted.internet import defer -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import ( + Equals, + IsInstance, + HasLength, + Contains, +) from allmydata import uri from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION @@ -29,7 +35,7 @@ from ..no_network import GridTestMixin from .util import PublishMixin from .. import common_util as testutil -class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ +class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ PublishMixin): def setUp(self): GridTestMixin.setUp(self) @@ -47,8 +53,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d = self.nm.create_mutable_file(MutableData(data), version=MDMF_VERSION) def _then(n): - assert isinstance(n, MutableFileNode) - assert n._protocol_version == MDMF_VERSION + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) self.mdmf_node = n return n d.addCallback(_then) @@ -59,8 +65,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ data = self.small_data d = self.nm.create_mutable_file(MutableData(data)) def _then(n): - assert isinstance(n, MutableFileNode) - assert n._protocol_version == SDMF_VERSION + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) self.sdmf_node = n return n d.addCallback(_then) @@ -69,9 +75,9 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def do_upload_empty_sdmf(self): d = self.nm.create_mutable_file(MutableData(b"")) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.sdmf_zero_length_node = n - assert n._protocol_version == SDMF_VERSION + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) return n d.addCallback(_then) return d @@ -95,7 +101,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ debug.find_shares(fso) sharefiles = fso.stdout.getvalue().splitlines() expected = self.nm.default_encoding_parameters["n"] - self.failUnlessEqual(len(sharefiles), expected) + self.assertThat(sharefiles, HasLength(expected)) do = debug.DumpOptions() do["filename"] = sharefiles[0] @@ -103,17 +109,17 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ debug.dump_share(do) output = do.stdout.getvalue() lines = set(output.splitlines()) - self.failUnless("Mutable slot found:" in lines, output) - self.failUnless(" share_type: MDMF" in lines, output) - self.failUnless(" num_extra_leases: 0" in lines, output) - self.failUnless(" MDMF contents:" in lines, output) - self.failUnless(" seqnum: 1" in lines, output) - self.failUnless(" required_shares: 3" in lines, output) - self.failUnless(" total_shares: 10" in lines, output) - self.failUnless(" segsize: 131073" in lines, output) - self.failUnless(" datalen: %d" % len(self.data) in lines, output) + self.assertTrue("Mutable slot found:" in lines, output) + self.assertTrue(" share_type: MDMF" in lines, output) + self.assertTrue(" num_extra_leases: 0" in lines, output) + self.assertTrue(" MDMF contents:" in lines, output) + self.assertTrue(" seqnum: 1" in lines, output) + self.assertTrue(" required_shares: 3" in lines, output) + self.assertTrue(" total_shares: 10" in lines, output) + self.assertTrue(" segsize: 131073" in lines, output) + self.assertTrue(" datalen: %d" % len(self.data) in lines, output) vcap = str(n.get_verify_cap().to_string(), "utf-8") - self.failUnless(" verify-cap: %s" % vcap in lines, output) + self.assertTrue(" verify-cap: %s" % vcap in lines, output) cso = debug.CatalogSharesOptions() cso.nodedirs = fso.nodedirs cso.stdout = StringIO() @@ -122,13 +128,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ shares = cso.stdout.getvalue().splitlines() oneshare = shares[0] # all shares should be MDMF self.failIf(oneshare.startswith("UNKNOWN"), oneshare) - self.failUnless(oneshare.startswith("MDMF"), oneshare) + self.assertTrue(oneshare.startswith("MDMF"), oneshare) fields = oneshare.split() - self.failUnlessEqual(fields[0], "MDMF") - self.failUnlessEqual(fields[1].encode("ascii"), storage_index) - self.failUnlessEqual(fields[2], "3/10") - self.failUnlessEqual(fields[3], "%d" % len(self.data)) - self.failUnless(fields[4].startswith("#1:"), fields[3]) + self.assertThat(fields[0], Equals("MDMF")) + self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) + self.assertThat(fields[2], Equals("3/10")) + self.assertThat(fields[3], Equals("%d" % len(self.data))) + self.assertTrue(fields[4].startswith("#1:"), fields[3]) # the rest of fields[4] is the roothash, which depends upon # encryption salts and is not constant. fields[5] is the # remaining time on the longest lease, which is timing dependent. @@ -140,11 +146,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d = self.do_upload() d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 1)) + self.assertThat(bv.get_sequence_number(), Equals(1))) d.addCallback(lambda ignored: self.sdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 1)) + self.assertThat(bv.get_sequence_number(), Equals(1))) # Now update. The sequence number in both cases should be 1 in # both cases. def _do_update(ignored): @@ -158,11 +164,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 2)) + self.assertThat(bv.get_sequence_number(), Equals(2))) d.addCallback(lambda ignored: self.sdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 2)) + self.assertThat(bv.get_sequence_number(), Equals(2))) return d @@ -175,10 +181,10 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def _then(ign): mdmf_uri = self.mdmf_node.get_uri() cap = uri.from_string(mdmf_uri) - self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI)) + self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() cap = uri.from_string(readonly_mdmf_uri) - self.failUnless(isinstance(cap, uri.ReadonlyMDMFFileURI)) + self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) d.addCallback(_then) return d @@ -189,16 +195,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version()) def _check_mdmf(bv): n = self.mdmf_node - self.failUnlessEqual(bv.get_writekey(), n.get_writekey()) - self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index()) - self.failIf(bv.is_readonly()) + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) d.addCallback(_check_mdmf) d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version()) def _check_sdmf(bv): n = self.sdmf_node - self.failUnlessEqual(bv.get_writekey(), n.get_writekey()) - self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index()) - self.failIf(bv.is_readonly()) + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) d.addCallback(_check_sdmf) return d @@ -206,21 +212,21 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def test_get_readonly_version(self): d = self.do_upload() d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.failUnless(bv.is_readonly())) + d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) # Attempting to get a mutable version of a mutable file from a # filenode initialized with a readcap should return a readonly # version of that same node. d.addCallback(lambda ign: self.mdmf_node.get_readonly()) d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.failUnless(v.is_readonly())) + d.addCallback(lambda v: self.assertTrue(v.is_readonly())) d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.failUnless(bv.is_readonly())) + d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) d.addCallback(lambda ign: self.sdmf_node.get_readonly()) d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.failUnless(v.is_readonly())) + d.addCallback(lambda v: self.assertTrue(v.is_readonly())) return d @@ -232,13 +238,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, b"foo bar baz" * 100000)) + self.assertThat(data, Equals(b"foo bar baz" * 100000))) d.addCallback(lambda ignored: self.sdmf_node.overwrite(new_small_data)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, b"foo bar baz" * 10)) + self.assertThat(data, Equals(b"foo bar baz" * 10))) return d @@ -250,13 +256,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) d.addCallback(lambda ignored: self.sdmf_node.modify(modifier)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) return d @@ -271,13 +277,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) d.addCallback(lambda ignored: self.sdmf_node.modify(modifier)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) return d @@ -308,13 +314,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self._fn.download_version(self.servermap, self.version1)) d.addCallback(lambda results: - self.failUnlessEqual(self.CONTENTS[self.version1_index], - results)) + self.assertThat(self.CONTENTS[self.version1_index], + Equals(results))) d.addCallback(lambda ignored: self._fn.download_version(self.servermap, self.version2)) d.addCallback(lambda results: - self.failUnlessEqual(self.CONTENTS[self.version2_index], - results)) + self.assertThat(self.CONTENTS[self.version2_index], + Equals(results))) return d @@ -344,7 +350,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ for i in range(0, len(expected), step): d2.addCallback(lambda ignored, i=i: version.read(c, i, step)) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c.chunks))) + self.assertThat(expected, Equals(b"".join(c.chunks)))) return d2 d.addCallback(_read_data) return d @@ -447,16 +453,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d2 = defer.succeed(None) d2.addCallback(lambda ignored: version.read(c)) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c.chunks))) + self.assertThat(expected, Equals(b"".join(c.chunks)))) d2.addCallback(lambda ignored: version.read(c2, offset=0, size=len(expected))) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c2.chunks))) + self.assertThat(expected, Equals(b"".join(c2.chunks)))) return d2 d.addCallback(_read_data) d.addCallback(lambda ignored: node.download_best_version()) - d.addCallback(lambda data: self.failUnlessEqual(expected, data)) + d.addCallback(lambda data: self.assertThat(expected, Equals(data))) return d def test_read_and_download_mdmf(self): From a3168b384450dbd4a9a576425442d0a92267e950 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 13 Sep 2021 23:45:16 +0100 Subject: [PATCH 010/916] test.mutable : refactored test_update.py Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_update.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index da5d53e4c..c3ba1e9f7 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -11,7 +11,12 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import ( + Equals, + IsInstance, + GreaterThan, +) from twisted.internet import defer from allmydata.interfaces import MDMF_VERSION from allmydata.mutable.filenode import MutableFileNode @@ -25,7 +30,7 @@ from .. import common_util as testutil # this up. SEGSIZE = 128*1024 -class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Update(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): def setUp(self): GridTestMixin.setUp(self) self.basedir = self.mktemp() @@ -35,14 +40,14 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # self.data should be at least three segments long. td = b"testdata " self.data = td*(int(3*SEGSIZE//len(td))+10) # currently about 400kB - assert len(self.data) > 3*SEGSIZE + self.assertThat(len(self.data), GreaterThan(3*SEGSIZE)) self.small_data = b"test data" * 10 # 90 B; SDMF def do_upload_sdmf(self): d = self.nm.create_mutable_file(MutableData(self.small_data)) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.sdmf_node = n d.addCallback(_then) return d @@ -51,7 +56,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d = self.nm.create_mutable_file(MutableData(self.data), version=MDMF_VERSION) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.mdmf_node = n d.addCallback(_then) return d @@ -185,7 +190,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): len(self.data))) d.addCallback(lambda ign: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -201,7 +206,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): len(self.small_data))) d.addCallback(lambda ign: self.sdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -221,7 +226,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): replace_offset)) d.addCallback(lambda ign: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -242,7 +247,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): replace_offset)) d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 From 5db3540c206b8bc9b12b50d61800b669334e3555 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 10:56:28 +0100 Subject: [PATCH 011/916] update NEWS.txt for release --- NEWS.rst | 98 ++++++++++++++++++++++++++++++++ newsfragments/1549.installation | 1 - newsfragments/2928.minor | 0 newsfragments/3037.other | 1 - newsfragments/3283.minor | 0 newsfragments/3314.minor | 0 newsfragments/3326.installation | 1 - newsfragments/3384.minor | 0 newsfragments/3385.minor | 0 newsfragments/3390.minor | 0 newsfragments/3399.feature | 1 - newsfragments/3404.minor | 0 newsfragments/3428.minor | 0 newsfragments/3432.minor | 0 newsfragments/3433.installation | 1 - newsfragments/3434.minor | 0 newsfragments/3435.minor | 0 newsfragments/3454.minor | 0 newsfragments/3459.minor | 0 newsfragments/3460.minor | 0 newsfragments/3465.minor | 0 newsfragments/3466.minor | 0 newsfragments/3467.minor | 0 newsfragments/3468.minor | 0 newsfragments/3470.minor | 0 newsfragments/3471.minor | 0 newsfragments/3472.minor | 0 newsfragments/3473.minor | 0 newsfragments/3474.minor | 0 newsfragments/3475.minor | 0 newsfragments/3477.minor | 0 newsfragments/3478.minor | 1 - newsfragments/3479.minor | 0 newsfragments/3481.minor | 0 newsfragments/3482.minor | 0 newsfragments/3483.minor | 0 newsfragments/3485.minor | 0 newsfragments/3486.installation | 1 - newsfragments/3488.minor | 0 newsfragments/3490.minor | 0 newsfragments/3491.minor | 0 newsfragments/3492.minor | 0 newsfragments/3493.minor | 0 newsfragments/3496.minor | 0 newsfragments/3497.installation | 1 - newsfragments/3499.minor | 0 newsfragments/3500.minor | 0 newsfragments/3501.minor | 0 newsfragments/3502.minor | 0 newsfragments/3503.other | 1 - newsfragments/3504.configuration | 1 - newsfragments/3509.bugfix | 1 - newsfragments/3510.bugfix | 1 - newsfragments/3511.minor | 0 newsfragments/3513.minor | 0 newsfragments/3514.minor | 0 newsfragments/3515.minor | 0 newsfragments/3517.minor | 0 newsfragments/3518.removed | 1 - newsfragments/3520.minor | 0 newsfragments/3521.minor | 0 newsfragments/3522.minor | 0 newsfragments/3523.minor | 0 newsfragments/3524.minor | 0 newsfragments/3528.minor | 0 newsfragments/3529.minor | 0 newsfragments/3532.minor | 0 newsfragments/3533.minor | 0 newsfragments/3534.minor | 0 newsfragments/3536.minor | 0 newsfragments/3537.minor | 0 newsfragments/3539.bugfix | 1 - newsfragments/3542.minor | 0 newsfragments/3544.minor | 0 newsfragments/3545.other | 1 - newsfragments/3546.minor | 0 newsfragments/3547.minor | 0 newsfragments/3549.removed | 1 - newsfragments/3550.removed | 1 - newsfragments/3551.minor | 0 newsfragments/3552.minor | 0 newsfragments/3553.minor | 0 newsfragments/3555.minor | 0 newsfragments/3557.minor | 0 newsfragments/3558.minor | 0 newsfragments/3560.minor | 0 newsfragments/3563.minor | 0 newsfragments/3564.minor | 0 newsfragments/3565.minor | 0 newsfragments/3566.minor | 0 newsfragments/3567.minor | 0 newsfragments/3568.minor | 0 newsfragments/3572.minor | 0 newsfragments/3574.minor | 0 newsfragments/3575.minor | 0 newsfragments/3576.minor | 0 newsfragments/3577.minor | 0 newsfragments/3578.minor | 0 newsfragments/3579.minor | 0 newsfragments/3580.minor | 0 newsfragments/3582.minor | 0 newsfragments/3583.removed | 1 - newsfragments/3584.bugfix | 1 - newsfragments/3587.minor | 1 - newsfragments/3588.incompat | 1 - newsfragments/3588.minor | 0 newsfragments/3589.minor | 0 newsfragments/3590.bugfix | 1 - newsfragments/3591.minor | 0 newsfragments/3592.minor | 0 newsfragments/3593.minor | 0 newsfragments/3594.minor | 0 newsfragments/3595.minor | 0 newsfragments/3596.minor | 0 newsfragments/3599.minor | 0 newsfragments/3600.minor | 0 newsfragments/3603.minor.rst | 0 newsfragments/3605.minor | 0 newsfragments/3606.minor | 0 newsfragments/3607.minor | 0 newsfragments/3608.minor | 0 newsfragments/3611.minor | 0 newsfragments/3612.minor | 0 newsfragments/3613.minor | 0 newsfragments/3615.minor | 0 newsfragments/3616.minor | 0 newsfragments/3617.minor | 0 newsfragments/3618.minor | 0 newsfragments/3619.minor | 0 newsfragments/3620.minor | 0 newsfragments/3621.minor | 0 newsfragments/3623.minor | 1 - newsfragments/3624.minor | 0 newsfragments/3625.minor | 0 newsfragments/3626.minor | 0 newsfragments/3628.minor | 0 newsfragments/3629.feature | 1 - newsfragments/3630.minor | 0 newsfragments/3631.minor | 0 newsfragments/3632.minor | 0 newsfragments/3633.installation | 1 - newsfragments/3634.minor | 0 newsfragments/3635.minor | 0 newsfragments/3637.minor | 0 newsfragments/3638.minor | 0 newsfragments/3640.minor | 0 newsfragments/3642.minor | 0 newsfragments/3644.other | 1 - newsfragments/3645.minor | 0 newsfragments/3646.minor | 0 newsfragments/3647.minor | 0 newsfragments/3648.minor | 0 newsfragments/3649.minor | 0 newsfragments/3650.bugfix | 1 - newsfragments/3651.minor | 1 - newsfragments/3652.removed | 1 - newsfragments/3653.minor | 0 newsfragments/3654.minor | 0 newsfragments/3655.minor | 0 newsfragments/3656.minor | 0 newsfragments/3657.minor | 0 newsfragments/3658.minor | 0 newsfragments/3659.documentation | 0 newsfragments/3662.minor | 0 newsfragments/3663.other | 1 - newsfragments/3664.documentation | 1 - newsfragments/3666.documentation | 1 - newsfragments/3667.minor | 0 newsfragments/3669.minor | 0 newsfragments/3670.minor | 0 newsfragments/3671.minor | 0 newsfragments/3672.minor | 0 newsfragments/3674.minor | 0 newsfragments/3675.minor | 0 newsfragments/3676.minor | 0 newsfragments/3677.documentation | 1 - newsfragments/3678.minor | 0 newsfragments/3679.minor | 0 newsfragments/3681.minor | 8 --- newsfragments/3682.documentation | 1 - newsfragments/3683.minor | 0 newsfragments/3686.minor | 0 newsfragments/3687.minor | 0 newsfragments/3691.minor | 0 newsfragments/3692.minor | 0 newsfragments/3699.minor | 0 newsfragments/3700.minor | 0 newsfragments/3701.minor | 0 newsfragments/3702.minor | 0 newsfragments/3703.minor | 0 newsfragments/3704.minor | 0 newsfragments/3705.minor | 0 newsfragments/3707.minor | 0 newsfragments/3708.minor | 0 newsfragments/3709.minor | 0 newsfragments/3711.minor | 0 newsfragments/3712.installation | 1 - newsfragments/3713.minor | 0 newsfragments/3714.minor | 0 newsfragments/3715.minor | 0 newsfragments/3716.incompat | 1 - newsfragments/3717.minor | 0 newsfragments/3718.minor | 0 newsfragments/3721.documentation | 1 - newsfragments/3722.minor | 0 newsfragments/3723.minor | 0 newsfragments/3726.documentation | 1 - newsfragments/3727.minor | 0 newsfragments/3728.minor | 0 newsfragments/3729.minor | 0 newsfragments/3730.minor | 0 newsfragments/3731.minor | 0 newsfragments/3732.minor | 0 newsfragments/3733.installation | 1 - newsfragments/3734.minor | 0 newsfragments/3735.minor | 0 newsfragments/3736.minor | 0 newsfragments/3738.bugfix | 1 - newsfragments/3739.bugfix | 1 - newsfragments/3741.minor | 0 newsfragments/3743.minor | 0 newsfragments/3744.minor | 0 newsfragments/3745.minor | 0 newsfragments/3746.minor | 0 newsfragments/3747.documentation | 1 - newsfragments/3749.documentation | 1 - newsfragments/3751.minor | 0 newsfragments/3757.other | 1 - newsfragments/3759.minor | 0 newsfragments/3760.minor | 0 newsfragments/3763.minor | 0 newsfragments/3764.documentation | 1 - newsfragments/3765.documentation | 1 - newsfragments/3769.documentation | 1 - newsfragments/3773.minor | 0 newsfragments/3774.documentation | 1 - newsfragments/3777.documentation | 1 - newsfragments/3779.bugfix | 1 - newsfragments/3781.minor | 0 newsfragments/3782.documentation | 1 - newsfragments/3785.documentation | 1 - 241 files changed, 98 insertions(+), 60 deletions(-) delete mode 100644 newsfragments/1549.installation delete mode 100644 newsfragments/2928.minor delete mode 100644 newsfragments/3037.other delete mode 100644 newsfragments/3283.minor delete mode 100644 newsfragments/3314.minor delete mode 100644 newsfragments/3326.installation delete mode 100644 newsfragments/3384.minor delete mode 100644 newsfragments/3385.minor delete mode 100644 newsfragments/3390.minor delete mode 100644 newsfragments/3399.feature delete mode 100644 newsfragments/3404.minor delete mode 100644 newsfragments/3428.minor delete mode 100644 newsfragments/3432.minor delete mode 100644 newsfragments/3433.installation delete mode 100644 newsfragments/3434.minor delete mode 100644 newsfragments/3435.minor delete mode 100644 newsfragments/3454.minor delete mode 100644 newsfragments/3459.minor delete mode 100644 newsfragments/3460.minor delete mode 100644 newsfragments/3465.minor delete mode 100644 newsfragments/3466.minor delete mode 100644 newsfragments/3467.minor delete mode 100644 newsfragments/3468.minor delete mode 100644 newsfragments/3470.minor delete mode 100644 newsfragments/3471.minor delete mode 100644 newsfragments/3472.minor delete mode 100644 newsfragments/3473.minor delete mode 100644 newsfragments/3474.minor delete mode 100644 newsfragments/3475.minor delete mode 100644 newsfragments/3477.minor delete mode 100644 newsfragments/3478.minor delete mode 100644 newsfragments/3479.minor delete mode 100644 newsfragments/3481.minor delete mode 100644 newsfragments/3482.minor delete mode 100644 newsfragments/3483.minor delete mode 100644 newsfragments/3485.minor delete mode 100644 newsfragments/3486.installation delete mode 100644 newsfragments/3488.minor delete mode 100644 newsfragments/3490.minor delete mode 100644 newsfragments/3491.minor delete mode 100644 newsfragments/3492.minor delete mode 100644 newsfragments/3493.minor delete mode 100644 newsfragments/3496.minor delete mode 100644 newsfragments/3497.installation delete mode 100644 newsfragments/3499.minor delete mode 100644 newsfragments/3500.minor delete mode 100644 newsfragments/3501.minor delete mode 100644 newsfragments/3502.minor delete mode 100644 newsfragments/3503.other delete mode 100644 newsfragments/3504.configuration delete mode 100644 newsfragments/3509.bugfix delete mode 100644 newsfragments/3510.bugfix delete mode 100644 newsfragments/3511.minor delete mode 100644 newsfragments/3513.minor delete mode 100644 newsfragments/3514.minor delete mode 100644 newsfragments/3515.minor delete mode 100644 newsfragments/3517.minor delete mode 100644 newsfragments/3518.removed delete mode 100644 newsfragments/3520.minor delete mode 100644 newsfragments/3521.minor delete mode 100644 newsfragments/3522.minor delete mode 100644 newsfragments/3523.minor delete mode 100644 newsfragments/3524.minor delete mode 100644 newsfragments/3528.minor delete mode 100644 newsfragments/3529.minor delete mode 100644 newsfragments/3532.minor delete mode 100644 newsfragments/3533.minor delete mode 100644 newsfragments/3534.minor delete mode 100644 newsfragments/3536.minor delete mode 100644 newsfragments/3537.minor delete mode 100644 newsfragments/3539.bugfix delete mode 100644 newsfragments/3542.minor delete mode 100644 newsfragments/3544.minor delete mode 100644 newsfragments/3545.other delete mode 100644 newsfragments/3546.minor delete mode 100644 newsfragments/3547.minor delete mode 100644 newsfragments/3549.removed delete mode 100644 newsfragments/3550.removed delete mode 100644 newsfragments/3551.minor delete mode 100644 newsfragments/3552.minor delete mode 100644 newsfragments/3553.minor delete mode 100644 newsfragments/3555.minor delete mode 100644 newsfragments/3557.minor delete mode 100644 newsfragments/3558.minor delete mode 100644 newsfragments/3560.minor delete mode 100644 newsfragments/3563.minor delete mode 100644 newsfragments/3564.minor delete mode 100644 newsfragments/3565.minor delete mode 100644 newsfragments/3566.minor delete mode 100644 newsfragments/3567.minor delete mode 100644 newsfragments/3568.minor delete mode 100644 newsfragments/3572.minor delete mode 100644 newsfragments/3574.minor delete mode 100644 newsfragments/3575.minor delete mode 100644 newsfragments/3576.minor delete mode 100644 newsfragments/3577.minor delete mode 100644 newsfragments/3578.minor delete mode 100644 newsfragments/3579.minor delete mode 100644 newsfragments/3580.minor delete mode 100644 newsfragments/3582.minor delete mode 100644 newsfragments/3583.removed delete mode 100644 newsfragments/3584.bugfix delete mode 100644 newsfragments/3587.minor delete mode 100644 newsfragments/3588.incompat delete mode 100644 newsfragments/3588.minor delete mode 100644 newsfragments/3589.minor delete mode 100644 newsfragments/3590.bugfix delete mode 100644 newsfragments/3591.minor delete mode 100644 newsfragments/3592.minor delete mode 100644 newsfragments/3593.minor delete mode 100644 newsfragments/3594.minor delete mode 100644 newsfragments/3595.minor delete mode 100644 newsfragments/3596.minor delete mode 100644 newsfragments/3599.minor delete mode 100644 newsfragments/3600.minor delete mode 100644 newsfragments/3603.minor.rst delete mode 100644 newsfragments/3605.minor delete mode 100644 newsfragments/3606.minor delete mode 100644 newsfragments/3607.minor delete mode 100644 newsfragments/3608.minor delete mode 100644 newsfragments/3611.minor delete mode 100644 newsfragments/3612.minor delete mode 100644 newsfragments/3613.minor delete mode 100644 newsfragments/3615.minor delete mode 100644 newsfragments/3616.minor delete mode 100644 newsfragments/3617.minor delete mode 100644 newsfragments/3618.minor delete mode 100644 newsfragments/3619.minor delete mode 100644 newsfragments/3620.minor delete mode 100644 newsfragments/3621.minor delete mode 100644 newsfragments/3623.minor delete mode 100644 newsfragments/3624.minor delete mode 100644 newsfragments/3625.minor delete mode 100644 newsfragments/3626.minor delete mode 100644 newsfragments/3628.minor delete mode 100644 newsfragments/3629.feature delete mode 100644 newsfragments/3630.minor delete mode 100644 newsfragments/3631.minor delete mode 100644 newsfragments/3632.minor delete mode 100644 newsfragments/3633.installation delete mode 100644 newsfragments/3634.minor delete mode 100644 newsfragments/3635.minor delete mode 100644 newsfragments/3637.minor delete mode 100644 newsfragments/3638.minor delete mode 100644 newsfragments/3640.minor delete mode 100644 newsfragments/3642.minor delete mode 100644 newsfragments/3644.other delete mode 100644 newsfragments/3645.minor delete mode 100644 newsfragments/3646.minor delete mode 100644 newsfragments/3647.minor delete mode 100644 newsfragments/3648.minor delete mode 100644 newsfragments/3649.minor delete mode 100644 newsfragments/3650.bugfix delete mode 100644 newsfragments/3651.minor delete mode 100644 newsfragments/3652.removed delete mode 100644 newsfragments/3653.minor delete mode 100644 newsfragments/3654.minor delete mode 100644 newsfragments/3655.minor delete mode 100644 newsfragments/3656.minor delete mode 100644 newsfragments/3657.minor delete mode 100644 newsfragments/3658.minor delete mode 100644 newsfragments/3659.documentation delete mode 100644 newsfragments/3662.minor delete mode 100644 newsfragments/3663.other delete mode 100644 newsfragments/3664.documentation delete mode 100644 newsfragments/3666.documentation delete mode 100644 newsfragments/3667.minor delete mode 100644 newsfragments/3669.minor delete mode 100644 newsfragments/3670.minor delete mode 100644 newsfragments/3671.minor delete mode 100644 newsfragments/3672.minor delete mode 100644 newsfragments/3674.minor delete mode 100644 newsfragments/3675.minor delete mode 100644 newsfragments/3676.minor delete mode 100644 newsfragments/3677.documentation delete mode 100644 newsfragments/3678.minor delete mode 100644 newsfragments/3679.minor delete mode 100644 newsfragments/3681.minor delete mode 100644 newsfragments/3682.documentation delete mode 100644 newsfragments/3683.minor delete mode 100644 newsfragments/3686.minor delete mode 100644 newsfragments/3687.minor delete mode 100644 newsfragments/3691.minor delete mode 100644 newsfragments/3692.minor delete mode 100644 newsfragments/3699.minor delete mode 100644 newsfragments/3700.minor delete mode 100644 newsfragments/3701.minor delete mode 100644 newsfragments/3702.minor delete mode 100644 newsfragments/3703.minor delete mode 100644 newsfragments/3704.minor delete mode 100644 newsfragments/3705.minor delete mode 100644 newsfragments/3707.minor delete mode 100644 newsfragments/3708.minor delete mode 100644 newsfragments/3709.minor delete mode 100644 newsfragments/3711.minor delete mode 100644 newsfragments/3712.installation delete mode 100644 newsfragments/3713.minor delete mode 100644 newsfragments/3714.minor delete mode 100644 newsfragments/3715.minor delete mode 100644 newsfragments/3716.incompat delete mode 100644 newsfragments/3717.minor delete mode 100644 newsfragments/3718.minor delete mode 100644 newsfragments/3721.documentation delete mode 100644 newsfragments/3722.minor delete mode 100644 newsfragments/3723.minor delete mode 100644 newsfragments/3726.documentation delete mode 100644 newsfragments/3727.minor delete mode 100644 newsfragments/3728.minor delete mode 100644 newsfragments/3729.minor delete mode 100644 newsfragments/3730.minor delete mode 100644 newsfragments/3731.minor delete mode 100644 newsfragments/3732.minor delete mode 100644 newsfragments/3733.installation delete mode 100644 newsfragments/3734.minor delete mode 100644 newsfragments/3735.minor delete mode 100644 newsfragments/3736.minor delete mode 100644 newsfragments/3738.bugfix delete mode 100644 newsfragments/3739.bugfix delete mode 100644 newsfragments/3741.minor delete mode 100644 newsfragments/3743.minor delete mode 100644 newsfragments/3744.minor delete mode 100644 newsfragments/3745.minor delete mode 100644 newsfragments/3746.minor delete mode 100644 newsfragments/3747.documentation delete mode 100644 newsfragments/3749.documentation delete mode 100644 newsfragments/3751.minor delete mode 100644 newsfragments/3757.other delete mode 100644 newsfragments/3759.minor delete mode 100644 newsfragments/3760.minor delete mode 100644 newsfragments/3763.minor delete mode 100644 newsfragments/3764.documentation delete mode 100644 newsfragments/3765.documentation delete mode 100644 newsfragments/3769.documentation delete mode 100644 newsfragments/3773.minor delete mode 100644 newsfragments/3774.documentation delete mode 100644 newsfragments/3777.documentation delete mode 100644 newsfragments/3779.bugfix delete mode 100644 newsfragments/3781.minor delete mode 100644 newsfragments/3782.documentation delete mode 100644 newsfragments/3785.documentation diff --git a/NEWS.rst b/NEWS.rst index 1cfc726ae..a7d814c70 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,104 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.15.1.post2188.dev0 (2021-09-17)Release 1.15.1.post2188.dev0 (2021-09-17) +''''''''''''''''''''''''''''''''''''''''' + +Backwards Incompatible Changes +------------------------------ + +- The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale. (`#3588 `_) +- tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. (`#3716 `_) + + +Features +-------- + +- Added 'typechecks' environment for tox running mypy and performing static typechecks. (`#3399 `_) +- The NixOS-packaged Tahoe-LAFS now knows its own version. (`#3629 `_) + + +Bug Fixes +--------- + +- Fix regression that broke flogtool results on Python 2. (`#3509 `_) +- Fix a logging regression on Python 2 involving unicode strings. (`#3510 `_) +- Certain implementation-internal weakref KeyErrors are now handled and should no longer cause user-initiated operations to fail. (`#3539 `_) +- SFTP public key auth likely works more consistently, and SFTP in general was previously broken. (`#3584 `_) +- Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work. (`#3590 `_) +- ``tahoe invite`` will now read share encoding/placement configuration values from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. (`#3650 `_) +- Fix regression where uploading files with non-ASCII names failed. (`#3738 `_) +- Fixed annoying UnicodeWarning message on Python 2 when running CLI tools. (`#3739 `_) +- Fixed bug where share corruption events were not logged on storage servers running on Windows. (`#3779 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS now requires Twisted 19.10.0 or newer. As a result, it now has a transitive dependency on bcrypt. (`#1549 `_) +- Debian 8 support has been replaced with Debian 10 support. (`#3326 `_) +- Tahoe-LAFS no longer depends on Nevow. (`#3433 `_) +- Tahoe-LAFS now requires the `netifaces` Python package and no longer requires the external `ip`, `ifconfig`, or `route.exe` executables. (`#3486 `_) +- The Tahoe-LAFS project no longer commits to maintaining binary packages for all dependencies at . Please use PyPI instead. (`#3497 `_) +- Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support. (`#3633 `_) +- The Nix package now includes correct version information. (`#3712 `_) +- Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows. VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult. (`#3733 `_) + + +Configuration Changes +--------------------- + +- The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file. (`#3504 `_) + + +Documentation Changes +--------------------- + +- (`#3659 `_) +- Documentation now has its own towncrier category. (`#3664 `_) +- `tox -e docs` will treat warnings about docs as errors. (`#3666 `_) +- The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. (`#3677 `_) +- A cheatsheet-style document for contributors was created at CONTRIBUTORS.rst (`#3682 `_) +- Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat. (`#3721 `_) +- Tahoe-LAFS project is now registered with Libera.Chat IRC network. (`#3726 `_) +- Rewriting the installation guide for Tahoe-LAFS. (`#3747 `_) +- Documentation and installation links in the README have been fixed. (`#3749 `_) +- The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. (`#3764 `_) +- The Great Black Swamp proposed specification now includes a glossary. (`#3765 `_) +- The Great Black Swamp specification now allows parallel upload of immutable share data. (`#3769 `_) +- There is now a specification for the scheme which Tahoe-LAFS storage clients use to derive their lease renewal secrets. (`#3774 `_) +- The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. (`#3777 `_) +- tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. (`#3782 `_) +- The Great Black Swamp specification now describes the required authorization scheme. (`#3785 `_) + + +Removed Features +---------------- + +- Announcements delivered through the introducer system are no longer automatically annotated with copious information about the Tahoe-LAFS software version nor the versions of its dependencies. (`#3518 `_) +- The stats gatherer, broken since at least Tahoe-LAFS 1.13.0, has been removed. The ``[client]stats_gatherer.furl`` configuration item in ``tahoe.cfg`` is no longer allowed. The Tahoe-LAFS project recommends using a third-party metrics aggregation tool instead. (`#3549 `_) +- The deprecated ``tahoe`` start, restart, stop, and daemonize sub-commands have been removed. (`#3550 `_) +- FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead. (`#3583 `_) +- Removed support for the Account Server frontend authentication type. (`#3652 `_) + + +Other Changes +------------- + +- The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. (`#3037 `_) +- The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. (`#3503 `_) +- The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. (`#3545 `_) +- The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. (`#3644 `_) +- You can run `make livehtml` in docs directory to invoke sphinx-autobuild. (`#3663 `_) +- Refactored test_introducer in web tests to use custom base test cases (`#3757 `_) + + +Misc/Other +---------- + +- `#2928 `_, `#3283 `_, `#3314 `_, `#3384 `_, `#3385 `_, `#3390 `_, `#3404 `_, `#3428 `_, `#3432 `_, `#3434 `_, `#3435 `_, `#3454 `_, `#3459 `_, `#3460 `_, `#3465 `_, `#3466 `_, `#3467 `_, `#3468 `_, `#3470 `_, `#3471 `_, `#3472 `_, `#3473 `_, `#3474 `_, `#3475 `_, `#3477 `_, `#3478 `_, `#3479 `_, `#3481 `_, `#3482 `_, `#3483 `_, `#3485 `_, `#3488 `_, `#3490 `_, `#3491 `_, `#3492 `_, `#3493 `_, `#3496 `_, `#3499 `_, `#3500 `_, `#3501 `_, `#3502 `_, `#3511 `_, `#3513 `_, `#3514 `_, `#3515 `_, `#3517 `_, `#3520 `_, `#3521 `_, `#3522 `_, `#3523 `_, `#3524 `_, `#3528 `_, `#3529 `_, `#3532 `_, `#3533 `_, `#3534 `_, `#3536 `_, `#3537 `_, `#3542 `_, `#3544 `_, `#3546 `_, `#3547 `_, `#3551 `_, `#3552 `_, `#3553 `_, `#3555 `_, `#3557 `_, `#3558 `_, `#3560 `_, `#3563 `_, `#3564 `_, `#3565 `_, `#3566 `_, `#3567 `_, `#3568 `_, `#3572 `_, `#3574 `_, `#3575 `_, `#3576 `_, `#3577 `_, `#3578 `_, `#3579 `_, `#3580 `_, `#3582 `_, `#3587 `_, `#3588 `_, `#3589 `_, `#3591 `_, `#3592 `_, `#3593 `_, `#3594 `_, `#3595 `_, `#3596 `_, `#3599 `_, `#3600 `_, `#3603 `_, `#3605 `_, `#3606 `_, `#3607 `_, `#3608 `_, `#3611 `_, `#3612 `_, `#3613 `_, `#3615 `_, `#3616 `_, `#3617 `_, `#3618 `_, `#3619 `_, `#3620 `_, `#3621 `_, `#3623 `_, `#3624 `_, `#3625 `_, `#3626 `_, `#3628 `_, `#3630 `_, `#3631 `_, `#3632 `_, `#3634 `_, `#3635 `_, `#3637 `_, `#3638 `_, `#3640 `_, `#3642 `_, `#3645 `_, `#3646 `_, `#3647 `_, `#3648 `_, `#3649 `_, `#3651 `_, `#3653 `_, `#3654 `_, `#3655 `_, `#3656 `_, `#3657 `_, `#3658 `_, `#3662 `_, `#3667 `_, `#3669 `_, `#3670 `_, `#3671 `_, `#3672 `_, `#3674 `_, `#3675 `_, `#3676 `_, `#3678 `_, `#3679 `_, `#3681 `_, `#3683 `_, `#3686 `_, `#3687 `_, `#3691 `_, `#3692 `_, `#3699 `_, `#3700 `_, `#3701 `_, `#3702 `_, `#3703 `_, `#3704 `_, `#3705 `_, `#3707 `_, `#3708 `_, `#3709 `_, `#3711 `_, `#3713 `_, `#3714 `_, `#3715 `_, `#3717 `_, `#3718 `_, `#3722 `_, `#3723 `_, `#3727 `_, `#3728 `_, `#3729 `_, `#3730 `_, `#3731 `_, `#3732 `_, `#3734 `_, `#3735 `_, `#3736 `_, `#3741 `_, `#3743 `_, `#3744 `_, `#3745 `_, `#3746 `_, `#3751 `_, `#3759 `_, `#3760 `_, `#3763 `_, `#3773 `_, `#3781 `_ + + Release 1.15.1 '''''''''''''' diff --git a/newsfragments/1549.installation b/newsfragments/1549.installation deleted file mode 100644 index cbb91cea5..000000000 --- a/newsfragments/1549.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now requires Twisted 19.10.0 or newer. As a result, it now has a transitive dependency on bcrypt. diff --git a/newsfragments/2928.minor b/newsfragments/2928.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3037.other b/newsfragments/3037.other deleted file mode 100644 index 947dc8f60..000000000 --- a/newsfragments/3037.other +++ /dev/null @@ -1 +0,0 @@ -The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. \ No newline at end of file diff --git a/newsfragments/3283.minor b/newsfragments/3283.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3314.minor b/newsfragments/3314.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3326.installation b/newsfragments/3326.installation deleted file mode 100644 index 2a3a64e32..000000000 --- a/newsfragments/3326.installation +++ /dev/null @@ -1 +0,0 @@ -Debian 8 support has been replaced with Debian 10 support. diff --git a/newsfragments/3384.minor b/newsfragments/3384.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3385.minor b/newsfragments/3385.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3390.minor b/newsfragments/3390.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature deleted file mode 100644 index d30a91679..000000000 --- a/newsfragments/3399.feature +++ /dev/null @@ -1 +0,0 @@ -Added 'typechecks' environment for tox running mypy and performing static typechecks. diff --git a/newsfragments/3404.minor b/newsfragments/3404.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3428.minor b/newsfragments/3428.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3432.minor b/newsfragments/3432.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3433.installation b/newsfragments/3433.installation deleted file mode 100644 index 3c06e53d3..000000000 --- a/newsfragments/3433.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS no longer depends on Nevow. \ No newline at end of file diff --git a/newsfragments/3434.minor b/newsfragments/3434.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3435.minor b/newsfragments/3435.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3454.minor b/newsfragments/3454.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3459.minor b/newsfragments/3459.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3460.minor b/newsfragments/3460.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3465.minor b/newsfragments/3465.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3466.minor b/newsfragments/3466.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3467.minor b/newsfragments/3467.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3468.minor b/newsfragments/3468.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3470.minor b/newsfragments/3470.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3471.minor b/newsfragments/3471.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3472.minor b/newsfragments/3472.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3473.minor b/newsfragments/3473.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3474.minor b/newsfragments/3474.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3475.minor b/newsfragments/3475.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3477.minor b/newsfragments/3477.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3478.minor b/newsfragments/3478.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3478.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3479.minor b/newsfragments/3479.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3481.minor b/newsfragments/3481.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3482.minor b/newsfragments/3482.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3483.minor b/newsfragments/3483.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3485.minor b/newsfragments/3485.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3486.installation b/newsfragments/3486.installation deleted file mode 100644 index 7b24956b2..000000000 --- a/newsfragments/3486.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now requires the `netifaces` Python package and no longer requires the external `ip`, `ifconfig`, or `route.exe` executables. diff --git a/newsfragments/3488.minor b/newsfragments/3488.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3490.minor b/newsfragments/3490.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3491.minor b/newsfragments/3491.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3492.minor b/newsfragments/3492.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3493.minor b/newsfragments/3493.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3496.minor b/newsfragments/3496.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3497.installation b/newsfragments/3497.installation deleted file mode 100644 index 4a50be97e..000000000 --- a/newsfragments/3497.installation +++ /dev/null @@ -1 +0,0 @@ -The Tahoe-LAFS project no longer commits to maintaining binary packages for all dependencies at . Please use PyPI instead. diff --git a/newsfragments/3499.minor b/newsfragments/3499.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3500.minor b/newsfragments/3500.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3501.minor b/newsfragments/3501.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3502.minor b/newsfragments/3502.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3503.other b/newsfragments/3503.other deleted file mode 100644 index 5d0c681b6..000000000 --- a/newsfragments/3503.other +++ /dev/null @@ -1 +0,0 @@ -The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. diff --git a/newsfragments/3504.configuration b/newsfragments/3504.configuration deleted file mode 100644 index 9ff74482c..000000000 --- a/newsfragments/3504.configuration +++ /dev/null @@ -1 +0,0 @@ -The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file. \ No newline at end of file diff --git a/newsfragments/3509.bugfix b/newsfragments/3509.bugfix deleted file mode 100644 index 4d633feab..000000000 --- a/newsfragments/3509.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression that broke flogtool results on Python 2. \ No newline at end of file diff --git a/newsfragments/3510.bugfix b/newsfragments/3510.bugfix deleted file mode 100644 index d4a2bd5dc..000000000 --- a/newsfragments/3510.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a logging regression on Python 2 involving unicode strings. \ No newline at end of file diff --git a/newsfragments/3511.minor b/newsfragments/3511.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3513.minor b/newsfragments/3513.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3514.minor b/newsfragments/3514.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3515.minor b/newsfragments/3515.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3517.minor b/newsfragments/3517.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3518.removed b/newsfragments/3518.removed deleted file mode 100644 index 460af5142..000000000 --- a/newsfragments/3518.removed +++ /dev/null @@ -1 +0,0 @@ -Announcements delivered through the introducer system are no longer automatically annotated with copious information about the Tahoe-LAFS software version nor the versions of its dependencies. diff --git a/newsfragments/3520.minor b/newsfragments/3520.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3521.minor b/newsfragments/3521.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3522.minor b/newsfragments/3522.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3523.minor b/newsfragments/3523.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3524.minor b/newsfragments/3524.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3528.minor b/newsfragments/3528.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3529.minor b/newsfragments/3529.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3532.minor b/newsfragments/3532.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3533.minor b/newsfragments/3533.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3534.minor b/newsfragments/3534.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3536.minor b/newsfragments/3536.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3537.minor b/newsfragments/3537.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3539.bugfix b/newsfragments/3539.bugfix deleted file mode 100644 index ed4aeb9af..000000000 --- a/newsfragments/3539.bugfix +++ /dev/null @@ -1 +0,0 @@ -Certain implementation-internal weakref KeyErrors are now handled and should no longer cause user-initiated operations to fail. diff --git a/newsfragments/3542.minor b/newsfragments/3542.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3544.minor b/newsfragments/3544.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3545.other b/newsfragments/3545.other deleted file mode 100644 index fd8adc37b..000000000 --- a/newsfragments/3545.other +++ /dev/null @@ -1 +0,0 @@ -The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. \ No newline at end of file diff --git a/newsfragments/3546.minor b/newsfragments/3546.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3547.minor b/newsfragments/3547.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3549.removed b/newsfragments/3549.removed deleted file mode 100644 index 53c7a7de1..000000000 --- a/newsfragments/3549.removed +++ /dev/null @@ -1 +0,0 @@ -The stats gatherer, broken since at least Tahoe-LAFS 1.13.0, has been removed. The ``[client]stats_gatherer.furl`` configuration item in ``tahoe.cfg`` is no longer allowed. The Tahoe-LAFS project recommends using a third-party metrics aggregation tool instead. diff --git a/newsfragments/3550.removed b/newsfragments/3550.removed deleted file mode 100644 index 2074bf676..000000000 --- a/newsfragments/3550.removed +++ /dev/null @@ -1 +0,0 @@ -The deprecated ``tahoe`` start, restart, stop, and daemonize sub-commands have been removed. \ No newline at end of file diff --git a/newsfragments/3551.minor b/newsfragments/3551.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3552.minor b/newsfragments/3552.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3553.minor b/newsfragments/3553.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3555.minor b/newsfragments/3555.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3557.minor b/newsfragments/3557.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3558.minor b/newsfragments/3558.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3560.minor b/newsfragments/3560.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3564.minor b/newsfragments/3564.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3565.minor b/newsfragments/3565.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3566.minor b/newsfragments/3566.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3567.minor b/newsfragments/3567.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3568.minor b/newsfragments/3568.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3572.minor b/newsfragments/3572.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3574.minor b/newsfragments/3574.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3575.minor b/newsfragments/3575.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3576.minor b/newsfragments/3576.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3577.minor b/newsfragments/3577.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3578.minor b/newsfragments/3578.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3579.minor b/newsfragments/3579.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3580.minor b/newsfragments/3580.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3582.minor b/newsfragments/3582.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3583.removed b/newsfragments/3583.removed deleted file mode 100644 index a3fce48be..000000000 --- a/newsfragments/3583.removed +++ /dev/null @@ -1 +0,0 @@ -FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead. \ No newline at end of file diff --git a/newsfragments/3584.bugfix b/newsfragments/3584.bugfix deleted file mode 100644 index faf57713b..000000000 --- a/newsfragments/3584.bugfix +++ /dev/null @@ -1 +0,0 @@ -SFTP public key auth likely works more consistently, and SFTP in general was previously broken. \ No newline at end of file diff --git a/newsfragments/3587.minor b/newsfragments/3587.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3587.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3588.incompat b/newsfragments/3588.incompat deleted file mode 100644 index 402ae8479..000000000 --- a/newsfragments/3588.incompat +++ /dev/null @@ -1 +0,0 @@ -The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale. diff --git a/newsfragments/3588.minor b/newsfragments/3588.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3590.bugfix b/newsfragments/3590.bugfix deleted file mode 100644 index aa504a5e3..000000000 --- a/newsfragments/3590.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work. \ No newline at end of file diff --git a/newsfragments/3591.minor b/newsfragments/3591.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3592.minor b/newsfragments/3592.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3593.minor b/newsfragments/3593.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3594.minor b/newsfragments/3594.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3595.minor b/newsfragments/3595.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3596.minor b/newsfragments/3596.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3599.minor b/newsfragments/3599.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3600.minor b/newsfragments/3600.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3603.minor.rst b/newsfragments/3603.minor.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3605.minor b/newsfragments/3605.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3606.minor b/newsfragments/3606.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3607.minor b/newsfragments/3607.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3608.minor b/newsfragments/3608.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3611.minor b/newsfragments/3611.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3612.minor b/newsfragments/3612.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3613.minor b/newsfragments/3613.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3615.minor b/newsfragments/3615.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3616.minor b/newsfragments/3616.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3617.minor b/newsfragments/3617.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3618.minor b/newsfragments/3618.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3619.minor b/newsfragments/3619.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3620.minor b/newsfragments/3620.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3621.minor b/newsfragments/3621.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3623.minor b/newsfragments/3623.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3623.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3624.minor b/newsfragments/3624.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3625.minor b/newsfragments/3625.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3626.minor b/newsfragments/3626.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3628.minor b/newsfragments/3628.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3629.feature b/newsfragments/3629.feature deleted file mode 100644 index cdca48a18..000000000 --- a/newsfragments/3629.feature +++ /dev/null @@ -1 +0,0 @@ -The NixOS-packaged Tahoe-LAFS now knows its own version. diff --git a/newsfragments/3630.minor b/newsfragments/3630.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3631.minor b/newsfragments/3631.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3632.minor b/newsfragments/3632.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3633.installation b/newsfragments/3633.installation deleted file mode 100644 index 8f6d7efdd..000000000 --- a/newsfragments/3633.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support. diff --git a/newsfragments/3634.minor b/newsfragments/3634.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3635.minor b/newsfragments/3635.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3637.minor b/newsfragments/3637.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3638.minor b/newsfragments/3638.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3640.minor b/newsfragments/3640.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3642.minor b/newsfragments/3642.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3644.other b/newsfragments/3644.other deleted file mode 100644 index 4b159e45d..000000000 --- a/newsfragments/3644.other +++ /dev/null @@ -1 +0,0 @@ -The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. \ No newline at end of file diff --git a/newsfragments/3645.minor b/newsfragments/3645.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3646.minor b/newsfragments/3646.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3647.minor b/newsfragments/3647.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3648.minor b/newsfragments/3648.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3649.minor b/newsfragments/3649.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3650.bugfix b/newsfragments/3650.bugfix deleted file mode 100644 index 09a810239..000000000 --- a/newsfragments/3650.bugfix +++ /dev/null @@ -1 +0,0 @@ -``tahoe invite`` will now read share encoding/placement configuration values from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. diff --git a/newsfragments/3651.minor b/newsfragments/3651.minor deleted file mode 100644 index 9a2f5a0ed..000000000 --- a/newsfragments/3651.minor +++ /dev/null @@ -1 +0,0 @@ -We added documentation detailing the project's ticket triage process diff --git a/newsfragments/3652.removed b/newsfragments/3652.removed deleted file mode 100644 index a3e964702..000000000 --- a/newsfragments/3652.removed +++ /dev/null @@ -1 +0,0 @@ -Removed support for the Account Server frontend authentication type. diff --git a/newsfragments/3653.minor b/newsfragments/3653.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3654.minor b/newsfragments/3654.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3655.minor b/newsfragments/3655.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3656.minor b/newsfragments/3656.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3657.minor b/newsfragments/3657.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3658.minor b/newsfragments/3658.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3659.documentation b/newsfragments/3659.documentation deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3662.minor b/newsfragments/3662.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3663.other b/newsfragments/3663.other deleted file mode 100644 index 62abf2666..000000000 --- a/newsfragments/3663.other +++ /dev/null @@ -1 +0,0 @@ -You can run `make livehtml` in docs directory to invoke sphinx-autobuild. diff --git a/newsfragments/3664.documentation b/newsfragments/3664.documentation deleted file mode 100644 index ab5de8884..000000000 --- a/newsfragments/3664.documentation +++ /dev/null @@ -1 +0,0 @@ -Documentation now has its own towncrier category. diff --git a/newsfragments/3666.documentation b/newsfragments/3666.documentation deleted file mode 100644 index 3f9e34777..000000000 --- a/newsfragments/3666.documentation +++ /dev/null @@ -1 +0,0 @@ -`tox -e docs` will treat warnings about docs as errors. diff --git a/newsfragments/3667.minor b/newsfragments/3667.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3669.minor b/newsfragments/3669.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3670.minor b/newsfragments/3670.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3671.minor b/newsfragments/3671.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3672.minor b/newsfragments/3672.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3674.minor b/newsfragments/3674.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3675.minor b/newsfragments/3675.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3676.minor b/newsfragments/3676.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3677.documentation b/newsfragments/3677.documentation deleted file mode 100644 index 51730e765..000000000 --- a/newsfragments/3677.documentation +++ /dev/null @@ -1 +0,0 @@ -The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. diff --git a/newsfragments/3678.minor b/newsfragments/3678.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3679.minor b/newsfragments/3679.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3681.minor b/newsfragments/3681.minor deleted file mode 100644 index bc84b6b8f..000000000 --- a/newsfragments/3681.minor +++ /dev/null @@ -1,8 +0,0 @@ -(The below text is no longer valid: netifaces has released a 64-bit -Python 2.7 wheel for Windows. Ticket #3733 made the switch in CI. We -should be able to test and run Tahoe-LAFS without needing vcpython27 -now.) - -Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has -removed vcpython27 compiler downloads from their site, and Tahoe-LAFS -needs vcpython27 to build and install netifaces on 64-bit Windows. diff --git a/newsfragments/3682.documentation b/newsfragments/3682.documentation deleted file mode 100644 index 5cf78bd90..000000000 --- a/newsfragments/3682.documentation +++ /dev/null @@ -1 +0,0 @@ -A cheatsheet-style document for contributors was created at CONTRIBUTORS.rst \ No newline at end of file diff --git a/newsfragments/3683.minor b/newsfragments/3683.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3686.minor b/newsfragments/3686.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3687.minor b/newsfragments/3687.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3691.minor b/newsfragments/3691.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3692.minor b/newsfragments/3692.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3699.minor b/newsfragments/3699.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3700.minor b/newsfragments/3700.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3701.minor b/newsfragments/3701.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3702.minor b/newsfragments/3702.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3703.minor b/newsfragments/3703.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3704.minor b/newsfragments/3704.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3705.minor b/newsfragments/3705.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3707.minor b/newsfragments/3707.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3708.minor b/newsfragments/3708.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3711.minor b/newsfragments/3711.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3712.installation b/newsfragments/3712.installation deleted file mode 100644 index b80e1558b..000000000 --- a/newsfragments/3712.installation +++ /dev/null @@ -1 +0,0 @@ -The Nix package now includes correct version information. \ No newline at end of file diff --git a/newsfragments/3713.minor b/newsfragments/3713.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3714.minor b/newsfragments/3714.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3715.minor b/newsfragments/3715.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3716.incompat b/newsfragments/3716.incompat deleted file mode 100644 index aa03eea47..000000000 --- a/newsfragments/3716.incompat +++ /dev/null @@ -1 +0,0 @@ -tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. \ No newline at end of file diff --git a/newsfragments/3717.minor b/newsfragments/3717.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3718.minor b/newsfragments/3718.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3721.documentation b/newsfragments/3721.documentation deleted file mode 100644 index 36ae33236..000000000 --- a/newsfragments/3721.documentation +++ /dev/null @@ -1 +0,0 @@ -Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat. diff --git a/newsfragments/3722.minor b/newsfragments/3722.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3723.minor b/newsfragments/3723.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3726.documentation b/newsfragments/3726.documentation deleted file mode 100644 index fb94fff32..000000000 --- a/newsfragments/3726.documentation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS project is now registered with Libera.Chat IRC network. diff --git a/newsfragments/3727.minor b/newsfragments/3727.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3728.minor b/newsfragments/3728.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3729.minor b/newsfragments/3729.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3730.minor b/newsfragments/3730.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3731.minor b/newsfragments/3731.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3732.minor b/newsfragments/3732.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3733.installation b/newsfragments/3733.installation deleted file mode 100644 index c1cac649b..000000000 --- a/newsfragments/3733.installation +++ /dev/null @@ -1 +0,0 @@ -Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows. VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult. diff --git a/newsfragments/3734.minor b/newsfragments/3734.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3735.minor b/newsfragments/3735.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3736.minor b/newsfragments/3736.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3738.bugfix b/newsfragments/3738.bugfix deleted file mode 100644 index 6a4bc1cd9..000000000 --- a/newsfragments/3738.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression where uploading files with non-ASCII names failed. \ No newline at end of file diff --git a/newsfragments/3739.bugfix b/newsfragments/3739.bugfix deleted file mode 100644 index 875941cf8..000000000 --- a/newsfragments/3739.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed annoying UnicodeWarning message on Python 2 when running CLI tools. \ No newline at end of file diff --git a/newsfragments/3741.minor b/newsfragments/3741.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3743.minor b/newsfragments/3743.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3745.minor b/newsfragments/3745.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3746.minor b/newsfragments/3746.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3747.documentation b/newsfragments/3747.documentation deleted file mode 100644 index a2559a6a0..000000000 --- a/newsfragments/3747.documentation +++ /dev/null @@ -1 +0,0 @@ -Rewriting the installation guide for Tahoe-LAFS. diff --git a/newsfragments/3749.documentation b/newsfragments/3749.documentation deleted file mode 100644 index 554564a0b..000000000 --- a/newsfragments/3749.documentation +++ /dev/null @@ -1 +0,0 @@ -Documentation and installation links in the README have been fixed. diff --git a/newsfragments/3751.minor b/newsfragments/3751.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3757.other b/newsfragments/3757.other deleted file mode 100644 index 3d2d3f272..000000000 --- a/newsfragments/3757.other +++ /dev/null @@ -1 +0,0 @@ -Refactored test_introducer in web tests to use custom base test cases \ No newline at end of file diff --git a/newsfragments/3759.minor b/newsfragments/3759.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3760.minor b/newsfragments/3760.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3763.minor b/newsfragments/3763.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3764.documentation b/newsfragments/3764.documentation deleted file mode 100644 index d473cd27c..000000000 --- a/newsfragments/3764.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. \ No newline at end of file diff --git a/newsfragments/3765.documentation b/newsfragments/3765.documentation deleted file mode 100644 index a3b59c4d6..000000000 --- a/newsfragments/3765.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp proposed specification now includes a glossary. \ No newline at end of file diff --git a/newsfragments/3769.documentation b/newsfragments/3769.documentation deleted file mode 100644 index 3d4ef7d4c..000000000 --- a/newsfragments/3769.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp specification now allows parallel upload of immutable share data. diff --git a/newsfragments/3773.minor b/newsfragments/3773.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3774.documentation b/newsfragments/3774.documentation deleted file mode 100644 index d58105966..000000000 --- a/newsfragments/3774.documentation +++ /dev/null @@ -1 +0,0 @@ -There is now a specification for the scheme which Tahoe-LAFS storage clients use to derive their lease renewal secrets. diff --git a/newsfragments/3777.documentation b/newsfragments/3777.documentation deleted file mode 100644 index 7635cc1e6..000000000 --- a/newsfragments/3777.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. diff --git a/newsfragments/3779.bugfix b/newsfragments/3779.bugfix deleted file mode 100644 index 073046474..000000000 --- a/newsfragments/3779.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where share corruption events were not logged on storage servers running on Windows. \ No newline at end of file diff --git a/newsfragments/3781.minor b/newsfragments/3781.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3782.documentation b/newsfragments/3782.documentation deleted file mode 100644 index 5e5cecc13..000000000 --- a/newsfragments/3782.documentation +++ /dev/null @@ -1 +0,0 @@ -tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. diff --git a/newsfragments/3785.documentation b/newsfragments/3785.documentation deleted file mode 100644 index 4eb268f79..000000000 --- a/newsfragments/3785.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp specification now describes the required authorization scheme. From 0377c619cb358697e0379d664bc8eee487338f0b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 10:58:07 +0100 Subject: [PATCH 012/916] release 1.16.0-rc1 Signed-off-by: fenn-cs --- newsfragments/3754.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3754.minor diff --git a/newsfragments/3754.minor b/newsfragments/3754.minor new file mode 100644 index 000000000..e69de29bb From ac603c5e17f32cf18fcde479ab289329dfc3570b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:05:47 +0100 Subject: [PATCH 013/916] added proper release title Signed-off-by: fenn-cs --- NEWS.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index a7d814c70..366e45907 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,9 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.15.1.post2188.dev0 (2021-09-17)Release 1.15.1.post2188.dev0 (2021-09-17) -''''''''''''''''''''''''''''''''''''''''' + +Release 1.16.0 (2021-09-17) +''''''''''''''''''''''''''' Backwards Incompatible Changes ------------------------------ From 3d9644f42915fc3140bf88007020d6526b32f139 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:06:19 +0100 Subject: [PATCH 014/916] release notes for 1.16.0 Signed-off-by: fenn-cs --- relnotes.txt | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index 4afbd6cc5..c97b42664 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.15.1 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.16.0 -The Tahoe-LAFS team is pleased to announce version 1.15.1 of +The Tahoe-LAFS team is pleased to announce version 1.16.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -16,14 +16,23 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html The previous stable release of Tahoe-LAFS was v1.15.0, released on -January 19, 2021. +March 23rd, 2021. -In this release: PyPI does not accept uploads of packages that use -PEP-508 version specifiers. +The major change in this release is the completion of the Python 3 +port -- while maintaining support for Python 2. A future release will +remove Python 2 support. -Note that Python3 porting is underway but not yet complete in this -release. Developers may notice python3 as new targets for certain -tools. +The previously deprecated subcommands "start", "stop", "restart" and +"daemonize" have been removed. You must now use "tahoe run" (possibly +along with your favourite daemonization software). + +Several features are now removed: the Account Server, stats-gatherer +and FTP support. + +There are several dependency changes that will be interesting for +distribution maintainers. + +As well 196 bugs have been fixed since the last release. Please see ``NEWS.rst`` for a more complete list of changes. @@ -142,19 +151,19 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -meejah +fenn-cs on behalf of the Tahoe-LAFS team -March 23, 2021 +September 16, 2021 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.15.1/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.16.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 87ea676502cc5a231a2efabfc50f2cb7fd42d9bf Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:06:37 +0100 Subject: [PATCH 015/916] update nix version Signed-off-by: fenn-cs --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 35b29f1cc..2aff6af18 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.15.1). + # Most of the time this is not exactly the release version (eg 1.16.0). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.15.1.post1"; + version = "1.16.0.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From d26101c82528243af03d427e0c6f411d114572a1 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:07:24 +0100 Subject: [PATCH 016/916] acknowledge new contributors Signed-off-by: fenn-cs --- CREDITS | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CREDITS b/CREDITS index b0923fc35..8a6e876ec 100644 --- a/CREDITS +++ b/CREDITS @@ -240,3 +240,23 @@ N: Lukas Pirl E: tahoe@lukas-pirl.de W: http://lukas-pirl.de D: Buildslaves (Debian, Fedora, CentOS; 2016-2021) + +N: Anxhelo Lushka +E: anxhelo1995@gmail.com +D: Web site design and updates + +N: Fon E. Noel +E: fenn25.fn@gmail.com +D: bug-fixes and refactoring + +N: Jehad Baeth +E: jehad@leastauthority.com +D: Documentation improvement + +N: May-Lee Sia +E: mayleesia@gmail.com +D: Community-manager and documentation improvements + +N: Yash Nayani +E: yashaswi.nram@gmail.com +D: Installation Guide improvements From f6a96ae3976ee21ad0376f7b6a22fc3d12110dce Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:07:58 +0100 Subject: [PATCH 017/916] fix tarballs target for release Signed-off-by: fenn-cs --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9b0f71038..3aa6a6d43 100644 --- a/tox.ini +++ b/tox.ini @@ -258,7 +258,8 @@ commands= pyinstaller -y --clean pyinstaller.spec [testenv:tarballs] +basepython = python3 deps = commands = python setup.py update_version - python setup.py sdist --formats=bztar,gztar,zip bdist_wheel + python setup.py sdist --formats=bztar,gztar,zip bdist_wheel --universal From 5bd5ee580acd3d0a95b190074d2da1fc5d98975e Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 18 Sep 2021 23:50:34 +0100 Subject: [PATCH 018/916] layout for tests that check if log_call_deffered decorates parametized functions correctly Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 3f915ecd2..7db23ce9b 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -281,3 +281,44 @@ class LogCallDeferredTests(TestCase): ), ), ) + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_gets_positional_arguments(self, logger): + """ + Check that positional arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(a): + return a ** 2 + self.assertThat( + f(4), succeeded(Equals(16))) + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_gets_keyword_arguments(self, logger): + """ + Check that keyword arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(base, exp): + return base ** exp + self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) + + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_gets_keyword_and_positional_arguments(self, logger): + """ + Check that both keyword and positional arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(base, exp, message): + return base ** exp + self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) \ No newline at end of file From dd8aa8a66648a9582a8732afde704cebdd4b16fa Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 22 Sep 2021 23:37:33 +0100 Subject: [PATCH 019/916] test if log_call_deffered decorates parametized functions correctly Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 9 ++++++++- src/allmydata/util/eliotutil.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 7db23ce9b..7edd4e780 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -56,6 +56,7 @@ from eliot.testing import ( capture_logging, assertHasAction, swap_logger, + assertContainsFields, ) from twisted.internet.defer import ( @@ -295,6 +296,8 @@ class LogCallDeferredTests(TestCase): return a ** 2 self.assertThat( f(4), succeeded(Equals(16))) + msg = logger.messages[0] + assertContainsFields(self, msg, {"args": (4,)}) @capture_logging( lambda self, logger: @@ -308,6 +311,8 @@ class LogCallDeferredTests(TestCase): def f(base, exp): return base ** exp self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) + msg = logger.messages[0] + assertContainsFields(self, msg, {"base": 10, "exp": 2}) @capture_logging( @@ -321,4 +326,6 @@ class LogCallDeferredTests(TestCase): @log_call_deferred(action_type=u"the-action") def f(base, exp, message): return base ** exp - self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) \ No newline at end of file + self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) + msg = logger.messages[0] + assertContainsFields(self, msg, {"args": (10, 2), "message": "an exponential function"}) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index d989c9e2a..ac2d3e4e0 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -324,8 +324,8 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - args = {k: bytes_to_unicode(True, kw[k]) for k in kw} - with start_action(action_type=action_type, **args).context(): + kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} + with start_action(action_type=action_type, args=a, **kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From 88cbb7b109946709fb93e982a73b4b4e84fda595 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 24 Sep 2021 23:04:01 +0100 Subject: [PATCH 020/916] remove methods that break test_filenode with AsyncBrokenTest Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_filenode.py | 95 +++++++++++---------- 1 file changed, 52 insertions(+), 43 deletions(-) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index de03afc5a..748df1fde 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -13,6 +13,14 @@ if PY2: from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor from twisted.trial import unittest +from ..common import AsyncTestCase, AsyncBrokenTestCase +from testtools.matchers import ( + Equals, + Contains, + HasLength, + Is, + IsInstance, +) from allmydata import uri, client from allmydata.util.consumer import MemoryConsumer from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION, DownloadStopped @@ -29,12 +37,13 @@ from .util import ( make_peer, ) -class Filenode(unittest.TestCase, testutil.ShouldFailMixin): +class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # this used to be in Publish, but we removed the limit. Some of # these tests test whether the new code correctly allows files # larger than the limit. OLD_MAX_SEGMENT_SIZE = 3500000 def setUp(self): + super(Filenode, self).setUp() self._storage = FakeStorage() self._peers = list( make_peer(self._storage, n) @@ -48,12 +57,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create(self): d = self.nodemaker.create_mutable_file() def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker peer0 = sorted(sb.get_all_serverids())[0] shnums = self._storage._peers[peer0].keys() - self.failUnlessEqual(len(shnums), 1) + self.assertThat(shnums, HasLength(1)) d.addCallback(_created) return d @@ -61,12 +70,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_mdmf(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker peer0 = sorted(sb.get_all_serverids())[0] shnums = self._storage._peers[peer0].keys() - self.failUnlessEqual(len(shnums), 1) + self.assertThat(shnums, HasLength(1)) d.addCallback(_created) return d @@ -80,7 +89,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored, v=v: self.nodemaker.create_mutable_file(version=v)) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) self._node = n return n d.addCallback(_created) @@ -89,19 +98,19 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: self._node.download_best_version()) d.addCallback(lambda contents: - self.failUnlessEqual(contents, b"Contents" * 50000)) + self.assertThat(contents, Equals(b"Contents" * 50000))) return d def test_max_shares(self): self.nodemaker.default_encoding_parameters['n'] = 255 d = self.nodemaker.create_mutable_file(version=SDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker num_shares = sum([len(self._storage._peers[x].keys()) for x \ in sb.get_all_serverids()]) - self.failUnlessEqual(num_shares, 255) + self.assertThat(num_shares, Equals(255)) self._node = n return n d.addCallback(_created) @@ -121,12 +130,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): self.nodemaker.default_encoding_parameters['n'] = 255 d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker num_shares = sum([len(self._storage._peers[x].keys()) for x \ in sb.get_all_serverids()]) - self.failUnlessEqual(num_shares, 255) + self.assertThat(num_shares, Equals(255)) self._node = n return n d.addCallback(_created) @@ -135,20 +144,20 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: self._node.download_best_version()) d.addCallback(lambda contents: - self.failUnlessEqual(contents, b"contents" * 50000)) + self.assertThat(contents, Equals(b"contents" * 50000))) return d def test_mdmf_filenode_cap(self): # Test that an MDMF filenode, once created, returns an MDMF URI. d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) cap = n.get_cap() - self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI)) + self.assertThat(cap, IsInstance(uri.WriteableMDMFFileURI)) rcap = n.get_readcap() - self.failUnless(isinstance(rcap, uri.ReadonlyMDMFFileURI)) + self.assertThat(rcap, IsInstance(uri.ReadonlyMDMFFileURI)) vcap = n.get_verify_cap() - self.failUnless(isinstance(vcap, uri.MDMFVerifierURI)) + self.assertThat(vcap, IsInstance(uri.MDMFVerifierURI)) d.addCallback(_created) return d @@ -158,13 +167,13 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): # filenode given an MDMF cap. d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_uri() self.failUnless(s.startswith(b"URI:MDMF")) n2 = self.nodemaker.create_from_cap(s) - self.failUnless(isinstance(n2, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n2.get_storage_index()) - self.failUnlessEqual(n.get_uri(), n2.get_uri()) + self.assertThat(n2, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index())) + self.assertThat(n.get_uri(), Equals(n2.get_uri())) d.addCallback(_created) return d @@ -172,10 +181,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_from_mdmf_readcap(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_readonly_uri() n2 = self.nodemaker.create_from_cap(s) - self.failUnless(isinstance(n2, MutableFileNode)) + self.assertThat(n2, IsInstance(MutableFileNode)) # Check that it's a readonly node self.failUnless(n2.is_readonly()) @@ -191,10 +200,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): self.uri = n.get_uri() - self.failUnlessEqual(n._protocol_version, MDMF_VERSION) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) n2 = self.nodemaker.create_from_cap(self.uri) - self.failUnlessEqual(n2._protocol_version, MDMF_VERSION) + self.assertThat(n2._protocol_version, Equals(MDMF_VERSION)) d.addCallback(_created) return d @@ -203,14 +212,14 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): n = MutableFileNode(None, None, {"k": 3, "n": 10}, None) calls = [] def _callback(*args, **kwargs): - self.failUnlessEqual(args, (4,) ) - self.failUnlessEqual(kwargs, {"foo": 5}) + self.assertThat(args, Equals((4,))) + self.assertThat(kwargs, Equals({"foo": 5})) calls.append(1) return 6 d = n._do_serialized(_callback, 4, foo=5) def _check_callback(res): - self.failUnlessEqual(res, 6) - self.failUnlessEqual(calls, [1]) + self.assertThat(res, Equals(6)) + self.assertThat(calls, Equals([1])) d.addCallback(_check_callback) def _errback(): @@ -229,24 +238,24 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda sio: self.failUnless("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) - d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) d.addCallback(lambda res: n.get_size_of_best_version()) d.addCallback(lambda size: - self.failUnlessEqual(size, len(b"contents 1"))) + self.assertThat(size, Equals(len(b"contents 1")))) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) d.addCallback(lambda res: n.get_servermap(MODE_WRITE)) d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING)) d.addCallback(lambda smap: n.download_version(smap, smap.best_recoverable_version())) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) # test a file that is large enough to overcome the # mapupdate-to-retrieve data caching (i.e. make the shares larger # than the default readsize, which is 2000 bytes). A 15kB file @@ -254,7 +263,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"large size file" * 1000))) d.addCallback(lambda res: n.download_best_version()) d.addCallback(lambda res: - self.failUnlessEqual(res, b"large size file" * 1000)) + self.assertThat(res, Equals(b"large size file" * 1000))) return d d.addCallback(_created) return d @@ -268,7 +277,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): n.get_servermap(MODE_READ)) def _then(servermap): dumped = servermap.dump(StringIO()) - self.failUnlessIn("3-of-10", dumped.getvalue()) + self.assertThat(dumped.getvalue(), Contains("3-of-10")) d.addCallback(_then) # Now overwrite the contents with some new contents. We want # to make them big enough to force the file to be uploaded @@ -431,7 +440,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_with_initial_contents_function(self): data = b"initial contents" def _make_contents(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() self.failUnless(isinstance(key, bytes), key) self.failUnlessEqual(len(key), 16) # AES key size @@ -447,7 +456,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_mdmf_with_initial_contents_function(self): data = b"initial contents" * 100000 def _make_contents(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() self.failUnless(isinstance(key, bytes), key) self.failUnlessEqual(len(key), 16) @@ -643,7 +652,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda sio: self.failUnless("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) - d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) From 49b6080097e12a9150db45534b81cfee48f39b9d Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 25 Sep 2021 21:03:01 +0100 Subject: [PATCH 021/916] remove depracated assert methods Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_filenode.py | 95 ++++++++++----------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 748df1fde..579734433 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -12,8 +12,7 @@ if PY2: from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor -from twisted.trial import unittest -from ..common import AsyncTestCase, AsyncBrokenTestCase +from ..common import AsyncBrokenTestCase from testtools.matchers import ( Equals, Contains, @@ -122,7 +121,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self._node.download_best_version()) # ...and check to make sure everything went okay. d.addCallback(lambda contents: - self.failUnlessEqual(b"contents" * 50000, contents)) + self.assertThat(b"contents" * 50000, Equals(contents))) return d def test_max_shares_mdmf(self): @@ -169,7 +168,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_uri() - self.failUnless(s.startswith(b"URI:MDMF")) + self.assertTrue(s.startswith(b"URI:MDMF")) n2 = self.nodemaker.create_from_cap(s) self.assertThat(n2, IsInstance(MutableFileNode)) self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index())) @@ -187,7 +186,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self.assertThat(n2, IsInstance(MutableFileNode)) # Check that it's a readonly node - self.failUnless(n2.is_readonly()) + self.assertTrue(n2.is_readonly()) d.addCallback(_created) return d @@ -236,7 +235,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.get_servermap(MODE_READ)) d.addCallback(lambda smap: smap.dump(StringIO())) d.addCallback(lambda sio: - self.failUnless("3-of-10" in sio.getvalue())) + self.assertTrue("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) @@ -289,7 +288,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, big_contents)) + self.assertThat(data, Equals(big_contents))) # Overwrite the contents again with some new contents. As # before, they need to be big enough to force multiple # segments, so that we make the downloader deal with @@ -301,7 +300,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, bigger_contents)) + self.assertThat(data, Equals(bigger_contents))) return d d.addCallback(_created) return d @@ -332,7 +331,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # Now we'll retrieve it into a pausing consumer. c = PausingConsumer() d = version.read(c) - d.addCallback(lambda ign: self.failUnlessEqual(c.size, len(data))) + d.addCallback(lambda ign: self.assertThat(c.size, Equals(len(data)))) c2 = PausingAndStoppingConsumer() d.addCallback(lambda ign: @@ -369,14 +368,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self.uri = node.get_uri() # also confirm that the cap has no extension fields pieces = self.uri.split(b":") - self.failUnlessEqual(len(pieces), 4) + self.assertThat(pieces, HasLength(4)) return node.overwrite(MutableData(b"contents1" * 100000)) def _then(ignored): node = self.nodemaker.create_from_cap(self.uri) return node.download_best_version() def _downloaded(data): - self.failUnlessEqual(data, b"contents1" * 100000) + self.assertThat(data, Equals(b"contents1" * 100000)) d.addCallback(_created) d.addCallback(_then) d.addCallback(_downloaded) @@ -406,11 +405,11 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d = self.nodemaker.create_mutable_file(upload1) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) upload2 = MutableData(b"contents 2") d.addCallback(lambda res: n.overwrite(upload2)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) return d d.addCallback(_created) return d @@ -424,15 +423,15 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.download_best_version() d.addCallback(lambda data: - self.failUnlessEqual(data, initial_contents)) + self.assertThat(data, Equals(initial_contents))) uploadable2 = MutableData(initial_contents + b"foobarbaz") d.addCallback(lambda ignored: n.overwrite(uploadable2)) d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, initial_contents + - b"foobarbaz")) + self.assertThat(data, Equals(initial_contents + + b"foobarbaz"))) return d d.addCallback(_created) return d @@ -442,14 +441,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _make_contents(n): self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() - self.failUnless(isinstance(key, bytes), key) - self.failUnlessEqual(len(key), 16) # AES key size + self.assertTrue(isinstance(key, bytes), key) + self.assertThat(key, HasLength(16)) # AES key size return MutableData(data) d = self.nodemaker.create_mutable_file(_make_contents) def _created(n): return n.download_best_version() d.addCallback(_created) - d.addCallback(lambda data2: self.failUnlessEqual(data2, data)) + d.addCallback(lambda data2: self.assertThat(data2, Equals(data))) return d @@ -458,15 +457,15 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _make_contents(n): self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() - self.failUnless(isinstance(key, bytes), key) - self.failUnlessEqual(len(key), 16) + self.assertTrue(isinstance(key, bytes), key) + self.assertThat(key, HasLength(16)) return MutableData(data) d = self.nodemaker.create_mutable_file(_make_contents, version=MDMF_VERSION) d.addCallback(lambda n: n.download_best_version()) d.addCallback(lambda data2: - self.failUnlessEqual(data2, data)) + self.assertThat(data2, Equals(data))) return d @@ -485,7 +484,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d = n.get_servermap(MODE_READ) d.addCallback(lambda servermap: servermap.best_recoverable_version()) d.addCallback(lambda verinfo: - self.failUnlessEqual(verinfo[0], expected_seqnum, which)) + self.assertThat(verinfo[0], Equals(expected_seqnum), which)) return d def test_modify(self): @@ -522,36 +521,36 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.modify(_modifier) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m")) d.addCallback(lambda res: n.modify(_non_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "non")) d.addCallback(lambda res: n.modify(_none_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "none")) d.addCallback(lambda res: self.shouldFail(ValueError, "error_modifier", None, n.modify, _error_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "err")) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "big")) d.addCallback(lambda res: n.modify(_ucw_error_modifier)) - d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2)) + d.addCallback(lambda res: self.assertThat(calls, HasLength(2))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "ucw")) def _reset_ucw_error_modifier(res): @@ -566,10 +565,10 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # will only be one larger than the previous test, not two (i.e. 4 # instead of 5). d.addCallback(lambda res: n.modify(_ucw_error_non_modifier)) - d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2)) + d.addCallback(lambda res: self.assertThat(calls, HasLength(2))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 4, "ucw")) d.addCallback(lambda res: n.modify(_toobig_modifier)) return d @@ -605,7 +604,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.modify(_modifier) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m")) d.addCallback(lambda res: @@ -614,7 +613,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): n.modify, _ucw_error_modifier, _backoff_stopper)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "stop")) def _reset_ucw_error_modifier(res): @@ -624,8 +623,8 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.modify(_ucw_error_modifier, _backoff_pauser)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "pause")) d.addCallback(lambda res: @@ -634,8 +633,8 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): n.modify, _always_ucw_error_modifier, giveuper.delay)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "giveup")) return d @@ -650,23 +649,23 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.get_servermap(MODE_READ)) d.addCallback(lambda smap: smap.dump(StringIO())) d.addCallback(lambda sio: - self.failUnless("3-of-10" in sio.getvalue())) + self.assertTrue("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) d.addCallback(lambda res: n.get_servermap(MODE_WRITE)) d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING)) d.addCallback(lambda smap: n.download_version(smap, smap.best_recoverable_version())) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) return d d.addCallback(_created) return d @@ -682,14 +681,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): return n.get_servermap(MODE_READ) d.addCallback(_created) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 0)) + self.assertThat(self.n.get_size(), Equals(0))) d.addCallback(lambda ignored: self.n.overwrite(MutableData(b"foobarbaz"))) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 9)) + self.assertThat(self.n.get_size(), Equals(9))) d.addCallback(lambda ignored: self.nodemaker.create_mutable_file(MutableData(b"foobarbaz"))) d.addCallback(_created) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 9)) + self.assertThat(self.n.get_size(), Equals(9))) return d From 759d4c85a295cfd620aacbc55225ee1d8aa236b2 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 28 Sep 2021 09:56:14 +0100 Subject: [PATCH 022/916] avoid argument collision in call of start_action in eliotutil Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 5 +++-- src/allmydata/test/web/test_logs.py | 2 +- src/allmydata/util/eliotutil.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 7edd4e780..1fbb9ec8d 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -312,7 +312,7 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"base": 10, "exp": 2}) + assertContainsFields(self, msg, {"kwargs": {"base": 10, "exp": 2}}) @capture_logging( @@ -328,4 +328,5 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2), "message": "an exponential function"}) + assertContainsFields(self, msg, {"args": (10, 2)}) + assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 043541690..fe0a0445d 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -121,7 +121,7 @@ class TestStreamingLogs(AsyncTestCase): self.assertThat(len(messages), Equals(3)) self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) - self.assertThat(messages[0]["arguments"], + self.assertThat(messages[0]["kwargs"]["arguments"], Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) self.assertThat(messages[1]["action_type"], Equals("test:cli:some-exciting-action")) self.assertThat("started", Equals(messages[0]["action_status"])) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ac2d3e4e0..e0c2fd8ae 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -325,7 +325,7 @@ def log_call_deferred(action_type): # Use the action's context method to avoid ending the action when # the `with` block ends. kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} - with start_action(action_type=action_type, args=a, **kwargs).context(): + with start_action(action_type=action_type, args=a, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From 5803d9999d3763e1cc7ac746273cf2873cede646 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 11 Oct 2021 13:49:29 +0100 Subject: [PATCH 023/916] remove unseriable args in log_call_deferred passed to start_action Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 4 ++-- src/allmydata/util/eliotutil.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 1fbb9ec8d..3d7b2bd42 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -297,7 +297,7 @@ class LogCallDeferredTests(TestCase): self.assertThat( f(4), succeeded(Equals(16))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (4,)}) + assertContainsFields(self, msg, {"args": {'arg_0': 4}}) @capture_logging( lambda self, logger: @@ -328,5 +328,5 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2)}) + assertContainsFields(self, msg, {"args": {'arg_0': 10, 'arg_1': 2}}) assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index e0c2fd8ae..fb18ed332 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -91,7 +91,7 @@ from .jsonbytes import ( AnyBytesJSONEncoder, bytes_to_unicode ) - +import json def validateInstanceOf(t): @@ -315,6 +315,14 @@ class _DestinationParser(object): _parse_destination_description = _DestinationParser().parse +def is_json_serializable(object): + try: + json.dumps(object) + return True + except (TypeError, OverflowError): + return False + + def log_call_deferred(action_type): """ Like ``eliot.log_call`` but for functions which return ``Deferred``. @@ -325,7 +333,14 @@ def log_call_deferred(action_type): # Use the action's context method to avoid ending the action when # the `with` block ends. kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} - with start_action(action_type=action_type, args=a, kwargs=kwargs).context(): + # Remove complex (unserializable) objects from positional args to + # prevent eliot from throwing errors when it attempts serialization + args = { + "arg_" + str(pos): bytes_to_unicode(True, a[pos]) + for pos in range(len(a)) + if is_json_serializable(a[pos]) + } + with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) @@ -339,3 +354,5 @@ if PY2: capture_logging = eliot_capture_logging else: capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) + + From 984b4ac45e89c3f53b5325006e0d33b15165ab26 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Oct 2021 13:56:14 -0400 Subject: [PATCH 024/916] News file. --- newsfragments/3812.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3812.minor diff --git a/newsfragments/3812.minor b/newsfragments/3812.minor new file mode 100644 index 000000000..e69de29bb From 57a0f76e1f6ed5116c1defa221ff93241f60291d Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 13 Oct 2021 23:41:42 +0100 Subject: [PATCH 025/916] maintain list of positional arguments as tuple Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 4 ++-- src/allmydata/util/eliotutil.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 3d7b2bd42..1fbb9ec8d 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -297,7 +297,7 @@ class LogCallDeferredTests(TestCase): self.assertThat( f(4), succeeded(Equals(16))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": {'arg_0': 4}}) + assertContainsFields(self, msg, {"args": (4,)}) @capture_logging( lambda self, logger: @@ -328,5 +328,5 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": {'arg_0': 10, 'arg_1': 2}}) + assertContainsFields(self, msg, {"args": (10, 2)}) assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index fb18ed332..997dadb8d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -335,11 +335,7 @@ def log_call_deferred(action_type): kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} # Remove complex (unserializable) objects from positional args to # prevent eliot from throwing errors when it attempts serialization - args = { - "arg_" + str(pos): bytes_to_unicode(True, a[pos]) - for pos in range(len(a)) - if is_json_serializable(a[pos]) - } + args = tuple([a[pos] for pos in range(len(a)) if is_json_serializable(a[pos])]) with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. From efc9dc831bb1f0f2915657294084bc4fb7e84d91 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 14 Oct 2021 11:01:37 -0400 Subject: [PATCH 026/916] Revert "a stab at using setup.cfg and setuptools_scm" This reverts commit 68e8e0a7d5b88568bd01c1a14957f972830d1f54. --- CLASSIFIERS.txt | 29 --- setup.cfg | 215 +--------------- setup.py | 429 ++++++++++++++++++++++++++++++- src/allmydata/__init__.py | 40 +-- pyproject.toml => towncrier.toml | 10 - tox.ini | 8 +- 6 files changed, 462 insertions(+), 269 deletions(-) delete mode 100644 CLASSIFIERS.txt rename pyproject.toml => towncrier.toml (80%) diff --git a/CLASSIFIERS.txt b/CLASSIFIERS.txt deleted file mode 100644 index 7aa2e35b1..000000000 --- a/CLASSIFIERS.txt +++ /dev/null @@ -1,29 +0,0 @@ -Development Status :: 5 - Production/Stable -Environment :: Console -Environment :: Web Environment -License :: OSI Approved :: GNU General Public License (GPL) -License :: DFSG approved -License :: Other/Proprietary License -Intended Audience :: Developers -Intended Audience :: End Users/Desktop -Intended Audience :: System Administrators -Operating System :: Microsoft -Operating System :: Microsoft :: Windows -Operating System :: Unix -Operating System :: POSIX :: Linux -Operating System :: POSIX -Operating System :: MacOS :: MacOS X -Operating System :: OS Independent -Natural Language :: English -Programming Language :: C -Programming Language :: Python -Programming Language :: Python :: 2 -Programming Language :: Python :: 2.7 -Topic :: Utilities -Topic :: System :: Systems Administration -Topic :: System :: Filesystems -Topic :: System :: Distributed Computing -Topic :: Software Development :: Libraries -Topic :: System :: Archiving :: Backup -Topic :: System :: Archiving :: Mirroring -Topic :: System :: Archiving diff --git a/setup.cfg b/setup.cfg index 21e1fb663..f4539279e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,211 +1,10 @@ -# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html#declarative-config -[metadata] -name = tahoe-lafs - -[options] -version = attr:allmydata._version.version -description = secure, decentralized, fault-tolerant file store -long_description = file: README.rst -author = the Tahoe-LAFS project -author_email = tahoe-dev@lists.tahoe-lafs.org -project_urls = - website=https://tahoe-lafs.org/ - documentation=https://tahoe-lafs.readthedocs.org/ - source=https://github.com/tahoe-lafs/tahoe-lafs -classifiers = file: CLASSIFIERS.txt -license_files = COPYING.GPL, COPYING.TGPPL.rst -# tell setuptools to find our package source files automatically -packages = find: -# find packages beneath the src directory -package_dir= - =src -include_package_data = True - -# We support Python 2.7 and many newer versions of Python 3. -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* - -install_requires = - # we don't need much out of setuptools but the version checking stuff - # needs pkg_resources and PEP 440 version specifiers. - setuptools >= 28.8.0 - - zfec >= 1.1.0 - - # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. - zope.interface >= 3.6.0 - - # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for - # transferring large mutable files of size N. - # * foolscap < 0.6 is incompatible with Twisted 10.2.0. - # * foolscap 0.6.1 quiets a DeprecationWarning. - # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. - # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, - # rather than 1024-bit RSA-with-MD5. This also allows us to work - # with a FIPS build of OpenSSL. - # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, - # and allocate_tcp_port - # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo - # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs - # * foolscap 0.13.2 drops i2p support completely - # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - foolscap == 0.13.1 ; python_version < '3.0' - foolscap >= 21.7.0 ; python_version > '3.0' - - # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that - # Twisted[conch] also depends on cryptography and Twisted[tls] - # transitively depends on cryptography. So it's anyone's guess what - # version of cryptography will *really* be installed. - cryptography >= 2.6 - - # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server - # rekeying bug - # * The SFTP frontend and manhole depend on the conch extra. However, we - # can't explicitly declare that without an undesirable dependency on gmpy, - # as explained in ticket #2740. - # * Due to a setuptools bug, we need to declare a dependency on the tls - # extra even though we only depend on it via foolscap. - # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. - # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, - # which otherwise causes test_system to fail (DirtyReactorError, due to - # leftover timers) - # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed - # for coverage testing - # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch - # extra, letting us use that extra instead of trying to duplicate its - # dependencies here. Twisted[conch] >18.7 introduces a dependency on - # bcrypt. It is nice to avoid that if the user ends up with an older - # version of Twisted. That's hard to express except by using the extra. - # - # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the - # * initializer, needed by logic in our custom `Request` subclass. - # - # In a perfect world, Twisted[conch] would be a dependency of an sftp - # extra. However, pip fails to resolve the dependencies all - # dependencies when asked for Twisted[tls] *and* Twisted[conch]. - # Specifically, Twisted[conch] (as the later requirement) is ignored. - # If there were an Tahoe-LAFS sftp extra that dependended on - # Twisted[conch] and install_requires only included Twisted[tls] then - # `pip install tahoe-lafs[sftp]` would not install requirements - # specified by Twisted[conch]. Since this would be the *whole point* of - # an sftp extra in Tahoe-LAFS, there is no point in having one. - # * Twisted 19.10 introduces Site.getContentFile which we use to get - # temporary upload files placed into a per-node temporary directory. - Twisted[tls,conch] >= 19.10.0 - - PyYAML >= 3.11 - - six >= 1.10.0 - - # for 'tahoe invite' and 'tahoe join' - magic-wormhole >= 0.10.2 - - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - eliot ~= 1.7 ; python_version < '3.0' - # On Python 3, we want a new enough version to support custom JSON encoders. - eliot >= 1.13.0 ; python_version > '3.0' - - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - pyrsistent < 0.17.0 ; python_version < '3.0' - pyrsistent ; python_version > '3.0' - - # A great way to define types of values. - attrs >= 18.2.0 - - # WebSocket library for twisted and asyncio - autobahn >= 19.5.2 - - # Support for Python 3 transition - future >= 0.18.2 - - # Discover local network configuration - netifaces - - # Utility code: - pyutil >= 3.3.0 - - # Linux distribution detection: - distro >= 1.4.0 - - # Backported configparser for Python 2: - configparser ; python_version < '3.0' - - # For the RangeMap datastructure. - collections-extended - - # Duplicate the Twisted pywin32 dependency here. See - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some - # discussion. - pywin32 != 226 ; sys_platform=="win32" - -[options.packages.find] -# inform the setuptools source discovery logic to start in this directory -where = src - -[options.package_data] -allmydata.web = - *.xhtml - static/*.js - static/*.png - static/*.css - static/img/*.png - static/css/*.css - -[options.extras_require] -test = - flake8 - # Pin a specific pyflakes so we don't have different folks - # disagreeing on what is or is not a lint issue. We can bump - # this version from time to time, but we will do it - # intentionally. - pyflakes == 2.2.0 - coverage ~= 5.0 - mock - tox - pytest - pytest-twisted - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - decorator < 5 - hypothesis >= 3.6.1 - treq - towncrier - testtools - fixtures - beautifulsoup4 - html5lib - junitxml - tenacity - paramiko - pytest-timeout - # Does our OpenMetrics endpoint adhere to the spec: - prometheus-client == 0.11.0 - - # Make sure the tor and i2p tests can run by duplicating the requirements - # for those extras here. - %(tor)s - %(i2p)s - -tor = - # This is exactly what `foolscap[tor]` means but pip resolves the pair of - # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose - # this if we don't declare it ourselves! - txtorcon >= 0.17.0 - -i2p = - # txi2p has Python 3 support in master branch, but it has not been - # released -- see https://github.com/str4d/txi2p/issues/10. We - # could use a fork for Python 3 until txi2p's maintainers are back - # in action. For Python 2, we could continue using the txi2p - # version about which no one has complained to us so far. - txi2p; python_version < '3.0' - txi2p-tahoe >= 0.3.5; python_version > '3.0' - -[options.entry_points] -console_scripts = - tahoe = allmydata.scripts.runner:run +[aliases] +build = update_version build +sdist = update_version sdist +install = update_version install +develop = update_version develop +bdist_egg = update_version bdist_egg +bdist_wheel = update_version bdist_wheel [flake8] # Enforce all pyflakes constraints, and also prohibit tabs for indentation. diff --git a/setup.py b/setup.py index 8bf1ba938..8c6396937 100644 --- a/setup.py +++ b/setup.py @@ -1,2 +1,427 @@ -from setuptools import setup -setup() +#! /usr/bin/env python +# -*- coding: utf-8 -*- +import sys + +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2006-2012 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +import os, subprocess, re +from io import open + +basedir = os.path.dirname(os.path.abspath(__file__)) + +# locate our version number + +def read_version_py(infname): + try: + verstrline = open(infname, "rt").read() + except EnvironmentError: + return None + else: + VSRE = r"^verstr = ['\"]([^'\"]*)['\"]" + mo = re.search(VSRE, verstrline, re.M) + if mo: + return mo.group(1) + +VERSION_PY_FILENAME = 'src/allmydata/_version.py' +version = read_version_py(VERSION_PY_FILENAME) + +install_requires = [ + # we don't need much out of setuptools but the version checking stuff + # needs pkg_resources and PEP 440 version specifiers. + "setuptools >= 28.8.0", + + "zfec >= 1.1.0", + + # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. + "zope.interface >= 3.6.0", + + # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for + # transferring large mutable files of size N. + # * foolscap < 0.6 is incompatible with Twisted 10.2.0. + # * foolscap 0.6.1 quiets a DeprecationWarning. + # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. + # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, + # rather than 1024-bit RSA-with-MD5. This also allows us to work + # with a FIPS build of OpenSSL. + # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, + # and allocate_tcp_port + # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo + # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs + # * foolscap 0.13.2 drops i2p support completely + # * foolscap >= 21.7 is necessary for Python 3 with i2p support. + "foolscap == 0.13.1 ; python_version < '3.0'", + "foolscap >= 21.7.0 ; python_version > '3.0'", + + # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that + # Twisted[conch] also depends on cryptography and Twisted[tls] + # transitively depends on cryptography. So it's anyone's guess what + # version of cryptography will *really* be installed. + "cryptography >= 2.6", + + # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server + # rekeying bug + # * The SFTP frontend and manhole depend on the conch extra. However, we + # can't explicitly declare that without an undesirable dependency on gmpy, + # as explained in ticket #2740. + # * Due to a setuptools bug, we need to declare a dependency on the tls + # extra even though we only depend on it via foolscap. + # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. + # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, + # which otherwise causes test_system to fail (DirtyReactorError, due to + # leftover timers) + # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed + # for coverage testing + # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch + # extra, letting us use that extra instead of trying to duplicate its + # dependencies here. Twisted[conch] >18.7 introduces a dependency on + # bcrypt. It is nice to avoid that if the user ends up with an older + # version of Twisted. That's hard to express except by using the extra. + # + # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the + # * initializer, needed by logic in our custom `Request` subclass. + # + # In a perfect world, Twisted[conch] would be a dependency of an "sftp" + # extra. However, pip fails to resolve the dependencies all + # dependencies when asked for Twisted[tls] *and* Twisted[conch]. + # Specifically, "Twisted[conch]" (as the later requirement) is ignored. + # If there were an Tahoe-LAFS sftp extra that dependended on + # Twisted[conch] and install_requires only included Twisted[tls] then + # `pip install tahoe-lafs[sftp]` would not install requirements + # specified by Twisted[conch]. Since this would be the *whole point* of + # an sftp extra in Tahoe-LAFS, there is no point in having one. + # * Twisted 19.10 introduces Site.getContentFile which we use to get + # temporary upload files placed into a per-node temporary directory. + "Twisted[tls,conch] >= 19.10.0", + + "PyYAML >= 3.11", + + "six >= 1.10.0", + + # for 'tahoe invite' and 'tahoe join' + "magic-wormhole >= 0.10.2", + + # Eliot is contemplating dropping Python 2 support. Stick to a version we + # know works on Python 2.7. + "eliot ~= 1.7 ; python_version < '3.0'", + # On Python 3, we want a new enough version to support custom JSON encoders. + "eliot >= 1.13.0 ; python_version > '3.0'", + + # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped + # Python 2 entirely; stick to the version known to work for us. + "pyrsistent < 0.17.0 ; python_version < '3.0'", + "pyrsistent ; python_version > '3.0'", + + # A great way to define types of values. + "attrs >= 18.2.0", + + # WebSocket library for twisted and asyncio + "autobahn >= 19.5.2", + + # Support for Python 3 transition + "future >= 0.18.2", + + # Discover local network configuration + "netifaces", + + # Utility code: + "pyutil >= 3.3.0", + + # Linux distribution detection: + "distro >= 1.4.0", + + # Backported configparser for Python 2: + "configparser ; python_version < '3.0'", + + # For the RangeMap datastructure. + "collections-extended", +] + +setup_requires = [ + 'setuptools >= 28.8.0', # for PEP-440 style versions +] + +tor_requires = [ + # This is exactly what `foolscap[tor]` means but pip resolves the pair of + # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose + # this if we don't declare it ourselves! + "txtorcon >= 0.17.0", +] + +i2p_requires = [ + # txi2p has Python 3 support in master branch, but it has not been + # released -- see https://github.com/str4d/txi2p/issues/10. We + # could use a fork for Python 3 until txi2p's maintainers are back + # in action. For Python 2, we could continue using the txi2p + # version about which no one has complained to us so far. + "txi2p; python_version < '3.0'", + "txi2p-tahoe >= 0.3.5; python_version > '3.0'", +] + +if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': + del sys.argv[1] + install_requires += ["fakedependency >= 1.0.0"] + +from setuptools import find_packages, setup +from setuptools import Command +from setuptools.command import install + + +trove_classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "License :: OSI Approved :: GNU General Public License (GPL)", + "License :: DFSG approved", + "License :: Other/Proprietary License", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: System Administrators", + "Operating System :: Microsoft", + "Operating System :: Microsoft :: Windows", + "Operating System :: Unix", + "Operating System :: POSIX :: Linux", + "Operating System :: POSIX", + "Operating System :: MacOS :: MacOS X", + "Operating System :: OS Independent", + "Natural Language :: English", + "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Topic :: Utilities", + "Topic :: System :: Systems Administration", + "Topic :: System :: Filesystems", + "Topic :: System :: Distributed Computing", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Archiving :: Backup", + "Topic :: System :: Archiving :: Mirroring", + "Topic :: System :: Archiving", + ] + + +GIT_VERSION_BODY = ''' +# This _version.py is generated from git metadata by the tahoe setup.py. + +__pkgname__ = "%(pkgname)s" +real_version = "%(version)s" +full_version = "%(full)s" +branch = "%(branch)s" +verstr = "%(normalized)s" +__version__ = verstr +''' + +def run_command(args, cwd=None): + use_shell = sys.platform == "win32" + try: + p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ + print("Warning: unable to run %r." % (" ".join(args),)) + print(e) + return None + stdout = p.communicate()[0].strip() + if p.returncode != 0: + print("Warning: %r returned error code %r." % (" ".join(args), p.returncode)) + return None + return stdout + + +def versions_from_git(tag_prefix): + # This runs 'git' from the directory that contains this file. That either + # means someone ran a setup.py command (and this code is in + # versioneer.py, thus the containing directory is the root of the source + # tree), or someone ran a project-specific entry point (and this code is + # in _version.py, thus the containing directory is somewhere deeper in + # the source tree). This only gets called if the git-archive 'subst' + # variables were *not* expanded, and _version.py hasn't already been + # rewritten with a short version string, meaning we're inside a checked + # out source tree. + + # versions_from_git (as copied from python-versioneer) returns strings + # like "1.9.0-25-gb73aba9-dirty", which means we're in a tree with + # uncommited changes (-dirty), the latest checkin is revision b73aba9, + # the most recent tag was 1.9.0, and b73aba9 has 25 commits that weren't + # in 1.9.0 . The narrow-minded NormalizedVersion parser that takes our + # output (meant to enable sorting of version strings) refuses most of + # that. Tahoe uses a function named suggest_normalized_version() that can + # handle "1.9.0.post25", so dumb down our output to match. + + try: + source_dir = os.path.dirname(os.path.abspath(__file__)) + except NameError as e: + # some py2exe/bbfreeze/non-CPython implementations don't do __file__ + print("Warning: unable to find version because we could not obtain the source directory.") + print(e) + return {} + stdout = run_command(["git", "describe", "--tags", "--dirty", "--always"], + cwd=source_dir) + if stdout is None: + # run_command already complained. + return {} + stdout = stdout.decode("ascii") + if not stdout.startswith(tag_prefix): + print("Warning: tag %r doesn't start with prefix %r." % (stdout, tag_prefix)) + return {} + version = stdout[len(tag_prefix):] + pieces = version.split("-") + if len(pieces) == 1: + normalized_version = pieces[0] + else: + normalized_version = "%s.post%s" % (pieces[0], pieces[1]) + + stdout = run_command(["git", "rev-parse", "HEAD"], cwd=source_dir) + if stdout is None: + # run_command already complained. + return {} + full = stdout.decode("ascii").strip() + if version.endswith("-dirty"): + full += "-dirty" + normalized_version += ".dev0" + + # Thanks to Jistanidiot at . + stdout = run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=source_dir) + branch = (stdout or b"unknown").decode("ascii").strip() + + # this returns native strings (bytes on py2, unicode on py3) + return {"version": version, "normalized": normalized_version, + "full": full, "branch": branch} + +# setup.cfg has an [aliases] section which runs "update_version" before many +# commands (like "build" and "sdist") that need to know our package version +# ahead of time. If you add different commands (or if we forgot some), you +# may need to add it to setup.cfg and configure it to run update_version +# before your command. + +class UpdateVersion(Command): + description = "update _version.py from revision-control metadata" + user_options = install.install.user_options + + def initialize_options(self): + pass + def finalize_options(self): + pass + def run(self): + global version + verstr = version + if os.path.isdir(os.path.join(basedir, ".git")): + verstr = self.try_from_git() + + if verstr: + self.distribution.metadata.version = verstr + else: + print("""\ +******************************************************************** +Warning: no version information found. This may cause tests to fail. +******************************************************************** +""") + + def try_from_git(self): + # If we change the release tag names, we must change this too + versions = versions_from_git("tahoe-lafs-") + + # setup.py might be run by either py2 or py3 (when run by tox, which + # uses py3 on modern debian/ubuntu distros). We want this generated + # file to contain native strings on both (str=bytes in py2, + # str=unicode in py3) + if versions: + body = GIT_VERSION_BODY % { + "pkgname": self.distribution.get_name(), + "version": versions["version"], + "normalized": versions["normalized"], + "full": versions["full"], + "branch": versions["branch"], + } + f = open(VERSION_PY_FILENAME, "wb") + f.write(body.encode("ascii")) + f.close() + print("Wrote normalized version %r into '%s'" % (versions["normalized"], VERSION_PY_FILENAME)) + + return versions.get("normalized", None) + +class PleaseUseTox(Command): + user_options = [] + def initialize_options(self): + pass + def finalize_options(self): + pass + + def run(self): + print("ERROR: Please use 'tox' to run the test suite.") + sys.exit(1) + +setup_args = {} +if version: + setup_args["version"] = version + +setup(name="tahoe-lafs", # also set in __init__.py + description='secure, decentralized, fault-tolerant file store', + long_description=open('README.rst', 'r', encoding='utf-8').read(), + author='the Tahoe-LAFS project', + author_email='tahoe-dev@lists.tahoe-lafs.org', + url='https://tahoe-lafs.org/', + license='GNU GPL', # see README.rst -- there is an alternative licence + cmdclass={"update_version": UpdateVersion, + "test": PleaseUseTox, + }, + package_dir = {'':'src'}, + packages=find_packages('src') + ['allmydata.test.plugins'], + classifiers=trove_classifiers, + # We support Python 2.7, and we're working on support for 3.6 (the + # highest version that PyPy currently supports). + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + install_requires=install_requires, + extras_require={ + # Duplicate the Twisted pywin32 dependency here. See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some + # discussion. + ':sys_platform=="win32"': ["pywin32 != 226"], + "test": [ + "flake8", + # Pin a specific pyflakes so we don't have different folks + # disagreeing on what is or is not a lint issue. We can bump + # this version from time to time, but we will do it + # intentionally. + "pyflakes == 2.2.0", + "coverage ~= 5.0", + "mock", + "tox", + "pytest", + "pytest-twisted", + # XXX: decorator isn't a direct dependency, but pytest-twisted + # depends on decorator, and decorator 5.x isn't compatible with + # Python 2.7. + "decorator < 5", + "hypothesis >= 3.6.1", + "treq", + "towncrier", + "testtools", + "fixtures", + "beautifulsoup4", + "html5lib", + "junitxml", + "tenacity", + "paramiko", + "pytest-timeout", + # Does our OpenMetrics endpoint adhere to the spec: + "prometheus-client == 0.11.0", + ] + tor_requires + i2p_requires, + "tor": tor_requires, + "i2p": i2p_requires, + }, + package_data={"allmydata.web": ["*.xhtml", + "static/*.js", "static/*.png", "static/*.css", + "static/img/*.png", + "static/css/*.css", + ], + "allmydata": ["ported-modules.txt"], + }, + include_package_data=True, + setup_requires=setup_requires, + entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, + **setup_args + ) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 42611810b..333394fc5 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -16,28 +16,36 @@ if PY2: __all__ = [ "__version__", + "full_version", + "branch", "__appname__", "__full_version__", ] -def _discover_version(): - try: - from allmydata._version import version - except ImportError: - # Perhaps we're running from a git checkout where the _version.py file - # hasn't been generated yet. Try to discover the version using git - # information instead. - try: - import setuptools_scm - return setuptools_scm.get_version() - except Exception: - return "unknown" - else: - return version +__version__ = "unknown" +try: + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore +except ImportError: + # We're running in a tree that hasn't run update_version, and didn't + # come with a _version.py, so we don't know what our version is. + # This should not happen very often. + pass + +full_version = "unknown" +branch = "unknown" +try: + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore +except ImportError: + # We're running in a tree that hasn't run update_version, and didn't + # come with a _version.py, so we don't know what our full version or + # branch is. This should not happen very often. + pass __appname__ = "tahoe-lafs" -__version__ = _discover_version() -del _discover_version # __full_version__ is the one that you ought to use when identifying yourself # in the "application" part of the Tahoe versioning scheme: diff --git a/pyproject.toml b/towncrier.toml similarity index 80% rename from pyproject.toml rename to towncrier.toml index 7c97001aa..b8b561a98 100644 --- a/pyproject.toml +++ b/towncrier.toml @@ -1,13 +1,3 @@ -# https://setuptools.pypa.io/en/latest/build_meta.html -# https://github.com/pypa/setuptools_scm -[build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"] -build-backend = "setuptools.build_meta" - -[tool.setuptools_scm] -write_to = "src/allmydata/_version.py" -tag_regex = "^tahoe-lafs-(?P[vV]?\\d+(?:\\.\\d+){0,2}[^\\+]*)(?:\\+.*)?$" - [tool.towncrier] package_dir = "src" package = "allmydata" diff --git a/tox.ini b/tox.ini index 3aa6116b8..610570be5 100644 --- a/tox.ini +++ b/tox.ini @@ -148,8 +148,8 @@ commands = # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See pyproject.toml for legal values. - python -m towncrier.check --config pyproject.toml + # file. See towncrier.toml for legal values. + python -m towncrier.check --config towncrier.toml [testenv:typechecks] @@ -177,7 +177,7 @@ deps = certifi towncrier==21.3.0 commands = - python -m towncrier --draft --config pyproject.toml + python -m towncrier --draft --config towncrier.toml [testenv:news] # On macOS, git invoked from Tox needs $HOME. @@ -189,7 +189,7 @@ deps = certifi towncrier==21.3.0 commands = - python -m towncrier --yes --config pyproject.toml + python -m towncrier --yes --config towncrier.toml # commit the changes git commit -m "update NEWS.txt for release" From 22aab98fcf2cf873b87b158e6a819a799f5e912b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Oct 2021 12:51:24 -0400 Subject: [PATCH 027/916] When callRemoteOnly as removed, these probably should've been changed to return a Deferred. --- src/allmydata/immutable/downloader/share.py | 2 +- src/allmydata/immutable/layout.py | 2 +- src/allmydata/storage_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 41e11426f..016f1c34d 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -475,7 +475,7 @@ class Share(object): # there was corruption somewhere in the given range reason = "corruption in share[%d-%d): %s" % (start, start+offset, str(f.value)) - self._rref.callRemote( + return self._rref.callRemote( "advise_corrupt_share", reason.encode("utf-8") ).addErrback(log.err, "Error from remote call to advise_corrupt_share") diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 6c7362b8a..79c886237 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -254,7 +254,7 @@ class WriteBucketProxy(object): return d def abort(self): - self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket") + return self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket") def get_servername(self): return self._server.get_name() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ac6c107d5..526e4e70d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1017,7 +1017,7 @@ class _StorageServer(object): shnum, reason, ): - self._rref.callRemote( + return self._rref.callRemote( "advise_corrupt_share", share_type, storage_index, From e099bc6736383bb8a66538e35bb1a2aa3c1c6cfc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Oct 2021 12:52:56 -0400 Subject: [PATCH 028/916] Tests for IStorageServer.advise_corrupt_share. --- src/allmydata/test/test_istorageserver.py | 54 ++++++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 40dcdc8bb..bd056ae13 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -20,7 +20,7 @@ if PY2: from random import Random -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import inlineCallbacks, returnValue from foolscap.api import Referenceable, RemoteException @@ -405,12 +405,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) @inlineCallbacks - def test_bucket_advise_corrupt_share(self): - """ - Calling ``advise_corrupt_share()`` on a bucket returned by - ``IStorageServer.get_buckets()`` does not result in error (other - behavior is opaque at this level of abstraction). - """ + def create_share(self): + """Create a share, return the storage index.""" storage_index = new_storage_index() (_, allocated) = yield self.storage_server.allocate_buckets( storage_index, @@ -423,10 +419,31 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, b"0123456789") yield allocated[0].callRemote("close") + returnValue(storage_index) + @inlineCallbacks + def test_bucket_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a bucket returned by + ``IStorageServer.get_buckets()`` does not result in error (other + behavior is opaque at this level of abstraction). + """ + storage_index = yield self.create_share() buckets = yield self.storage_server.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") + @inlineCallbacks + def test_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on an immutable share does not + result in error (other behavior is opaque at this level of + abstraction). + """ + storage_index = yield self.create_share() + yield self.storage_server.advise_corrupt_share( + b"immutable", storage_index, 0, b"ono" + ) + class IStorageServerMutableAPIsTestsMixin(object): """ @@ -780,6 +797,29 @@ class IStorageServerMutableAPIsTestsMixin(object): {0: [b"abcdefg"], 1: [b"0123456"], 2: [b"9876543"]}, ) + @inlineCallbacks + def test_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a mutable share does not + result in error (other behavior is opaque at this level of + abstraction). + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"abcdefg")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + yield self.storage_server.advise_corrupt_share( + b"mutable", storage_index, 0, b"ono" + ) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 1a12a8acdffea5491e223e82cafc557b1d8dbda6 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 15 Oct 2021 00:50:11 +0100 Subject: [PATCH 029/916] don't throw away unserializable parameter Signed-off-by: fenn-cs --- src/allmydata/util/eliotutil.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 997dadb8d..f2272a731 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -335,7 +335,12 @@ def log_call_deferred(action_type): kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} # Remove complex (unserializable) objects from positional args to # prevent eliot from throwing errors when it attempts serialization - args = tuple([a[pos] for pos in range(len(a)) if is_json_serializable(a[pos])]) + args = tuple( + a[pos] + if is_json_serializable(a[pos]) + else str(a[pos]) + for pos in range(len(a)) + ) with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. From 347377aaab20b0a806af2661bca8093f5644fea8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:43:34 -0400 Subject: [PATCH 030/916] Get rid of `check_memory` which depends on the control port This was some kind of memory usage analysis tool. It depends on the control port so it cannot work after I delete the control port. The code itself is messy, undocumented, and has no automated tests. I don't know if it works at all anymore. Even if it does, no one ever runs it. Measuring Tahoe-LAFS' memory usage over the course of maintenance and development is a lovely idea but the project has not managed to adopt (or maintain?) that practice based on this tool. Given sufficient interest we can resurrect this idea using a more streamlined process and less invasive tools in the future. --- .gitignore | 1 - misc/checkers/check_memory.py | 522 ---------------------------------- tox.ini | 11 - 3 files changed, 534 deletions(-) delete mode 100644 misc/checkers/check_memory.py diff --git a/.gitignore b/.gitignore index d6a58b88b..50a1352a2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ zope.interface-*.egg /src/allmydata/test/plugins/dropin.cache /_trial_temp* -/_test_memory/ /tmp* /*.patch /dist/ diff --git a/misc/checkers/check_memory.py b/misc/checkers/check_memory.py deleted file mode 100644 index 268d77451..000000000 --- a/misc/checkers/check_memory.py +++ /dev/null @@ -1,522 +0,0 @@ -from __future__ import print_function - -import os, shutil, sys, urllib, time, stat, urlparse - -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 -from six.moves import cStringIO as StringIO - -from twisted.python.filepath import ( - FilePath, -) -from twisted.internet import defer, reactor, protocol, error -from twisted.application import service, internet -from twisted.web import client as tw_client -from twisted.python import log, procutils -from foolscap.api import Tub, fireEventually, flushEventualQueue - -from allmydata import client, introducer -from allmydata.immutable import upload -from allmydata.scripts import create_node -from allmydata.util import fileutil, pollmixin -from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_filesystem_encoding - -from allmydata.scripts.common import ( - write_introducer, -) - -class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter, object): - full_speed_ahead = False - _bytes_so_far = 0 - stalled = None - def handleResponsePart(self, data): - self._bytes_so_far += len(data) - if not self.factory.do_stall: - return - if self.full_speed_ahead: - return - if self._bytes_so_far > 1e6+100: - if not self.stalled: - print("STALLING") - self.transport.pauseProducing() - self.stalled = reactor.callLater(10.0, self._resume_speed) - def _resume_speed(self): - print("RESUME SPEED") - self.stalled = None - self.full_speed_ahead = True - self.transport.resumeProducing() - def handleResponseEnd(self): - if self.stalled: - print("CANCEL") - self.stalled.cancel() - self.stalled = None - return tw_client.HTTPPageGetter.handleResponseEnd(self) - -class StallableDiscardingHTTPClientFactory(tw_client.HTTPClientFactory, object): - protocol = StallableHTTPGetterDiscarder - -def discardPage(url, stall=False, *args, **kwargs): - """Start fetching the URL, but stall our pipe after the first 1MB. - Wait 10 seconds, then resume downloading (and discarding) everything. - """ - # adapted from twisted.web.client.getPage . We can't just wrap or - # subclass because it provides no way to override the HTTPClientFactory - # that it creates. - scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) - assert scheme == 'http' - host, port = netloc, 80 - if ":" in host: - host, port = host.split(":") - port = int(port) - factory = StallableDiscardingHTTPClientFactory(url, *args, **kwargs) - factory.do_stall = stall - reactor.connectTCP(host, port, factory) - return factory.deferred - -class ChildDidNotStartError(Exception): - pass - -class SystemFramework(pollmixin.PollMixin): - numnodes = 7 - - def __init__(self, basedir, mode): - self.basedir = basedir = abspath_expanduser_unicode(str(basedir)) - if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep): - raise AssertionError("safety issue: basedir must be a subdir") - self.testdir = testdir = os.path.join(basedir, "test") - if os.path.exists(testdir): - shutil.rmtree(testdir) - fileutil.make_dirs(testdir) - self.sparent = service.MultiService() - self.sparent.startService() - self.proc = None - self.tub = Tub() - self.tub.setOption("expose-remote-exception-types", False) - self.tub.setServiceParent(self.sparent) - self.mode = mode - self.failed = False - self.keepalive_file = None - - def run(self): - framelog = os.path.join(self.basedir, "driver.log") - log.startLogging(open(framelog, "a"), setStdout=False) - log.msg("CHECK_MEMORY(mode=%s) STARTING" % self.mode) - #logfile = open(os.path.join(self.testdir, "log"), "w") - #flo = log.FileLogObserver(logfile) - #log.startLoggingWithObserver(flo.emit, setStdout=False) - d = fireEventually() - d.addCallback(lambda res: self.setUp()) - d.addCallback(lambda res: self.record_initial_memusage()) - d.addCallback(lambda res: self.make_nodes()) - d.addCallback(lambda res: self.wait_for_client_connected()) - d.addCallback(lambda res: self.do_test()) - d.addBoth(self.tearDown) - def _err(err): - self.failed = err - log.err(err) - print(err) - d.addErrback(_err) - def _done(res): - reactor.stop() - return res - d.addBoth(_done) - reactor.run() - if self.failed: - # raiseException doesn't work for CopiedFailures - self.failed.raiseException() - - def setUp(self): - #print("STARTING") - self.stats = {} - self.statsfile = open(os.path.join(self.basedir, "stats.out"), "a") - self.make_introducer() - d = self.start_client() - def _record_control_furl(control_furl): - self.control_furl = control_furl - #print("OBTAINING '%s'" % (control_furl,)) - return self.tub.getReference(self.control_furl) - d.addCallback(_record_control_furl) - def _record_control(control_rref): - self.control_rref = control_rref - d.addCallback(_record_control) - def _ready(res): - #print("CLIENT READY") - pass - d.addCallback(_ready) - return d - - def record_initial_memusage(self): - print() - print("Client started (no connections yet)") - d = self._print_usage() - d.addCallback(self.stash_stats, "init") - return d - - def wait_for_client_connected(self): - print() - print("Client connecting to other nodes..") - return self.control_rref.callRemote("wait_for_client_connections", - self.numnodes+1) - - def tearDown(self, passthrough): - # the client node will shut down in a few seconds - #os.remove(os.path.join(self.clientdir, client.Client.EXIT_TRIGGER_FILE)) - log.msg("shutting down SystemTest services") - if self.keepalive_file and os.path.exists(self.keepalive_file): - age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME] - log.msg("keepalive file at shutdown was %ds old" % age) - d = defer.succeed(None) - if self.proc: - d.addCallback(lambda res: self.kill_client()) - d.addCallback(lambda res: self.sparent.stopService()) - d.addCallback(lambda res: flushEventualQueue()) - def _close_statsfile(res): - self.statsfile.close() - d.addCallback(_close_statsfile) - d.addCallback(lambda res: passthrough) - return d - - def make_introducer(self): - iv_basedir = os.path.join(self.testdir, "introducer") - os.mkdir(iv_basedir) - self.introducer = introducer.IntroducerNode(basedir=iv_basedir) - self.introducer.setServiceParent(self) - self.introducer_furl = self.introducer.introducer_url - - def make_nodes(self): - root = FilePath(self.testdir) - self.nodes = [] - for i in range(self.numnodes): - nodedir = root.child("node%d" % (i,)) - private = nodedir.child("private") - private.makedirs() - write_introducer(nodedir, "default", self.introducer_url) - config = ( - "[client]\n" - "shares.happy = 1\n" - "[storage]\n" - ) - # the only tests for which we want the internal nodes to actually - # retain shares are the ones where somebody's going to download - # them. - if self.mode in ("download", "download-GET", "download-GET-slow"): - # retain shares - pass - else: - # for these tests, we tell the storage servers to pretend to - # accept shares, but really just throw them out, since we're - # only testing upload and not download. - config += "debug_discard = true\n" - if self.mode in ("receive",): - # for this mode, the client-under-test gets all the shares, - # so our internal nodes can refuse requests - config += "readonly = true\n" - nodedir.child("tahoe.cfg").setContent(config) - c = client.Client(basedir=nodedir.path) - c.setServiceParent(self) - self.nodes.append(c) - # the peers will start running, eventually they will connect to each - # other and the introducer - - def touch_keepalive(self): - if os.path.exists(self.keepalive_file): - age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME] - log.msg("touching keepalive file, was %ds old" % age) - f = open(self.keepalive_file, "w") - f.write("""\ -If the node notices this file at startup, it will poll every 5 seconds and -terminate if the file is more than 10 seconds old, or if it has been deleted. -If the test harness has an internal failure and neglects to kill off the node -itself, this helps to avoid leaving processes lying around. The contents of -this file are ignored. - """) - f.close() - - def start_client(self): - # this returns a Deferred that fires with the client's control.furl - log.msg("MAKING CLIENT") - # self.testdir is an absolute Unicode path - clientdir = self.clientdir = os.path.join(self.testdir, u"client") - clientdir_str = clientdir.encode(get_filesystem_encoding()) - quiet = StringIO() - create_node.create_node({'basedir': clientdir}, out=quiet) - log.msg("DONE MAKING CLIENT") - write_introducer(clientdir, "default", self.introducer_furl) - # now replace tahoe.cfg - # set webport=0 and then ask the node what port it picked. - f = open(os.path.join(clientdir, "tahoe.cfg"), "w") - f.write("[node]\n" - "web.port = tcp:0:interface=127.0.0.1\n" - "[client]\n" - "shares.happy = 1\n" - "[storage]\n" - ) - - if self.mode in ("upload-self", "receive"): - # accept and store shares, to trigger the memory consumption bugs - pass - else: - # don't accept any shares - f.write("readonly = true\n") - ## also, if we do receive any shares, throw them away - #f.write("debug_discard = true") - if self.mode == "upload-self": - pass - f.close() - self.keepalive_file = os.path.join(clientdir, - client.Client.EXIT_TRIGGER_FILE) - # now start updating the mtime. - self.touch_keepalive() - ts = internet.TimerService(1.0, self.touch_keepalive) - ts.setServiceParent(self.sparent) - - pp = ClientWatcher() - self.proc_done = pp.d = defer.Deferred() - logfile = os.path.join(self.basedir, "client.log") - tahoes = procutils.which("tahoe") - if not tahoes: - raise RuntimeError("unable to find a 'tahoe' executable") - cmd = [tahoes[0], "run", ".", "-l", logfile] - env = os.environ.copy() - self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir_str) - log.msg("CLIENT STARTED") - - # now we wait for the client to get started. we're looking for the - # control.furl file to appear. - furl_file = os.path.join(clientdir, "private", "control.furl") - url_file = os.path.join(clientdir, "node.url") - def _check(): - if pp.ended and pp.ended.value.status != 0: - # the twistd process ends normally (with rc=0) if the child - # is successfully launched. It ends abnormally (with rc!=0) - # if the child cannot be launched. - raise ChildDidNotStartError("process ended while waiting for startup") - return os.path.exists(furl_file) - d = self.poll(_check, 0.1) - # once it exists, wait a moment before we read from it, just in case - # it hasn't finished writing the whole thing. Ideally control.furl - # would be created in some atomic fashion, or made non-readable until - # it's ready, but I can't think of an easy way to do that, and I - # think the chances that we'll observe a half-write are pretty low. - def _stall(res): - d2 = defer.Deferred() - reactor.callLater(0.1, d2.callback, None) - return d2 - d.addCallback(_stall) - def _read(res): - # read the node's URL - self.webish_url = open(url_file, "r").read().strip() - if self.webish_url[-1] == "/": - # trim trailing slash, since the rest of the code wants it gone - self.webish_url = self.webish_url[:-1] - f = open(furl_file, "r") - furl = f.read() - return furl.strip() - d.addCallback(_read) - return d - - - def kill_client(self): - # returns a Deferred that fires when the process exits. This may only - # be called once. - try: - self.proc.signalProcess("INT") - except error.ProcessExitedAlready: - pass - return self.proc_done - - - def create_data(self, name, size): - filename = os.path.join(self.testdir, name + ".data") - f = open(filename, "wb") - block = "a" * 8192 - while size > 0: - l = min(size, 8192) - f.write(block[:l]) - size -= l - return filename - - def stash_stats(self, stats, name): - self.statsfile.write("%s %s: %d\n" % (self.mode, name, stats['VmPeak'])) - self.statsfile.flush() - self.stats[name] = stats['VmPeak'] - - def POST(self, urlpath, **fields): - url = self.webish_url + urlpath - sepbase = "boogabooga" - sep = "--" + sepbase - form = [] - form.append(sep) - form.append('Content-Disposition: form-data; name="_charset"') - form.append('') - form.append('UTF-8') - form.append(sep) - for name, value in fields.iteritems(): - if isinstance(value, tuple): - filename, value = value - form.append('Content-Disposition: form-data; name="%s"; ' - 'filename="%s"' % (name, filename)) - else: - form.append('Content-Disposition: form-data; name="%s"' % name) - form.append('') - form.append(value) - form.append(sep) - form[-1] += "--" - body = "\r\n".join(form) + "\r\n" - headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase, - } - return tw_client.getPage(url, method="POST", postdata=body, - headers=headers, followRedirect=False) - - def GET_discard(self, urlpath, stall): - url = self.webish_url + urlpath + "?filename=dummy-get.out" - return discardPage(url, stall) - - def _print_usage(self, res=None): - d = self.control_rref.callRemote("get_memory_usage") - def _print(stats): - print("VmSize: %9d VmPeak: %9d" % (stats["VmSize"], - stats["VmPeak"])) - return stats - d.addCallback(_print) - return d - - def _do_upload(self, res, size, files, uris): - name = '%d' % size - print() - print("uploading %s" % name) - if self.mode in ("upload", "upload-self"): - d = self.control_rref.callRemote("upload_random_data_from_file", - size, - convergence="check-memory") - elif self.mode == "upload-POST": - data = "a" * size - url = "/uri" - d = self.POST(url, t="upload", file=("%d.data" % size, data)) - elif self.mode in ("receive", - "download", "download-GET", "download-GET-slow"): - # mode=receive: upload the data from a local peer, so that the - # client-under-test receives and stores the shares - # - # mode=download*: upload the data from a local peer, then have - # the client-under-test download it. - # - # we need to wait until the uploading node has connected to all - # peers, since the wait_for_client_connections() above doesn't - # pay attention to our self.nodes[] and their connections. - files[name] = self.create_data(name, size) - u = self.nodes[0].getServiceNamed("uploader") - d = self.nodes[0].debug_wait_for_client_connections(self.numnodes+1) - d.addCallback(lambda res: - u.upload(upload.FileName(files[name], - convergence="check-memory"))) - d.addCallback(lambda results: results.get_uri()) - else: - raise ValueError("unknown mode=%s" % self.mode) - def _complete(uri): - uris[name] = uri - print("uploaded %s" % name) - d.addCallback(_complete) - return d - - def _do_download(self, res, size, uris): - if self.mode not in ("download", "download-GET", "download-GET-slow"): - return - name = '%d' % size - print("downloading %s" % name) - uri = uris[name] - - if self.mode == "download": - d = self.control_rref.callRemote("download_to_tempfile_and_delete", - uri) - elif self.mode == "download-GET": - url = "/uri/%s" % uri - d = self.GET_discard(urllib.quote(url), stall=False) - elif self.mode == "download-GET-slow": - url = "/uri/%s" % uri - d = self.GET_discard(urllib.quote(url), stall=True) - - def _complete(res): - print("downloaded %s" % name) - return res - d.addCallback(_complete) - return d - - def do_test(self): - #print("CLIENT STARTED") - #print("FURL", self.control_furl) - #print("RREF", self.control_rref) - #print() - kB = 1000; MB = 1000*1000 - files = {} - uris = {} - - d = self._print_usage() - d.addCallback(self.stash_stats, "0B") - - for i in range(10): - d.addCallback(self._do_upload, 10*kB+i, files, uris) - d.addCallback(self._do_download, 10*kB+i, uris) - d.addCallback(self._print_usage) - d.addCallback(self.stash_stats, "10kB") - - for i in range(3): - d.addCallback(self._do_upload, 10*MB+i, files, uris) - d.addCallback(self._do_download, 10*MB+i, uris) - d.addCallback(self._print_usage) - d.addCallback(self.stash_stats, "10MB") - - for i in range(1): - d.addCallback(self._do_upload, 50*MB+i, files, uris) - d.addCallback(self._do_download, 50*MB+i, uris) - d.addCallback(self._print_usage) - d.addCallback(self.stash_stats, "50MB") - - #for i in range(1): - # d.addCallback(self._do_upload, 100*MB+i, files, uris) - # d.addCallback(self._do_download, 100*MB+i, uris) - # d.addCallback(self._print_usage) - #d.addCallback(self.stash_stats, "100MB") - - #d.addCallback(self.stall) - def _done(res): - print("FINISHING") - d.addCallback(_done) - return d - - def stall(self, res): - d = defer.Deferred() - reactor.callLater(5, d.callback, None) - return d - - -class ClientWatcher(protocol.ProcessProtocol, object): - ended = False - def outReceived(self, data): - print("OUT:", data) - def errReceived(self, data): - print("ERR:", data) - def processEnded(self, reason): - self.ended = reason - self.d.callback(None) - - -if __name__ == '__main__': - mode = "upload" - if len(sys.argv) > 1: - mode = sys.argv[1] - if sys.maxsize == 2147483647: - bits = "32" - elif sys.maxsize == 9223372036854775807: - bits = "64" - else: - bits = "?" - print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) - # put the logfile and stats.out in _test_memory/ . These stick around. - # put the nodes and other files in _test_memory/test/ . These are - # removed each time we run. - sf = SystemFramework("_test_memory", mode) - sf.run() diff --git a/tox.ini b/tox.ini index 610570be5..1b1e8e5e3 100644 --- a/tox.ini +++ b/tox.ini @@ -206,17 +206,6 @@ commands = flogtool --version python misc/build_helpers/run-deprecations.py --package allmydata --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} -[testenv:checkmemory] -commands = - rm -rf _test_memory - python src/allmydata/test/check_memory.py upload - python src/allmydata/test/check_memory.py upload-self - python src/allmydata/test/check_memory.py upload-POST - python src/allmydata/test/check_memory.py download - python src/allmydata/test/check_memory.py download-GET - python src/allmydata/test/check_memory.py download-GET-slow - python src/allmydata/test/check_memory.py receive - # Use 'tox -e docs' to check formatting and cross-references in docs .rst # files. The published docs are built by code run over at readthedocs.org, # which does not use this target (but does something similar). From 1b8e013991ffb71620fcd6589945624e8cac0e97 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:46:34 -0400 Subject: [PATCH 031/916] Get rid of `check_speed` The motivation and justification here are roughly the same as for `check_memory`. --- Makefile | 28 +---- misc/checkers/check_speed.py | 234 ----------------------------------- 2 files changed, 1 insertion(+), 261 deletions(-) delete mode 100644 misc/checkers/check_speed.py diff --git a/Makefile b/Makefile index f7a357588..5d8bf18ba 100644 --- a/Makefile +++ b/Makefile @@ -142,31 +142,6 @@ count-lines: # src/allmydata/test/bench_dirnode.py -# The check-speed and check-grid targets are disabled, since they depend upon -# the pre-located $(TAHOE) executable that was removed when we switched to -# tox. They will eventually be resurrected as dedicated tox environments. - -# The check-speed target uses a pre-established client node to run a canned -# set of performance tests against a test network that is also -# pre-established (probably on a remote machine). Provide it with the path to -# a local directory where this client node has been created (and populated -# with the necessary FURLs of the test network). This target will start that -# client with the current code and then run the tests. Afterwards it will -# stop the client. -# -# The 'sleep 5' is in there to give the new client a chance to connect to its -# storageservers, since check_speed.py has no good way of doing that itself. - -##.PHONY: check-speed -##check-speed: .built -## if [ -z '$(TESTCLIENTDIR)' ]; then exit 1; fi -## @echo "stopping any leftover client code" -## -$(TAHOE) stop $(TESTCLIENTDIR) -## $(TAHOE) start $(TESTCLIENTDIR) -## sleep 5 -## $(TAHOE) @src/allmydata/test/check_speed.py $(TESTCLIENTDIR) -## $(TAHOE) stop $(TESTCLIENTDIR) - # The check-grid target also uses a pre-established client node, along with a # long-term directory that contains some well-known files. See the docstring # in src/allmydata/test/check_grid.py to see how to set this up. @@ -195,12 +170,11 @@ test-clean: # Use 'make distclean' instead to delete all generated files. .PHONY: clean clean: - rm -rf build _trial_temp _test_memory .built + rm -rf build _trial_temp .built rm -f `find src *.egg -name '*.so' -or -name '*.pyc'` rm -rf support dist rm -rf `ls -d *.egg | grep -vEe"setuptools-|setuptools_darcs-|darcsver-"` rm -rf *.pyc - rm -f bin/tahoe bin/tahoe.pyscript rm -f *.pkg .PHONY: distclean diff --git a/misc/checkers/check_speed.py b/misc/checkers/check_speed.py deleted file mode 100644 index 2fce53387..000000000 --- a/misc/checkers/check_speed.py +++ /dev/null @@ -1,234 +0,0 @@ -from __future__ import print_function - -import os, sys -from twisted.internet import reactor, defer -from twisted.python import log -from twisted.application import service -from foolscap.api import Tub, fireEventually - -MB = 1000000 - -class SpeedTest(object): - DO_IMMUTABLE = True - DO_MUTABLE_CREATE = True - DO_MUTABLE = True - - def __init__(self, test_client_dir): - #self.real_stderr = sys.stderr - log.startLogging(open("st.log", "a"), setStdout=False) - f = open(os.path.join(test_client_dir, "private", "control.furl"), "r") - self.control_furl = f.read().strip() - f.close() - self.base_service = service.MultiService() - self.failed = None - self.upload_times = {} - self.download_times = {} - - def run(self): - print("STARTING") - d = fireEventually() - d.addCallback(lambda res: self.setUp()) - d.addCallback(lambda res: self.do_test()) - d.addBoth(self.tearDown) - def _err(err): - self.failed = err - log.err(err) - print(err) - d.addErrback(_err) - def _done(res): - reactor.stop() - return res - d.addBoth(_done) - reactor.run() - if self.failed: - print("EXCEPTION") - print(self.failed) - sys.exit(1) - - def setUp(self): - self.base_service.startService() - self.tub = Tub() - self.tub.setOption("expose-remote-exception-types", False) - self.tub.setServiceParent(self.base_service) - d = self.tub.getReference(self.control_furl) - def _gotref(rref): - self.client_rref = rref - print("Got Client Control reference") - return self.stall(5) - d.addCallback(_gotref) - return d - - def stall(self, delay, result=None): - d = defer.Deferred() - reactor.callLater(delay, d.callback, result) - return d - - def record_times(self, times, key): - print("TIME (%s): %s up, %s down" % (key, times[0], times[1])) - self.upload_times[key], self.download_times[key] = times - - def one_test(self, res, name, count, size, mutable): - # values for 'mutable': - # False (upload a different CHK file for each 'count') - # "create" (upload different contents into a new SSK file) - # "upload" (upload different contents into the same SSK file. The - # time consumed does not include the creation of the file) - d = self.client_rref.callRemote("speed_test", count, size, mutable) - d.addCallback(self.record_times, name) - return d - - def measure_rtt(self, res): - # use RIClient.get_nodeid() to measure the foolscap-level RTT - d = self.client_rref.callRemote("measure_peer_response_time") - def _got(res): - assert len(res) # need at least one peer - times = res.values() - self.total_rtt = sum(times) - self.average_rtt = sum(times) / len(times) - self.max_rtt = max(times) - print("num-peers: %d" % len(times)) - print("total-RTT: %f" % self.total_rtt) - print("average-RTT: %f" % self.average_rtt) - print("max-RTT: %f" % self.max_rtt) - d.addCallback(_got) - return d - - def do_test(self): - print("doing test") - d = defer.succeed(None) - d.addCallback(self.one_test, "startup", 1, 1000, False) #ignore this one - d.addCallback(self.measure_rtt) - - if self.DO_IMMUTABLE: - # immutable files - d.addCallback(self.one_test, "1x 200B", 1, 200, False) - d.addCallback(self.one_test, "10x 200B", 10, 200, False) - def _maybe_do_100x_200B(res): - if self.upload_times["10x 200B"] < 5: - print("10x 200B test went too fast, doing 100x 200B test") - return self.one_test(None, "100x 200B", 100, 200, False) - return - d.addCallback(_maybe_do_100x_200B) - d.addCallback(self.one_test, "1MB", 1, 1*MB, False) - d.addCallback(self.one_test, "10MB", 1, 10*MB, False) - def _maybe_do_100MB(res): - if self.upload_times["10MB"] > 30: - print("10MB test took too long, skipping 100MB test") - return - return self.one_test(None, "100MB", 1, 100*MB, False) - d.addCallback(_maybe_do_100MB) - - if self.DO_MUTABLE_CREATE: - # mutable file creation - d.addCallback(self.one_test, "10x 200B SSK creation", 10, 200, - "create") - - if self.DO_MUTABLE: - # mutable file upload/download - d.addCallback(self.one_test, "10x 200B SSK", 10, 200, "upload") - def _maybe_do_100x_200B_SSK(res): - if self.upload_times["10x 200B SSK"] < 5: - print("10x 200B SSK test went too fast, doing 100x 200B SSK") - return self.one_test(None, "100x 200B SSK", 100, 200, - "upload") - return - d.addCallback(_maybe_do_100x_200B_SSK) - d.addCallback(self.one_test, "1MB SSK", 1, 1*MB, "upload") - - d.addCallback(self.calculate_speeds) - return d - - def calculate_speeds(self, res): - # time = A*size+B - # we assume that A*200bytes is negligible - - if self.DO_IMMUTABLE: - # upload - if "100x 200B" in self.upload_times: - B = self.upload_times["100x 200B"] / 100 - else: - B = self.upload_times["10x 200B"] / 10 - print("upload per-file time: %.3fs" % B) - print("upload per-file times-avg-RTT: %f" % (B / self.average_rtt)) - print("upload per-file times-total-RTT: %f" % (B / self.total_rtt)) - A1 = 1*MB / (self.upload_times["1MB"] - B) # in bytes per second - print("upload speed (1MB):", self.number(A1, "Bps")) - A2 = 10*MB / (self.upload_times["10MB"] - B) - print("upload speed (10MB):", self.number(A2, "Bps")) - if "100MB" in self.upload_times: - A3 = 100*MB / (self.upload_times["100MB"] - B) - print("upload speed (100MB):", self.number(A3, "Bps")) - - # download - if "100x 200B" in self.download_times: - B = self.download_times["100x 200B"] / 100 - else: - B = self.download_times["10x 200B"] / 10 - print("download per-file time: %.3fs" % B) - print("download per-file times-avg-RTT: %f" % (B / self.average_rtt)) - print("download per-file times-total-RTT: %f" % (B / self.total_rtt)) - A1 = 1*MB / (self.download_times["1MB"] - B) # in bytes per second - print("download speed (1MB):", self.number(A1, "Bps")) - A2 = 10*MB / (self.download_times["10MB"] - B) - print("download speed (10MB):", self.number(A2, "Bps")) - if "100MB" in self.download_times: - A3 = 100*MB / (self.download_times["100MB"] - B) - print("download speed (100MB):", self.number(A3, "Bps")) - - if self.DO_MUTABLE_CREATE: - # SSK creation - B = self.upload_times["10x 200B SSK creation"] / 10 - print("create per-file time SSK: %.3fs" % B) - - if self.DO_MUTABLE: - # upload SSK - if "100x 200B SSK" in self.upload_times: - B = self.upload_times["100x 200B SSK"] / 100 - else: - B = self.upload_times["10x 200B SSK"] / 10 - print("upload per-file time SSK: %.3fs" % B) - A1 = 1*MB / (self.upload_times["1MB SSK"] - B) # in bytes per second - print("upload speed SSK (1MB):", self.number(A1, "Bps")) - - # download SSK - if "100x 200B SSK" in self.download_times: - B = self.download_times["100x 200B SSK"] / 100 - else: - B = self.download_times["10x 200B SSK"] / 10 - print("download per-file time SSK: %.3fs" % B) - A1 = 1*MB / (self.download_times["1MB SSK"] - B) # in bytes per - # second - print("download speed SSK (1MB):", self.number(A1, "Bps")) - - def number(self, value, suffix=""): - scaling = 1 - if value < 1: - fmt = "%1.2g%s" - elif value < 100: - fmt = "%.1f%s" - elif value < 1000: - fmt = "%d%s" - elif value < 1e6: - fmt = "%.2fk%s"; scaling = 1e3 - elif value < 1e9: - fmt = "%.2fM%s"; scaling = 1e6 - elif value < 1e12: - fmt = "%.2fG%s"; scaling = 1e9 - elif value < 1e15: - fmt = "%.2fT%s"; scaling = 1e12 - elif value < 1e18: - fmt = "%.2fP%s"; scaling = 1e15 - else: - fmt = "huge! %g%s" - return fmt % (value / scaling, suffix) - - def tearDown(self, res): - d = self.base_service.stopService() - d.addCallback(lambda ignored: res) - return d - - -if __name__ == '__main__': - test_client_dir = sys.argv[1] - st = SpeedTest(test_client_dir) - st.run() From 1aae92b18e0180b31f1bb91eca2cfda2d8fb5058 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:47:05 -0400 Subject: [PATCH 032/916] Get rid of `getmem.py` helper Platforms provide an interface for retrieving this information. Just use those interfaces instead. --- misc/operations_helpers/getmem.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 misc/operations_helpers/getmem.py diff --git a/misc/operations_helpers/getmem.py b/misc/operations_helpers/getmem.py deleted file mode 100644 index b3c6285fe..000000000 --- a/misc/operations_helpers/getmem.py +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env python - -from __future__ import print_function - -from foolscap import Tub -from foolscap.eventual import eventually -import sys -from twisted.internet import reactor - -def go(): - t = Tub() - d = t.getReference(sys.argv[1]) - d.addCallback(lambda rref: rref.callRemote("get_memory_usage")) - def _got(res): - print(res) - reactor.stop() - d.addCallback(_got) - -eventually(go) -reactor.run() From 95b765e3092b8f076fe8e72053e1986a4e642086 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:54:18 -0400 Subject: [PATCH 033/916] stop creating a control tub for the introducer --- src/allmydata/introducer/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 1e28f511b..950602f98 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -39,7 +39,6 @@ from allmydata.introducer.common import unsign_from_foolscap, \ from allmydata.node import read_config from allmydata.node import create_node_dir from allmydata.node import create_connection_handlers -from allmydata.node import create_control_tub from allmydata.node import create_tub_options from allmydata.node import create_main_tub @@ -88,7 +87,7 @@ def create_introducer(basedir=u"."): config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = create_control_tub() + control_tub = None node = _IntroducerNode( config, From e0312eae57fd35e7f68a62dd4ceb4957082bf6fa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:02:24 -0400 Subject: [PATCH 034/916] stop creating a control tub for client nodes --- src/allmydata/client.py | 10 +--------- src/allmydata/test/test_system.py | 20 -------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index aabae9065..8a953937a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -40,7 +40,6 @@ from allmydata.storage.server import StorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper -from allmydata.control import ControlServer from allmydata.introducer.client import IntroducerClient from allmydata.util import ( hashutil, base32, pollmixin, log, idlib, @@ -283,7 +282,7 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = node.create_control_tub() + control_tub = None introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) storage_broker = create_storage_farm_broker( @@ -648,7 +647,6 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_stats_provider() self.init_secrets() self.init_node_key() - self.init_control() self._key_generator = KeyGenerator() key_gen_furl = config.get_config("client", "key_generator.furl", None) if key_gen_furl: @@ -985,12 +983,6 @@ class _Client(node.Node, pollmixin.PollMixin): def get_history(self): return self.history - def init_control(self): - c = ControlServer() - c.setServiceParent(self) - control_url = self.control_tub.registerReference(c) - self.config.write_private_config("control.furl", control_url + "\n") - def init_helper(self): self.helper = Helper(self.config.get_config_path("helper"), self.storage_broker, self._secret_holder, diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 3e1bdcdd4..c01dd0afc 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -780,7 +780,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self._check_publish_private) d.addCallback(self.log, "did _check_publish_private") d.addCallback(self._test_web) - d.addCallback(self._test_control) d.addCallback(self._test_cli) # P now has four top-level children: # P/personal/sekrit data @@ -1343,25 +1342,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): if line.startswith("CHK %s " % storage_index_s)] self.failUnlessEqual(len(matching), 10) - def _test_control(self, res): - # exercise the remote-control-the-client foolscap interfaces in - # allmydata.control (mostly used for performance tests) - c0 = self.clients[0] - control_furl_file = c0.config.get_private_path("control.furl") - control_furl = ensure_str(open(control_furl_file, "r").read().strip()) - # it doesn't really matter which Tub we use to connect to the client, - # so let's just use our IntroducerNode's - d = self.introducer.tub.getReference(control_furl) - d.addCallback(self._test_control2, control_furl_file) - return d - def _test_control2(self, rref, filename): - d = defer.succeed(None) - d.addCallback(lambda res: rref.callRemote("speed_test", 1, 200, False)) - if sys.platform in ("linux2", "linux3"): - d.addCallback(lambda res: rref.callRemote("get_memory_usage")) - d.addCallback(lambda res: rref.callRemote("measure_peer_response_time")) - return d - def _test_cli(self, res): # run various CLI commands (in a thread, since they use blocking # network calls) From ddf5f461bf69224c47b0b8d41c84d2abdc63c5c4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:11:53 -0400 Subject: [PATCH 035/916] Stop half-pretending to have a control port --- src/allmydata/test/no_network.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 2f75f9274..3b88a1cc6 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -17,8 +17,7 @@ from __future__ import unicode_literals # This should be useful for tests which want to examine and/or manipulate the # uploaded shares, checker/verifier/repairer tests, etc. The clients have no -# Tubs, so it is not useful for tests that involve a Helper or the -# control.furl . +# Tubs, so it is not useful for tests that involve a Helper. from future.utils import PY2 if PY2: @@ -274,8 +273,6 @@ class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 pass def init_introducer_client(self): pass - def create_control_tub(self): - pass def create_log_tub(self): pass def setup_logging(self): @@ -284,8 +281,6 @@ class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 service.MultiService.startService(self) def stopService(self): return service.MultiService.stopService(self) - def init_control(self): - pass def init_helper(self): pass def init_key_gen(self): From 1de480dc37b24c75441724a99bc0b265347afb16 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:12:03 -0400 Subject: [PATCH 036/916] Stop offering an API to create a control tub or handling the control tub --- src/allmydata/node.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 5a6f8c66f..08271fc5f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -919,18 +919,6 @@ def create_main_tub(config, tub_options, return tub -def create_control_tub(): - """ - Creates a Foolscap Tub for use by the control port. This is a - localhost-only ephemeral Tub, with no control over the listening - port or location - """ - control_tub = Tub() - portnum = iputil.listenOnUnused(control_tub) - log.msg("Control Tub location set to 127.0.0.1:%s" % (portnum,)) - return control_tub - - class Node(service.MultiService): """ This class implements common functionality of both Client nodes and Introducer nodes. @@ -967,10 +955,6 @@ class Node(service.MultiService): else: self.nodeid = self.short_nodeid = None - self.control_tub = control_tub - if self.control_tub is not None: - self.control_tub.setServiceParent(self) - self.log("Node constructed. " + __full_version__) iputil.increase_rlimits() From fe2e2cc1d697f562284cfffcfe3b250dea4ed36c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:12:19 -0400 Subject: [PATCH 037/916] Get rid of the control service --- src/allmydata/control.py | 273 --------------------------------------- 1 file changed, 273 deletions(-) delete mode 100644 src/allmydata/control.py diff --git a/src/allmydata/control.py b/src/allmydata/control.py deleted file mode 100644 index 7efa174ab..000000000 --- a/src/allmydata/control.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import os, time, tempfile -from zope.interface import implementer -from twisted.application import service -from twisted.internet import defer -from twisted.internet.interfaces import IConsumer -from foolscap.api import Referenceable -from allmydata.interfaces import RIControlClient, IFileNode -from allmydata.util import fileutil, mathutil -from allmydata.immutable import upload -from allmydata.mutable.publish import MutableData -from twisted.python import log - -def get_memory_usage(): - # this is obviously linux-specific - stat_names = (b"VmPeak", - b"VmSize", - #b"VmHWM", - b"VmData") - stats = {} - try: - with open("/proc/self/status", "rb") as f: - for line in f: - name, right = line.split(b":",2) - if name in stat_names: - assert right.endswith(b" kB\n") - right = right[:-4] - stats[name] = int(right) * 1024 - except: - # Probably not on (a compatible version of) Linux - stats['VmSize'] = 0 - stats['VmPeak'] = 0 - return stats - -def log_memory_usage(where=""): - stats = get_memory_usage() - log.msg("VmSize: %9d VmPeak: %9d %s" % (stats[b"VmSize"], - stats[b"VmPeak"], - where)) - -@implementer(IConsumer) -class FileWritingConsumer(object): - def __init__(self, filename): - self.done = False - self.f = open(filename, "wb") - def registerProducer(self, p, streaming): - if streaming: - p.resumeProducing() - else: - while not self.done: - p.resumeProducing() - def write(self, data): - self.f.write(data) - def unregisterProducer(self): - self.done = True - self.f.close() - -@implementer(RIControlClient) -class ControlServer(Referenceable, service.Service): - - def remote_wait_for_client_connections(self, num_clients): - return self.parent.debug_wait_for_client_connections(num_clients) - - def remote_upload_random_data_from_file(self, size, convergence): - tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "data") - f = open(filename, "wb") - block = b"a" * 8192 - while size > 0: - l = min(size, 8192) - f.write(block[:l]) - size -= l - f.close() - uploader = self.parent.getServiceNamed("uploader") - u = upload.FileName(filename, convergence=convergence) - # XXX should pass reactor arg - d = uploader.upload(u) - d.addCallback(lambda results: results.get_uri()) - def _done(uri): - os.remove(filename) - os.rmdir(tempdir) - return uri - d.addCallback(_done) - return d - - def remote_download_to_tempfile_and_delete(self, uri): - tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "data") - filenode = self.parent.create_node_from_uri(uri, name=filename) - if not IFileNode.providedBy(filenode): - raise AssertionError("The URI does not reference a file.") - c = FileWritingConsumer(filename) - d = filenode.read(c) - def _done(res): - os.remove(filename) - os.rmdir(tempdir) - return None - d.addCallback(_done) - return d - - def remote_speed_test(self, count, size, mutable): - assert size > 8 - log.msg("speed_test: count=%d, size=%d, mutable=%s" % (count, size, - mutable)) - st = SpeedTest(self.parent, count, size, mutable) - return st.run() - - def remote_get_memory_usage(self): - return get_memory_usage() - - def remote_measure_peer_response_time(self): - # I'd like to average together several pings, but I don't want this - # phase to take more than 10 seconds. Expect worst-case latency to be - # 300ms. - results = {} - sb = self.parent.get_storage_broker() - everyone = sb.get_connected_servers() - num_pings = int(mathutil.div_ceil(10, (len(everyone) * 0.3))) - everyone = list(everyone) * num_pings - d = self._do_one_ping(None, everyone, results) - return d - def _do_one_ping(self, res, everyone_left, results): - if not everyone_left: - return results - server = everyone_left.pop(0) - server_name = server.get_longname() - storage_server = server.get_storage_server() - start = time.time() - d = storage_server.get_buckets(b"\x00" * 16) - def _done(ignored): - stop = time.time() - elapsed = stop - start - if server_name in results: - results[server_name].append(elapsed) - else: - results[server_name] = [elapsed] - d.addCallback(_done) - d.addCallback(self._do_one_ping, everyone_left, results) - def _average(res): - averaged = {} - for server_name,times in results.items(): - averaged[server_name] = sum(times) / len(times) - return averaged - d.addCallback(_average) - return d - -class SpeedTest(object): - def __init__(self, parent, count, size, mutable): - self.parent = parent - self.count = count - self.size = size - self.mutable_mode = mutable - self.uris = {} - self.basedir = self.parent.config.get_config_path("_speed_test_data") - - def run(self): - self.create_data() - d = self.do_upload() - d.addCallback(lambda res: self.do_download()) - d.addBoth(self.do_cleanup) - d.addCallback(lambda res: (self.upload_time, self.download_time)) - return d - - def create_data(self): - fileutil.make_dirs(self.basedir) - for i in range(self.count): - s = self.size - fn = os.path.join(self.basedir, str(i)) - if os.path.exists(fn): - os.unlink(fn) - f = open(fn, "wb") - f.write(os.urandom(8)) - s -= 8 - while s > 0: - chunk = min(s, 4096) - f.write(b"\x00" * chunk) - s -= chunk - f.close() - - def do_upload(self): - d = defer.succeed(None) - def _create_slot(res): - d1 = self.parent.create_mutable_file(b"") - def _created(n): - self._n = n - d1.addCallback(_created) - return d1 - if self.mutable_mode == "upload": - d.addCallback(_create_slot) - def _start(res): - self._start = time.time() - d.addCallback(_start) - - def _record_uri(uri, i): - self.uris[i] = uri - def _upload_one_file(ignored, i): - if i >= self.count: - return - fn = os.path.join(self.basedir, str(i)) - if self.mutable_mode == "create": - data = open(fn,"rb").read() - d1 = self.parent.create_mutable_file(data) - d1.addCallback(lambda n: n.get_uri()) - elif self.mutable_mode == "upload": - data = open(fn,"rb").read() - d1 = self._n.overwrite(MutableData(data)) - d1.addCallback(lambda res: self._n.get_uri()) - else: - up = upload.FileName(fn, convergence=None) - d1 = self.parent.upload(up) - d1.addCallback(lambda results: results.get_uri()) - d1.addCallback(_record_uri, i) - d1.addCallback(_upload_one_file, i+1) - return d1 - d.addCallback(_upload_one_file, 0) - def _upload_done(ignored): - stop = time.time() - self.upload_time = stop - self._start - d.addCallback(_upload_done) - return d - - def do_download(self): - start = time.time() - d = defer.succeed(None) - def _download_one_file(ignored, i): - if i >= self.count: - return - n = self.parent.create_node_from_uri(self.uris[i]) - if not IFileNode.providedBy(n): - raise AssertionError("The URI does not reference a file.") - if n.is_mutable(): - d1 = n.download_best_version() - else: - d1 = n.read(DiscardingConsumer()) - d1.addCallback(_download_one_file, i+1) - return d1 - d.addCallback(_download_one_file, 0) - def _download_done(ignored): - stop = time.time() - self.download_time = stop - start - d.addCallback(_download_done) - return d - - def do_cleanup(self, res): - for i in range(self.count): - fn = os.path.join(self.basedir, str(i)) - os.unlink(fn) - return res - -@implementer(IConsumer) -class DiscardingConsumer(object): - def __init__(self): - self.done = False - def registerProducer(self, p, streaming): - if streaming: - p.resumeProducing() - else: - while not self.done: - p.resumeProducing() - def write(self, data): - pass - def unregisterProducer(self): - self.done = True From 0611af6b0b7de30ef9720ab08b796f615107f5bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 13:10:18 -0400 Subject: [PATCH 038/916] Stop passing even a dummy value for control tub into Nodes --- src/allmydata/client.py | 6 ++---- src/allmydata/introducer/server.py | 6 ++---- src/allmydata/node.py | 2 +- src/allmydata/test/no_network.py | 1 - src/allmydata/test/web/test_introducer.py | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 8a953937a..a2f88ebd6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -282,7 +282,6 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = None introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) storage_broker = create_storage_farm_broker( @@ -293,7 +292,6 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= client = _client_factory( config, main_tub, - control_tub, i2p_provider, tor_provider, introducer_clients, @@ -630,12 +628,12 @@ class _Client(node.Node, pollmixin.PollMixin): "max_segment_size": DEFAULT_MAX_SEGMENT_SIZE, } - def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider, introducer_clients, + def __init__(self, config, main_tub, i2p_provider, tor_provider, introducer_clients, storage_farm_broker): """ Use :func:`allmydata.client.create_client` to instantiate one of these. """ - node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) + node.Node.__init__(self, config, main_tub, i2p_provider, tor_provider) self.started_timestamp = time.time() self.logSource = "Client" diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 950602f98..8678ad5bf 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -87,12 +87,10 @@ def create_introducer(basedir=u"."): config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = None node = _IntroducerNode( config, main_tub, - control_tub, i2p_provider, tor_provider, ) @@ -104,8 +102,8 @@ def create_introducer(basedir=u"."): class _IntroducerNode(node.Node): NODETYPE = "introducer" - def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): - node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) + def __init__(self, config, main_tub, i2p_provider, tor_provider): + node.Node.__init__(self, config, main_tub, i2p_provider, tor_provider) self.init_introducer() webport = self.get_config("node", "web.port", None) if webport: diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 08271fc5f..3ac4c507b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -926,7 +926,7 @@ class Node(service.MultiService): NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): + def __init__(self, config, main_tub, i2p_provider, tor_provider): """ Initialize the node with the given configuration. Its base directory is the current directory by default. diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 3b88a1cc6..7a84580bf 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -250,7 +250,6 @@ def create_no_network_client(basedir): client = _NoNetworkClient( config, main_tub=None, - control_tub=None, i2p_provider=None, tor_provider=None, introducer_clients=[], diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index ba0a5beb9..4b5850cbc 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -211,7 +211,7 @@ class IntroducerRootTests(SyncTestCase): main_tub = Tub() main_tub.listenOn(b"tcp:0") main_tub.setLocation(b"tcp:127.0.0.1:1") - introducer_node = _IntroducerNode(config, main_tub, None, None, None) + introducer_node = _IntroducerNode(config, main_tub, None, None) introducer_service = introducer_node.getServiceNamed("introducer") for n in range(2): From 9e59e6922383639014f48d8c496bd91da8806a0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 13:13:08 -0400 Subject: [PATCH 039/916] news fragment --- newsfragments/3814.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3814.removed diff --git a/newsfragments/3814.removed b/newsfragments/3814.removed new file mode 100644 index 000000000..939d20ffc --- /dev/null +++ b/newsfragments/3814.removed @@ -0,0 +1 @@ +The little-used "control port" has been removed from all node types. From ad216e0f237fc7e32f296eaf0a38fac69e4ba70f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 13:13:37 -0400 Subject: [PATCH 040/916] remove unused import --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c01dd0afc..087a1c634 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -12,7 +12,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 from past.builtins import chr as byteschr, long -from six import ensure_text, ensure_str +from six import ensure_text import os, re, sys, time, json From 1c347c593130f606cfdc7d0e2f52a0ec1db5b20a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 15:05:21 -0400 Subject: [PATCH 041/916] replace sensitive introducer fURL with path where it can be found --- src/allmydata/introducer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 1e28f511b..aa0ae8336 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -136,7 +136,7 @@ class _IntroducerNode(node.Node): os.rename(old_public_fn, private_fn) furl = self.tub.registerReference(introducerservice, furlFile=private_fn) - self.log(" introducer is at %s" % furl, umid="qF2L9A") + self.log(" introducer can be found in {!r}".format(private_fn), umid="qF2L9A") self.introducer_url = furl # for tests def init_web(self, webport): From 67fb8aeb257ae55ede175eba8f704395b30b0273 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:08:01 -0400 Subject: [PATCH 042/916] add the security type --- towncrier.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/towncrier.toml b/towncrier.toml index b8b561a98..ae9c9d5a5 100644 --- a/towncrier.toml +++ b/towncrier.toml @@ -12,6 +12,11 @@ "~", ] + [[tool.towncrier.type]] + directory = "security" + name = "Security-related Changes" + showcontent = true + [[tool.towncrier.type]] directory = "incompat" name = "Backwards Incompatible Changes" From a7073fe531f56c1cb033fa815cd34ea1aa4962ca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:08:58 -0400 Subject: [PATCH 043/916] news fragment --- newsfragments/3815.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3815.documentation diff --git a/newsfragments/3815.documentation b/newsfragments/3815.documentation new file mode 100644 index 000000000..7abc70bd1 --- /dev/null +++ b/newsfragments/3815.documentation @@ -0,0 +1 @@ +The news file for future releases will include a section for changes with a security impact. \ No newline at end of file From 30ae30e3253d19926a75cd1430fae192b53307be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:11:49 -0400 Subject: [PATCH 044/916] fix the whitespace :/ --- towncrier.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/towncrier.toml b/towncrier.toml index ae9c9d5a5..e093d0cc4 100644 --- a/towncrier.toml +++ b/towncrier.toml @@ -14,8 +14,8 @@ [[tool.towncrier.type]] directory = "security" - name = "Security-related Changes" - showcontent = true + name = "Security-related Changes" + showcontent = true [[tool.towncrier.type]] directory = "incompat" From f2ef72e935126f55be103ac856836ebf4b2c140e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:14:42 -0400 Subject: [PATCH 045/916] newsfragment in temporary location --- newsfragments/LFS-01-001.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/LFS-01-001.security diff --git a/newsfragments/LFS-01-001.security b/newsfragments/LFS-01-001.security new file mode 100644 index 000000000..975fd0035 --- /dev/null +++ b/newsfragments/LFS-01-001.security @@ -0,0 +1 @@ +The introducer server no longer writes the sensitive introducer fURL value to its log at startup time. Instead it writes the well-known path of the file from which this value can be read. From 58112ba75b8a53e6f361600cc9a0b602e5aebc7c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Oct 2021 12:50:29 -0400 Subject: [PATCH 046/916] Plan of implementation for lease tests. --- newsfragments/3800.minor | 0 src/allmydata/test/test_istorageserver.py | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 newsfragments/3800.minor diff --git a/newsfragments/3800.minor b/newsfragments/3800.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index bd056ae13..1c5496ea4 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -444,6 +444,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): b"immutable", storage_index, 0, b"ono" ) + # TODO allocate_buckets creates lease + # TODO add_lease renews lease if existing storage index and secret + # TODO add_lease creates new lease if new secret + class IStorageServerMutableAPIsTestsMixin(object): """ @@ -820,6 +824,12 @@ class IStorageServerMutableAPIsTestsMixin(object): b"mutable", storage_index, 0, b"ono" ) + # TODO STARAW creates lease for new data + # TODO STARAW renews lease if same secret is used on existing data + # TODO STARAW creates new lease for existing data if new secret is given + # TODO add_lease renews lease if existing storage index and secret + # TODO add_lease creates new lease if new secret + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 2b40610a27c93ae7cf639063dafe518580193906 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Oct 2021 12:55:30 -0400 Subject: [PATCH 047/916] "Server" is extremely ambiguous, so let's just call this a client, which it is. --- src/allmydata/test/test_istorageserver.py | 66 +++++++++++------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 1c5496ea4..b34964463 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -56,7 +56,7 @@ class IStorageServerSharedAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s shared APIs. - ``self.storage_server`` is expected to provide ``IStorageServer``. + ``self.storage_client`` is expected to provide ``IStorageServer``. """ @inlineCallbacks @@ -65,7 +65,7 @@ class IStorageServerSharedAPIsTestsMixin(object): ``IStorageServer`` returns a dictionary where the key is an expected protocol version. """ - result = yield self.storage_server.get_version() + result = yield self.storage_client.get_version() self.assertIsInstance(result, dict) self.assertIn(b"http://allmydata.org/tahoe/protocols/storage/v1", result) @@ -74,10 +74,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s immutable APIs. - ``self.storage_server`` is expected to provide ``IStorageServer``. + ``self.storage_client`` is expected to provide ``IStorageServer``. ``self.disconnect()`` should disconnect and then reconnect, creating a new - ``self.storage_server``. Some implementations may wish to skip tests using + ``self.storage_client``. Some implementations may wish to skip tests using this; HTTP has no notion of disconnection. """ @@ -87,7 +87,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): allocate_buckets() with a new storage index returns the matching shares. """ - (already_got, allocated) = yield self.storage_server.allocate_buckets( + (already_got, allocated) = yield self.storage_client.allocate_buckets( new_storage_index(), renew_secret=new_secret(), cancel_secret=new_secret(), @@ -110,7 +110,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (already_got, allocated) = yield self.storage_server.allocate_buckets( + (already_got, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -118,7 +118,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): allocated_size=1024, canary=Referenceable(), ) - (already_got2, allocated2) = yield self.storage_server.allocate_buckets( + (already_got2, allocated2) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -146,7 +146,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -162,7 +162,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield abort_or_disconnect(allocated[0]) # Write different data with no complaint: - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -198,7 +198,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -219,7 +219,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 0 has partial write. yield allocated[0].callRemote("write", 0, b"1" * 512) - (already_got, _) = yield self.storage_server.allocate_buckets( + (already_got, _) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -242,7 +242,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -261,7 +261,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[2].callRemote("write", 0, b"3" * 512) yield allocated[2].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {1, 2}) self.assertEqual( @@ -282,7 +282,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -307,7 +307,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -321,7 +321,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 5, b"1" * 20) yield allocated[0].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {0}) self.assertEqual((yield buckets[0].callRemote("read", 0, 25)), b"1" * 25) @@ -346,7 +346,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``IStorageServer.get_buckets()`` implementations. """ storage_index = new_storage_index() - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret=new_secret(), cancel_secret=new_secret(), @@ -362,7 +362,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 2 is partially written yield allocated[2].callRemote("write", 0, b"1" * 5) - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {1}) @inlineCallbacks @@ -375,7 +375,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): length = 256 * 17 storage_index = new_storage_index() - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret=new_secret(), cancel_secret=new_secret(), @@ -388,7 +388,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, total_data) yield allocated[0].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) bucket = buckets[0] for start, to_read in [ (0, 250), # fraction @@ -408,7 +408,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): def create_share(self): """Create a share, return the storage index.""" storage_index = new_storage_index() - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret=new_secret(), cancel_secret=new_secret(), @@ -429,7 +429,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): behavior is opaque at this level of abstraction). """ storage_index = yield self.create_share() - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") @inlineCallbacks @@ -440,7 +440,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): abstraction). """ storage_index = yield self.create_share() - yield self.storage_server.advise_corrupt_share( + yield self.storage_client.advise_corrupt_share( b"immutable", storage_index, 0, b"ono" ) @@ -453,7 +453,7 @@ class IStorageServerMutableAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s mutable APIs. - ``self.storage_server`` is expected to provide ``IStorageServer``. + ``self.storage_client`` is expected to provide ``IStorageServer``. ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. """ @@ -464,7 +464,7 @@ class IStorageServerMutableAPIsTestsMixin(object): def staraw(self, *args, **kwargs): """Like ``slot_testv_and_readv_and_writev``, but less typing.""" - return self.storage_server.slot_testv_and_readv_and_writev(*args, **kwargs) + return self.storage_client.slot_testv_and_readv_and_writev(*args, **kwargs) @inlineCallbacks def test_STARAW_reads_after_write(self): @@ -760,7 +760,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(written, True) - reads = yield self.storage_server.slot_readv( + reads = yield self.storage_client.slot_readv( storage_index, shares=[0, 1], # Whole thing, partial, going beyond the edge, completely outside @@ -791,7 +791,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(written, True) - reads = yield self.storage_server.slot_readv( + reads = yield self.storage_client.slot_readv( storage_index, shares=[], readv=[(0, 7)], @@ -820,7 +820,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(written, True) - yield self.storage_server.advise_corrupt_share( + yield self.storage_client.advise_corrupt_share( b"mutable", storage_index, 0, b"ono" ) @@ -843,8 +843,8 @@ class _FoolscapMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_server = self._get_native_server().get_storage_server() - self.assertTrue(IStorageServer.providedBy(self.storage_server)) + self.storage_client = self._get_native_server().get_storage_server() + self.assertTrue(IStorageServer.providedBy(self.storage_client)) @inlineCallbacks def tearDown(self): @@ -856,10 +856,10 @@ class _FoolscapMixin(SystemTestMixin): """ Disconnect and then reconnect with a new ``IStorageServer``. """ - current = self.storage_server + current = self.storage_client yield self.bounce_client(0) - self.storage_server = self._get_native_server().get_storage_server() - assert self.storage_server is not current + self.storage_client = self._get_native_server().get_storage_server() + assert self.storage_client is not current class FoolscapSharedAPIsTests( From b7be91e3d0d02fb2d83957acfbc2b9f414e25c60 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Oct 2021 13:17:07 -0400 Subject: [PATCH 048/916] First test for leases. --- src/allmydata/test/test_istorageserver.py | 29 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index b34964463..3aa48c35b 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,15 +19,16 @@ if PY2: # fmt: on from random import Random +import time from twisted.internet.defer import inlineCallbacks, returnValue from foolscap.api import Referenceable, RemoteException -from allmydata.interfaces import IStorageServer +from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase - +from allmydata.storage.server import StorageServer # not a IStorageServer!! # Use random generator with known seed, so results are reproducible if tests # are run in the same order. @@ -79,6 +80,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``self.disconnect()`` should disconnect and then reconnect, creating a new ``self.storage_client``. Some implementations may wish to skip tests using this; HTTP has no notion of disconnection. + + ``self.server`` is expected to be the corresponding + ``allmydata.storage.server.StorageServer`` instance. """ @inlineCallbacks @@ -444,7 +448,17 @@ class IStorageServerImmutableAPIsTestsMixin(object): b"immutable", storage_index, 0, b"ono" ) - # TODO allocate_buckets creates lease + @inlineCallbacks + def test_allocate_buckets_creates_lease(self): + """ + When buckets are created using ``allocate_buckets()``, a lease is + created once writing is done. + """ + storage_index = yield self.create_share() + [lease] = self.server.get_leases(storage_index) + # Lease expires in 31 days. + assert lease.get_expiration_time() - time.time() > (31 * 24 * 60 * 60 - 10) + # TODO add_lease renews lease if existing storage index and secret # TODO add_lease creates new lease if new secret @@ -455,6 +469,9 @@ class IStorageServerMutableAPIsTestsMixin(object): ``self.storage_client`` is expected to provide ``IStorageServer``. + ``self.server`` is expected to be the corresponding + ``allmydata.storage.server.StorageServer`` instance. + ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. """ @@ -845,6 +862,12 @@ class _FoolscapMixin(SystemTestMixin): yield self.set_up_nodes(1) self.storage_client = self._get_native_server().get_storage_server() self.assertTrue(IStorageServer.providedBy(self.storage_client)) + self.server = None + for s in self.clients[0].services: + if isinstance(s, StorageServer): + self.server = s + break + assert self.server is not None, "Couldn't find StorageServer" @inlineCallbacks def tearDown(self): From 7d04e6ab8613010e98f807fa95826451d79b2d1d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 10:45:10 -0400 Subject: [PATCH 049/916] news fragment --- newsfragments/LFS-01-007.security | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/LFS-01-007.security diff --git a/newsfragments/LFS-01-007.security b/newsfragments/LFS-01-007.security new file mode 100644 index 000000000..75d9904a2 --- /dev/null +++ b/newsfragments/LFS-01-007.security @@ -0,0 +1,2 @@ +The storage protocol operation ``add_lease`` now safely rejects an attempt to add a 4,294,967,296th lease to an immutable share. +Previously this failed with an error after recording the new lease in the share file, resulting in the share file losing track of a one previous lease. From f60bbbd3e201b1b49598b7b2b6a05ad8db8a3dfd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 10:45:58 -0400 Subject: [PATCH 050/916] make it possible to test this behavior of `add_lease` --- src/allmydata/storage/immutable.py | 66 ++++++++++++++++++++++++++++-- src/allmydata/test/test_storage.py | 59 +++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index b8b18f140..acd09854f 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -53,13 +53,64 @@ from allmydata.storage.common import UnknownImmutableContainerVersionError # then the value stored in this field will be the actual share data length # modulo 2**32. +def _fix_lease_count_format(lease_count_format): + """ + Turn a single character struct format string into a format string suitable + for use in encoding and decoding the lease count value inside a share + file, if possible. + + :param str lease_count_format: A single character format string like + ``"B"`` or ``"L"``. + + :raise ValueError: If the given format string is not suitable for use + encoding and decoding a lease count. + + :return str: A complete format string which can safely be used to encode + and decode lease counts in a share file. + """ + if len(lease_count_format) != 1: + raise ValueError( + "Cannot construct ShareFile with lease_count_format={!r}; " + "format must accept a single value".format( + lease_count_format, + ), + ) + # Make it big-endian with standard size so all platforms agree on the + # result. + fixed = ">" + lease_count_format + if struct.calcsize(fixed) > 4: + # There is only room for at most 4 bytes in the share file format so + # we can't allow any larger formats. + raise ValueError( + "Cannot construct ShareFile with lease_count_format={!r}; " + "size must be smaller than size of '>L'".format( + lease_count_format, + ), + ) + return fixed + + class ShareFile(object): + """ + Support interaction with persistent storage of a share. + + :ivar str _lease_count_format: The format string which is used to encode + and decode the lease count inside the share file. As stated in the + comment in this module there is room for at most 4 bytes in this part + of the file. A format string that works on fewer bytes is allowed to + restrict the number of leases allowed in the share file to a smaller + number than could be supported by using the full 4 bytes. This is + mostly of interest for testing. + """ LEASE_SIZE = struct.calcsize(">L32s32sL") sharetype = "immutable" - def __init__(self, filename, max_size=None, create=False): + def __init__(self, filename, max_size=None, create=False, lease_count_format="L"): """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ precondition((max_size is not None) or (not create), max_size, create) + + self._lease_count_format = _fix_lease_count_format(lease_count_format) + self._lease_count_size = struct.calcsize(self._lease_count_format) self.home = filename self._max_size = max_size if create: @@ -126,12 +177,21 @@ class ShareFile(object): def _read_num_leases(self, f): f.seek(0x08) - (num_leases,) = struct.unpack(">L", f.read(4)) + (num_leases,) = struct.unpack( + self._lease_count_format, + f.read(self._lease_count_size), + ) return num_leases def _write_num_leases(self, f, num_leases): + self._write_encoded_num_leases( + f, + struct.pack(self._lease_count_format, num_leases), + ) + + def _write_encoded_num_leases(self, f, encoded_num_leases): f.seek(0x08) - f.write(struct.pack(">L", num_leases)) + f.write(encoded_num_leases) def _truncate_leases(self, f, num_leases): f.truncate(self._lease_offset + num_leases * self.LEASE_SIZE) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index d18960a1e..0a37dffc2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -19,6 +19,7 @@ import platform import stat import struct import shutil +from functools import partial from uuid import uuid4 from twisted.trial import unittest @@ -3009,8 +3010,8 @@ class Stats(unittest.TestCase): class ShareFileTests(unittest.TestCase): """Tests for allmydata.storage.immutable.ShareFile.""" - def get_sharefile(self): - sf = ShareFile(self.mktemp(), max_size=1000, create=True) + def get_sharefile(self, **kwargs): + sf = ShareFile(self.mktemp(), max_size=1000, create=True, **kwargs) sf.write_share_data(0, b"abc") sf.write_share_data(2, b"DEF") # Should be b'abDEF' now. @@ -3039,3 +3040,57 @@ class ShareFileTests(unittest.TestCase): sf = self.get_sharefile() with self.assertRaises(IndexError): sf.cancel_lease(b"garbage") + + def test_long_lease_count_format(self): + """ + ``ShareFile.__init__`` raises ``ValueError`` if the lease count format + given is longer than one character. + """ + with self.assertRaises(ValueError): + self.get_sharefile(lease_count_format="BB") + + def test_large_lease_count_format(self): + """ + ``ShareFile.__init__`` raises ``ValueError`` if the lease count format + encodes to a size larger than 8 bytes. + """ + with self.assertRaises(ValueError): + self.get_sharefile(lease_count_format="Q") + + def test_avoid_lease_overflow(self): + """ + If the share file already has the maximum number of leases supported then + ``ShareFile.add_lease`` raises ``struct.error`` and makes no changes + to the share file contents. + """ + make_lease = partial( + LeaseInfo, + renew_secret=b"r" * 32, + cancel_secret=b"c" * 32, + expiration_time=2 ** 31, + ) + # Make it a little easier to reach the condition by limiting the + # number of leases to only 255. + sf = self.get_sharefile(lease_count_format="B") + + # Add the leases. + for i in range(2 ** 8 - 1): + lease = make_lease(owner_num=i) + sf.add_lease(lease) + + # Capture the state of the share file at this point so we can + # determine whether the next operation modifies it or not. + with open(sf.home, "rb") as f: + before_data = f.read() + + # It is not possible to add a 256th lease. + lease = make_lease(owner_num=256) + with self.assertRaises(struct.error): + sf.add_lease(lease) + + # Compare the share file state to what we captured earlier. Any + # change is a bug. + with open(sf.home, "rb") as f: + after_data = f.read() + + self.assertEqual(before_data, after_data) From df64bbb1e443cbbad272067e7716b4d9a3f3408d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 10:50:28 -0400 Subject: [PATCH 051/916] fail to encode the lease count *before* changing anything This preserves the failure behavior - `struct.error` is raised - but leaves the actual share file contents untouched if the new lease count cannot be encoded. There are still two separate write operations so it is conceivable that some other problem could cause `write_lease_record` to happen but `write_encoded_num_leases` not to happen. As far as I can tell we have severely limited options for addressing that problem in general as long as share files are backed by a POSIX filesystem. However, by removing the failure mode that depends on user input, it may be that this is all that is needed to close the *security* hole. --- src/allmydata/storage/immutable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index acd09854f..887ccc931 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -209,8 +209,11 @@ class ShareFile(object): def add_lease(self, lease_info): with open(self.home, 'rb+') as f: num_leases = self._read_num_leases(f) + # Before we write the new lease record, make sure we can encode + # the new lease count. + new_lease_count = struct.pack(self._lease_count_format, num_leases + 1) self._write_lease_record(f, num_leases, lease_info) - self._write_num_leases(f, num_leases+1) + self._write_encoded_num_leases(f, new_lease_count) def renew_lease(self, renew_secret, new_expire_time): for i,lease in enumerate(self.get_leases()): From 4a5e4be0069ed41eb25deeb09828de76e1db041d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 14:35:11 -0400 Subject: [PATCH 052/916] news fragment --- newsfragments/LFS-01-008.security | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/LFS-01-008.security diff --git a/newsfragments/LFS-01-008.security b/newsfragments/LFS-01-008.security new file mode 100644 index 000000000..5d6c07ab5 --- /dev/null +++ b/newsfragments/LFS-01-008.security @@ -0,0 +1,2 @@ +The storage protocol operation ``readv`` now safely rejects attempts to read negative lengths. +Previously these read requests were satisfied with the complete contents of the share file (including trailing metadata) starting from the specified offset. From 5e58b62979b7ad2c813f95e1f50c550da1f69f36 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 14:36:24 -0400 Subject: [PATCH 053/916] Add a test for negative offset or length to MutableShareFile.readv --- src/allmydata/test/strategies.py | 15 ++++ src/allmydata/test/test_storage.py | 117 +++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index c0f558ef6..2bb23a373 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -16,6 +16,7 @@ from hypothesis.strategies import ( one_of, builds, binary, + integers, ) from ..uri import ( @@ -119,3 +120,17 @@ def dir2_mdmf_capabilities(): MDMFDirectoryURI, mdmf_capabilities(), ) + +def offsets(min_value=0, max_value=2 ** 16): + """ + Build ``int`` values that could be used as valid offsets into a sequence + (such as share data in a share file). + """ + return integers(min_value, max_value) + +def lengths(min_value=1, max_value=2 ** 16): + """ + Build ``int`` values that could be used as valid lengths of data (such as + share data in a share file). + """ + return integers(min_value, max_value) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 0a37dffc2..f19073f3e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,6 +13,9 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str +from io import ( + BytesIO, +) import time import os.path import platform @@ -59,6 +62,10 @@ from allmydata.storage_client import ( ) from .common import LoggingServiceParent, ShouldFailMixin from .common_util import FakeCanary +from .strategies import ( + offsets, + lengths, +) class UtilTests(unittest.TestCase): @@ -3094,3 +3101,113 @@ class ShareFileTests(unittest.TestCase): after_data = f.read() self.assertEqual(before_data, after_data) + + +class MutableShareFileTests(unittest.TestCase): + """ + Tests for allmydata.storage.mutable.MutableShareFile. + """ + def get_sharefile(self): + return MutableShareFile(self.mktemp()) + + @given( + nodeid=strategies.just(b"x" * 20), + write_enabler=strategies.just(b"y" * 32), + datav=strategies.lists( + # Limit the max size of these so we don't write *crazy* amounts of + # data to disk. + strategies.tuples(offsets(), strategies.binary(max_size=2 ** 8)), + max_size=2 ** 8, + ), + new_length=offsets(), + ) + def test_readv_reads_share_data(self, nodeid, write_enabler, datav, new_length): + """ + ``MutableShareFile.readv`` returns bytes from the share data portion + of the share file. + """ + sf = self.get_sharefile() + sf.create(my_nodeid=nodeid, write_enabler=write_enabler) + sf.writev(datav=datav, new_length=new_length) + + # Apply all of the writes to a simple in-memory buffer so we can + # resolve the final state of the share data. In particular, this + # helps deal with overlapping writes which otherwise make it tricky to + # figure out what data to expect to be able to read back. + buf = BytesIO() + for (offset, data) in datav: + buf.seek(offset) + buf.write(data) + buf.truncate(new_length) + + # Using that buffer, determine the expected result of a readv for all + # of the data just written. + def read_from_buf(offset, length): + buf.seek(offset) + return buf.read(length) + expected_data = list( + read_from_buf(offset, len(data)) + for (offset, data) + in datav + ) + + # Perform a read that gives back all of the data written to the share + # file. + read_vectors = list((offset, len(data)) for (offset, data) in datav) + read_data = sf.readv(read_vectors) + + # Make sure the read reproduces the value we computed using our local + # buffer. + self.assertEqual(expected_data, read_data) + + @given( + nodeid=strategies.just(b"x" * 20), + write_enabler=strategies.just(b"y" * 32), + readv=strategies.lists(strategies.tuples(offsets(), lengths()), min_size=1), + random=strategies.randoms(), + ) + def test_readv_rejects_negative_length(self, nodeid, write_enabler, readv, random): + """ + If a negative length is given to ``MutableShareFile.readv`` in a read + vector then ``AssertionError`` is raised. + """ + # Pick a read vector to break with a negative value + readv_index = random.randrange(len(readv)) + # Decide on whether we're breaking offset or length + offset_or_length = random.randrange(2) + + # A helper function that will take a valid offset and length and break + # one of them. + def corrupt(break_length, offset, length): + if break_length: + # length must not be 0 or flipping the sign does nothing + # length must not be negative or flipping the sign *fixes* it + assert length > 0 + return (offset, -length) + else: + if offset > 0: + # We can break offset just by flipping the sign. + return (-offset, length) + else: + # Otherwise it has to be zero. If it was negative, what's + # going on? + assert offset == 0 + # Since we can't just flip the sign on 0 to break things, + # replace a 0 offset with a simple negative value. All + # other negative values will be tested by the `offset > 0` + # case above. + return (-1, length) + + # Break the read vector very slightly! + broken_readv = readv[:] + broken_readv[readv_index] = corrupt( + offset_or_length, + *broken_readv[readv_index] + ) + + sf = self.get_sharefile() + sf.create(my_nodeid=nodeid, write_enabler=write_enabler) + + # A read with a broken read vector is an error. + with self.assertRaises(AssertionError): + sf.readv(broken_readv) From 3cd9a02c810f6aa1dba9dbd664980b49bec39048 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 20:13:24 -0400 Subject: [PATCH 054/916] Reject negative lengths in MutableShareFile._read_share_data and readv --- src/allmydata/storage/mutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 2ef0c3215..cdb4faeaf 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -120,6 +120,7 @@ class MutableShareFile(object): def _read_share_data(self, f, offset, length): precondition(offset >= 0) + precondition(length >= 0) data_length = self._read_data_length(f) if offset+length > data_length: # reads beyond the end of the data are truncated. Reads that @@ -454,4 +455,3 @@ def create_mutable_sharefile(filename, my_nodeid, write_enabler, parent): ms.create(my_nodeid, write_enabler) del ms return MutableShareFile(filename, parent) - From 4b8b6052f33c9513986170e5b6ad3066747b7560 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Oct 2021 09:05:48 -0400 Subject: [PATCH 055/916] Finish testing leases on immutables. --- src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++---- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 3aa48c35b..b6d3e4b9a 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,7 +19,6 @@ if PY2: # fmt: on from random import Random -import time from twisted.internet.defer import inlineCallbacks, returnValue @@ -82,7 +81,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): this; HTTP has no notion of disconnection. ``self.server`` is expected to be the corresponding - ``allmydata.storage.server.StorageServer`` instance. + ``allmydata.storage.server.StorageServer`` instance. Time should be + instrumented, such that ``self.fake_time()`` and ``self.fake_sleep()`` + return and advance the server time, respectively. """ @inlineCallbacks @@ -412,10 +413,12 @@ class IStorageServerImmutableAPIsTestsMixin(object): def create_share(self): """Create a share, return the storage index.""" storage_index = new_storage_index() + renew_secret = new_secret() + cancel_secret = new_secret() (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, - renew_secret=new_secret(), - cancel_secret=new_secret(), + renew_secret=renew_secret, + cancel_secret=cancel_secret, sharenums=set(range(1)), allocated_size=10, canary=Referenceable(), @@ -423,7 +426,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, b"0123456789") yield allocated[0].callRemote("close") - returnValue(storage_index) + returnValue((storage_index, renew_secret, cancel_secret)) @inlineCallbacks def test_bucket_advise_corrupt_share(self): @@ -432,7 +435,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``IStorageServer.get_buckets()`` does not result in error (other behavior is opaque at this level of abstraction). """ - storage_index = yield self.create_share() + storage_index, _, _ = yield self.create_share() buckets = yield self.storage_client.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") @@ -443,7 +446,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): result in error (other behavior is opaque at this level of abstraction). """ - storage_index = yield self.create_share() + storage_index, _, _ = yield self.create_share() yield self.storage_client.advise_corrupt_share( b"immutable", storage_index, 0, b"ono" ) @@ -454,13 +457,49 @@ class IStorageServerImmutableAPIsTestsMixin(object): When buckets are created using ``allocate_buckets()``, a lease is created once writing is done. """ - storage_index = yield self.create_share() + storage_index, _, _ = yield self.create_share() [lease] = self.server.get_leases(storage_index) # Lease expires in 31 days. - assert lease.get_expiration_time() - time.time() > (31 * 24 * 60 * 60 - 10) + assert lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) - # TODO add_lease renews lease if existing storage index and secret - # TODO add_lease creates new lease if new secret + @inlineCallbacks + def test_add_lease_renewal(self): + """ + If the lease secret is reused, ``add_lease()`` extends the existing + lease. + """ + storage_index, renew_secret, cancel_secret = yield self.create_share() + [lease] = self.server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(178) + + # We renew the lease: + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + [lease] = self.server.get_leases(storage_index) + new_expiration_time = lease.get_expiration_time() + self.assertEqual(new_expiration_time - initial_expiration_time, 178) + + @inlineCallbacks + def test_add_new_lease(self): + """ + If a new lease secret is used, ``add_lease()`` creates a new lease. + """ + storage_index, _, _ = yield self.create_share() + [lease] = self.server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(167) + + # We create a new lease: + renew_secret = new_secret() + cancel_secret = new_secret() + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + [lease1, lease2] = self.server.get_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time) + self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) class IStorageServerMutableAPIsTestsMixin(object): @@ -868,6 +907,16 @@ class _FoolscapMixin(SystemTestMixin): self.server = s break assert self.server is not None, "Couldn't find StorageServer" + self._current_time = 123456 + self.server._get_current_time = self.fake_time + + def fake_time(self): + """Return the current fake, test-controlled, time.""" + return self._current_time + + def fake_sleep(self, seconds): + """Advance the fake time by the given number of seconds.""" + self._current_time += seconds @inlineCallbacks def tearDown(self): From 2a5dbcb05edc2fdc325028b8ca58be0d2fe5ac21 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Oct 2021 09:28:11 -0400 Subject: [PATCH 056/916] Tests for mutable leases. --- src/allmydata/test/test_istorageserver.py | 134 ++++++++++++++++++++-- 1 file changed, 122 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index b6d3e4b9a..fe494a9d4 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -460,7 +460,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): storage_index, _, _ = yield self.create_share() [lease] = self.server.get_leases(storage_index) # Lease expires in 31 days. - assert lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) + self.assertTrue( + lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) + ) @inlineCallbacks def test_add_lease_renewal(self): @@ -858,12 +860,8 @@ class IStorageServerMutableAPIsTestsMixin(object): ) @inlineCallbacks - def test_advise_corrupt_share(self): - """ - Calling ``advise_corrupt_share()`` on a mutable share does not - result in error (other behavior is opaque at this level of - abstraction). - """ + def create_slot(self): + """Create a slot with sharenum 0.""" secrets = self.new_secrets() storage_index = new_storage_index() (written, _) = yield self.staraw( @@ -875,16 +873,128 @@ class IStorageServerMutableAPIsTestsMixin(object): r_vector=[], ) self.assertEqual(written, True) + returnValue((secrets, storage_index)) + + @inlineCallbacks + def test_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a mutable share does not + result in error (other behavior is opaque at this level of + abstraction). + """ + secrets, storage_index = yield self.create_slot() yield self.storage_client.advise_corrupt_share( b"mutable", storage_index, 0, b"ono" ) - # TODO STARAW creates lease for new data - # TODO STARAW renews lease if same secret is used on existing data - # TODO STARAW creates new lease for existing data if new secret is given - # TODO add_lease renews lease if existing storage index and secret - # TODO add_lease creates new lease if new secret + @inlineCallbacks + def test_STARAW_create_lease(self): + """ + When STARAW creates a new slot, it also creates a lease. + """ + _, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + # Lease expires in 31 days. + self.assertTrue( + lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) + ) + + @inlineCallbacks + def test_STARAW_renews_lease(self): + """ + When STARAW is run on an existing slot with same renewal secret, it + renews the lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expire = lease.get_expiration_time() + + # Time passes... + self.fake_sleep(17) + + # We do another write: + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"1234567")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # The lease has been renewed: + [lease] = self.server.get_slot_leases(storage_index) + self.assertEqual(lease.get_expiration_time() - initial_expire, 17) + + @inlineCallbacks + def test_STARAW_new_lease(self): + """ + When STARAW is run with a new renewal secret on an existing slot, it + adds a new lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expire = lease.get_expiration_time() + + # Time passes... + self.fake_sleep(19) + + # We do another write: + (written, _) = yield self.staraw( + storage_index, + (secrets[0], new_secret(), new_secret()), + tw_vectors={ + 0: ([], [(0, b"1234567")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # A new lease was added: + [lease1, lease2] = self.server.get_slot_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expire) + self.assertEqual(lease2.get_expiration_time() - initial_expire, 19) + + @inlineCallbacks + def test_add_lease_renewal(self): + """ + If the lease secret is reused, ``add_lease()`` extends the existing + lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(178) + + # We renew the lease: + yield self.storage_client.add_lease(storage_index, secrets[1], secrets[2]) + [lease] = self.server.get_slot_leases(storage_index) + new_expiration_time = lease.get_expiration_time() + self.assertEqual(new_expiration_time - initial_expiration_time, 178) + + @inlineCallbacks + def test_add_new_lease(self): + """ + If a new lease secret is used, ``add_lease()`` creates a new lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(167) + + # We create a new lease: + renew_secret = new_secret() + cancel_secret = new_secret() + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + [lease1, lease2] = self.server.get_slot_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time) + self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) class _FoolscapMixin(SystemTestMixin): From e1dfee1d7b35d494b55178568612f6f648cf1205 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 19 Oct 2021 23:20:38 +0100 Subject: [PATCH 057/916] put notes under correct categories Signed-off-by: fenn-cs --- NEWS.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 366e45907..e4fef833a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -59,7 +59,6 @@ Configuration Changes Documentation Changes --------------------- -- (`#3659 `_) - Documentation now has its own towncrier category. (`#3664 `_) - `tox -e docs` will treat warnings about docs as errors. (`#3666 `_) - The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. (`#3677 `_) @@ -75,6 +74,11 @@ Documentation Changes - The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. (`#3777 `_) - tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. (`#3782 `_) - The Great Black Swamp specification now describes the required authorization scheme. (`#3785 `_) +- The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. (`#3037 `_) +- The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. (`#3503 `_) +- The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. (`#3545 `_) +- The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. (`#3644 `_) +- You can run `make livehtml` in docs directory to invoke sphinx-autobuild. (`#3663 `_) Removed Features @@ -90,11 +94,6 @@ Removed Features Other Changes ------------- -- The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. (`#3037 `_) -- The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. (`#3503 `_) -- The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. (`#3545 `_) -- The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. (`#3644 `_) -- You can run `make livehtml` in docs directory to invoke sphinx-autobuild. (`#3663 `_) - Refactored test_introducer in web tests to use custom base test cases (`#3757 `_) From 20ad6cd9e79cc62b23c6de4c4dba8ff9300f7a2c Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 19 Oct 2021 23:57:52 +0100 Subject: [PATCH 058/916] iterate over args directly without indexing Signed-off-by: fenn-cs --- src/allmydata/util/eliotutil.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index f2272a731..fe431568f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -335,12 +335,7 @@ def log_call_deferred(action_type): kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} # Remove complex (unserializable) objects from positional args to # prevent eliot from throwing errors when it attempts serialization - args = tuple( - a[pos] - if is_json_serializable(a[pos]) - else str(a[pos]) - for pos in range(len(a)) - ) + args = tuple(arg if is_json_serializable(arg) else str(arg) for arg in a) with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. From 1e6265b87cdb5c0c04a79b69a82027edf07072a1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Oct 2021 17:24:29 -0600 Subject: [PATCH 059/916] update relnotes --- relnotes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relnotes.txt b/relnotes.txt index c97b42664..fc18f4e96 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -32,7 +32,7 @@ and FTP support. There are several dependency changes that will be interesting for distribution maintainers. -As well 196 bugs have been fixed since the last release. +In all, 240 issues have been fixed since the last release. Please see ``NEWS.rst`` for a more complete list of changes. From 4bfb9d21700b8084d5fb2c697ceeb7088dd97c37 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Oct 2021 17:25:34 -0600 Subject: [PATCH 060/916] correct previous-release version --- relnotes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relnotes.txt b/relnotes.txt index fc18f4e96..e5976a97b 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -15,7 +15,7 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.15.0, released on +The previous stable release of Tahoe-LAFS was v1.15.1, released on March 23rd, 2021. The major change in this release is the completion of the Python 3 From a7ce84f4d5a884e165232a4e009e345c976cabff Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Oct 2021 18:02:29 -0600 Subject: [PATCH 061/916] correct names, dates --- relnotes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index e5976a97b..2748bc4fa 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -151,10 +151,10 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -fenn-cs +fenn-cs + meejah on behalf of the Tahoe-LAFS team -September 16, 2021 +October 19, 2021 Planet Earth From 027df0982894642a5b7a8c645278106d5e8b118f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 Oct 2021 16:10:23 -0600 Subject: [PATCH 062/916] release two things: wheels, and a .tar.gz source dist --- docs/release-checklist.rst | 4 +--- tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index da1bbe16f..f943abb5d 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -120,10 +120,8 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2.py3-none-any.whl - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip Privileged Contributor diff --git a/tox.ini b/tox.ini index af79ea4c7..94c9835ad 100644 --- a/tox.ini +++ b/tox.ini @@ -264,4 +264,4 @@ basepython = python3 deps = commands = python setup.py update_version - python setup.py sdist --formats=bztar,gztar,zip bdist_wheel --universal + python setup.py sdist --formats=gztar bdist_wheel --universal From b8ff0e7fa913c4e6c73991c2ffd1a3278d688ceb Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 Oct 2021 16:11:03 -0600 Subject: [PATCH 063/916] news --- newsfragments/3735.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3735.feature diff --git a/newsfragments/3735.feature b/newsfragments/3735.feature new file mode 100644 index 000000000..5a86d5547 --- /dev/null +++ b/newsfragments/3735.feature @@ -0,0 +1 @@ +Tahoe-LAFS releases now have just a .tar.gz source release and a (universal) wheel From a8d3555ebb6eae1d65cf4cfc928357de8d9a2268 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 15:24:53 -0400 Subject: [PATCH 064/916] reference the eventually-public ticket number --- newsfragments/{LFS-01-001.security => 3819.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-001.security => 3819.security} (100%) diff --git a/newsfragments/LFS-01-001.security b/newsfragments/3819.security similarity index 100% rename from newsfragments/LFS-01-001.security rename to newsfragments/3819.security From 61a20e245029ecfa626aa5f522af2eb08b7e19d3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Oct 2021 10:10:53 -0400 Subject: [PATCH 065/916] Add concept of upload secret to immutable uploads. --- docs/proposed/http-storage-node-protocol.rst | 33 +++++++++++++++++--- newsfragments/3820.minor | 0 2 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3820.minor diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 521bf476d..16db0fed9 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -451,6 +451,7 @@ Details of the buckets to create are encoded in the request body. For example:: {"renew-secret": "efgh", "cancel-secret": "ijkl", + "upload-secret": "xyzf", "share-numbers": [1, 7, ...], "allocated-size": 12345} The response body includes encoded information about the created buckets. @@ -458,6 +459,8 @@ For example:: {"already-have": [1, ...], "allocated": [7, ...]} +The session secret is an opaque _byte_ string. + Discussion `````````` @@ -482,6 +485,13 @@ The response includes ``already-have`` and ``allocated`` for two reasons: This might be because a server has become unavailable and a remaining server needs to store more shares for the upload. It could also just be that the client's preferred servers have changed. +Regarding upload secrets, +the goal is for uploading and aborting (see next sections) to be authenticated by more than just the storage index. +In the future, we will want to generate them in a way that allows resuming/canceling when the client has issues. +In the short term, they can just be a random byte string. +The key security constraint is that each upload to each server has its own, unique upload key, +tied to uploading that particular storage index to this particular server. + ``PATCH /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -498,6 +508,12 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). +The request body looks this, with data and upload secret being bytes:: + + { "upload-secret": "xyzf", "data": "thedata" } + +Responses: + * When a chunk that does not complete the share is successfully uploaded the response is ``OK``. The response body indicates the range of share data that has yet to be uploaded. That is:: @@ -522,6 +538,10 @@ The server must recognize when all of the data has been received and mark the sh This cancels an *in-progress* upload. +The request body looks this:: + + { "upload-secret": "xyzf" } + The response code: * When the upload is still in progress and therefore the abort has succeeded, @@ -695,6 +715,7 @@ Immutable Data POST /v1/immutable/AAAAAAAAAAAAAAAA {"renew-secret": "efgh", "cancel-secret": "ijkl", + "upload-secret": "xyzf", "share-numbers": [1, 7], "allocated-size": 48} 200 OK @@ -704,25 +725,29 @@ Immutable Data PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 0-15/48 - + + {"upload-secret": b"xyzf", "data": "first 16 bytes!!" 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 16-31/48 - + + {"upload-secret": "xyzf", "data": "second 16 bytes!" 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 32-47/48 - + + {"upload-secret": "xyzf", "data": "final 16 bytes!!" 201 CREATED #. Download the content of the previously uploaded immutable share ``7``:: - GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7&offset=0&size=48 + GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7 + Range: bytes=0-47 200 OK diff --git a/newsfragments/3820.minor b/newsfragments/3820.minor new file mode 100644 index 000000000..e69de29bb From e0c8bab5d7a97d539a5364c962cd5861430432a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Oct 2021 10:32:44 -0400 Subject: [PATCH 066/916] Add proposal on how to generate upload secret. --- docs/proposed/http-storage-node-protocol.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 16db0fed9..d5b6653be 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -459,7 +459,13 @@ For example:: {"already-have": [1, ...], "allocated": [7, ...]} -The session secret is an opaque _byte_ string. +The uplaod secret is an opaque _byte_ string. +It will be generated by hashing a combination of:b + +1. A tag. +2. The storage index, so it's unique across different source files. +3. The server ID, so it's unique across different servers. +4. The convergence secret, so that servers can't guess the upload secret for other servers. Discussion `````````` @@ -492,6 +498,13 @@ In the short term, they can just be a random byte string. The key security constraint is that each upload to each server has its own, unique upload key, tied to uploading that particular storage index to this particular server. +Rejected designs for upload secrets: + +* Upload secret per share number. + In order to make the secret unguessable by attackers, which includes other servers, + it must contain randomness. + Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better. + ``PATCH /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 6c0ca0b88592bffd8954cf06142cd962c1a3c654 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 08:41:09 -0400 Subject: [PATCH 067/916] try making windows let us use longer paths --- src/allmydata/test/test_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index d61942839..8e7aa9d27 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -493,7 +493,7 @@ class DownloadTest(_Base, unittest.TestCase): d.addCallback(_done) return d - def test_simultaneous_onefails_onecancelled(self): + def test_simul_1fail_1cancel(self): # This exercises an mplayer behavior in ticket #1154. I believe that # mplayer made two simultaneous webapi GET requests: first one for an # index region at the end of the (mp3/video) file, then one for the From d8c466e9a7ba5f121cb6d9f891569db7e01e87b6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 12:35:11 -0400 Subject: [PATCH 068/916] try to explain `lease_count_format` more clearly --- src/allmydata/storage/immutable.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 887ccc931..e23abb080 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -106,7 +106,30 @@ class ShareFile(object): sharetype = "immutable" def __init__(self, filename, max_size=None, create=False, lease_count_format="L"): - """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ + """ + Initialize a ``ShareFile``. + + :param Optional[int] max_size: If given, the maximum number of bytes + that this ``ShareFile`` will accept to be stored. ``write`` will + accept in total. + + :param bool create: If ``True``, create the file (and fail if it + exists already). ``max_size`` must not be ``None`` in this case. + If ``False``, open an existing file for reading. + + :param str lease_count_format: A format character to use to encode and + decode the number of leases in the share file. There are only 4 + bytes available in the file so the format must be 4 bytes or + smaller. If different formats are used at different times with + the same share file, the result will likely be nonsense. + + This parameter is intended for the test suite to use to be able to + exercise values near the maximum encodeable value without having + to create billions of leases. + + :raise ValueError: If the encoding of ``lease_count_format`` is too + large or if it is not a single format character. + """ precondition((max_size is not None) or (not create), max_size, create) self._lease_count_format = _fix_lease_count_format(lease_count_format) From bcdfb8155c28c94e75b8e7acc7344dc1f01aa798 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 12:53:17 -0400 Subject: [PATCH 069/916] give the news fragment its proper name --- newsfragments/{LFS-01-007.security => 3821.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-007.security => 3821.security} (100%) diff --git a/newsfragments/LFS-01-007.security b/newsfragments/3821.security similarity index 100% rename from newsfragments/LFS-01-007.security rename to newsfragments/3821.security From 7f3d9316d2dc8d2fe99b211a006bc45749f184c3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 12:59:26 -0400 Subject: [PATCH 070/916] Give the news fragment its real name --- newsfragments/{LFS-01-008.security => 3822.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-008.security => 3822.security} (100%) diff --git a/newsfragments/LFS-01-008.security b/newsfragments/3822.security similarity index 100% rename from newsfragments/LFS-01-008.security rename to newsfragments/3822.security From ce30f9dd0663ba22a985571f8029ad35026bb91e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 15:04:45 -0400 Subject: [PATCH 071/916] clean up copyediting errors --- src/allmydata/storage/immutable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index e23abb080..55bcdda64 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -110,8 +110,7 @@ class ShareFile(object): Initialize a ``ShareFile``. :param Optional[int] max_size: If given, the maximum number of bytes - that this ``ShareFile`` will accept to be stored. ``write`` will - accept in total. + that this ``ShareFile`` will accept to be stored. :param bool create: If ``True``, create the file (and fail if it exists already). ``max_size`` must not be ``None`` in this case. From bb5b26638de4729254b6febb2549a08cd82471e7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:20:53 -0400 Subject: [PATCH 072/916] news fragment --- newsfragments/LFS-01-005.security | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/LFS-01-005.security diff --git a/newsfragments/LFS-01-005.security b/newsfragments/LFS-01-005.security new file mode 100644 index 000000000..135b2487c --- /dev/null +++ b/newsfragments/LFS-01-005.security @@ -0,0 +1,3 @@ +The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information. +Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. +Now this operation will fail with an exception and the lease will not be created. From c77425693769ecd1ce73fcf064b62ef0eaf29ec6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:25:01 -0400 Subject: [PATCH 073/916] Add a test for ``remote_add_lease`` with respect to reserved space --- src/allmydata/interfaces.py | 2 ++ src/allmydata/test/common.py | 37 ++++++++++++++++++++++++++++ src/allmydata/test/common_storage.py | 33 +++++++++++++++++++++++++ src/allmydata/test/test_storage.py | 35 +++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 src/allmydata/test/common_storage.py diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 5522663ee..f055a01e2 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -52,6 +52,8 @@ WriteEnablerSecret = Hash # used to protect mutable share modifications LeaseRenewSecret = Hash # used to protect lease renewal requests LeaseCancelSecret = Hash # was used to protect lease cancellation requests +class NoSpace(Exception): + """Storage space was not available for a space-allocating operation.""" class DataTooLargeError(Exception): """The write went past the expected size of the bucket.""" diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 0f2dc7c62..38282297a 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -87,6 +87,7 @@ from allmydata.interfaces import ( SDMF_VERSION, MDMF_VERSION, IAddressFamily, + NoSpace, ) from allmydata.check_results import CheckResults, CheckAndRepairResults, \ DeepCheckResults, DeepCheckAndRepairResults @@ -139,6 +140,42 @@ EMPTY_CLIENT_CONFIG = config_from_string( "" ) +@attr.s +class FakeDisk(object): + """ + Just enough of a disk to be able to report free / used information. + """ + total = attr.ib() + used = attr.ib() + + def use(self, num_bytes): + """ + Mark some amount of available bytes as used (and no longer available). + + :param int num_bytes: The number of bytes to use. + + :raise NoSpace: If there are fewer bytes available than ``num_bytes``. + + :return: ``None`` + """ + if num_bytes > self.total - self.used: + raise NoSpace() + self.used += num_bytes + + @property + def available(self): + return self.total - self.used + + def get_disk_stats(self, whichdir, reserved_space): + avail = self.available + return { + 'total': self.total, + 'free_for_root': avail, + 'free_for_nonroot': avail, + 'used': self.used, + 'avail': avail - reserved_space, + } + @attr.s class MemoryIntroducerClient(object): diff --git a/src/allmydata/test/common_storage.py b/src/allmydata/test/common_storage.py new file mode 100644 index 000000000..f020a8146 --- /dev/null +++ b/src/allmydata/test/common_storage.py @@ -0,0 +1,33 @@ + +from .common_util import ( + FakeCanary, +) + +def upload_immutable(storage_server, storage_index, renew_secret, cancel_secret, shares): + """ + Synchronously upload some immutable shares to a ``StorageServer``. + + :param allmydata.storage.server.StorageServer storage_server: The storage + server object to use to perform the upload. + + :param bytes storage_index: The storage index for the immutable shares. + + :param bytes renew_secret: The renew secret for the implicitly created lease. + :param bytes cancel_secret: The cancel secret for the implicitly created lease. + + :param dict[int, bytes] shares: A mapping from share numbers to share data + to upload. The data for all shares must be of the same length. + + :return: ``None`` + """ + already, writers = storage_server.remote_allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + shares.keys(), + len(next(iter(shares.values()))), + canary=FakeCanary(), + ) + for shnum, writer in writers.items(): + writer.remote_write(0, shares[shnum]) + writer.remote_close() diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f19073f3e..67d690047 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -60,8 +60,15 @@ from allmydata.test.no_network import NoNetworkServer from allmydata.storage_client import ( _StorageServer, ) -from .common import LoggingServiceParent, ShouldFailMixin +from .common import ( + LoggingServiceParent, + ShouldFailMixin, + FakeDisk, +) from .common_util import FakeCanary +from .common_storage import ( + upload_immutable, +) from .strategies import ( offsets, lengths, @@ -651,6 +658,32 @@ class Server(unittest.TestCase): self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + def test_reserved_space_immutable_lease(self): + """ + If there is not enough available space to store an additional lease then + ``remote_add_lease`` fails with ``NoSpace`` when an attempt is made to + use it to create a new lease. + """ + disk = FakeDisk(total=1024, used=0) + self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) + + ss = self.create("test_reserved_space_immutable_lease") + + storage_index = b"x" * 16 + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + shares = {0: b"y" * 500} + upload_immutable(ss, storage_index, renew_secret, cancel_secret, shares) + + # use up all the available space + disk.use(disk.available) + + # Different secrets to produce a different lease, not a renewal. + renew_secret = b"R" * 32 + cancel_secret = b"C" * 32 + with self.assertRaises(interfaces.NoSpace): + ss.remote_add_lease(storage_index, renew_secret, cancel_secret) + def test_reserved_space(self): reserved = 10000 allocated = 0 From b3aa1e224f226fb09fdc38312c189369a7aa8847 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:27:27 -0400 Subject: [PATCH 074/916] Add a helper to LeaseInfo for computing size This lets some code LBYL and avoid writing if the lease won't fit in the immutable share in the space available. --- src/allmydata/storage/lease.py | 14 ++++++++++++-- src/allmydata/test/test_storage.py | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 187f32406..d3b3eef88 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -13,6 +13,9 @@ if PY2: import struct, time +# struct format for representation of a lease in an immutable share +IMMUTABLE_FORMAT = ">L32s32sL" + class LeaseInfo(object): def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, expiration_time=None, nodeid=None): @@ -39,12 +42,19 @@ class LeaseInfo(object): (self.owner_num, self.renew_secret, self.cancel_secret, - self.expiration_time) = struct.unpack(">L32s32sL", data) + self.expiration_time) = struct.unpack(IMMUTABLE_FORMAT, data) self.nodeid = None return self + def immutable_size(self): + """ + :return int: The size, in bytes, of the representation of this lease in an + immutable share file. + """ + return struct.calcsize(IMMUTABLE_FORMAT) + def to_immutable_data(self): - return struct.pack(">L32s32sL", + return struct.pack(IMMUTABLE_FORMAT, self.owner_num, self.renew_secret, self.cancel_secret, int(self.expiration_time)) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 67d690047..329953e99 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -117,6 +117,29 @@ class FakeStatsProvider(object): def register_producer(self, producer): pass + +class LeaseInfoTests(unittest.TestCase): + """ + Tests for ``LeaseInfo``. + """ + @given( + strategies.tuples( + strategies.integers(min_value=0, max_value=2 ** 31 - 1), + strategies.binary(min_size=32, max_size=32), + strategies.binary(min_size=32, max_size=32), + strategies.integers(min_value=0, max_value=2 ** 31 - 1), + strategies.binary(min_size=20, max_size=20), + ), + ) + def test_immutable_size(self, initializer_args): + """ + ``LeaseInfo.immutable_size`` returns the length of the result of + ``LeaseInfo.to_immutable_data``. + """ + info = LeaseInfo(*initializer_args) + self.assertEqual(len(info.to_immutable_data()), info.immutable_size()) + + class Bucket(unittest.TestCase): def make_workdir(self, name): basedir = os.path.join("storage", "Bucket", name) From 1264c3be1e225e0573aa1e5b30ffa52f5af2d3be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:35:13 -0400 Subject: [PATCH 075/916] Use `_add_or_renew_leases` helper consistently in StorageServer This will make it easier to add a new argument to the underlying `add_or_renew_lease` call. --- src/allmydata/storage/server.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 041783a4e..21c612a59 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -286,7 +286,7 @@ class StorageServer(service.MultiService, Referenceable): # to a particular owner. start = self._get_current_time() self.count("allocate") - alreadygot = set() + alreadygot = {} bucketwriters = {} # k: shnum, v: BucketWriter si_dir = storage_index_to_dir(storage_index) si_s = si_b2a(storage_index) @@ -318,9 +318,8 @@ class StorageServer(service.MultiService, Referenceable): # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): - alreadygot.add(shnum) - sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + alreadygot[shnum] = ShareFile(fn) + self._add_or_renew_leases(alreadygot.values(), lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -352,7 +351,7 @@ class StorageServer(service.MultiService, Referenceable): fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) self.add_latency("allocate", self._get_current_time() - start) - return alreadygot, bucketwriters + return set(alreadygot), bucketwriters def remote_allocate_buckets(self, storage_index, renew_secret, cancel_secret, @@ -392,8 +391,10 @@ class StorageServer(service.MultiService, Referenceable): lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) - for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(lease_info) + self._add_or_renew_leases( + self._iter_share_files(storage_index), + lease_info, + ) self.add_latency("add-lease", self._get_current_time() - start) return None @@ -611,12 +612,12 @@ class StorageServer(service.MultiService, Referenceable): """ Put the given lease onto the given shares. - :param dict[int, MutableShareFile] shares: The shares to put the lease - onto. + :param Iterable[Union[MutableShareFile, ShareFile]] shares: The shares + to put the lease onto. :param LeaseInfo lease_info: The lease to put on the shares. """ - for share in six.viewvalues(shares): + for share in shares: share.add_or_renew_lease(lease_info) def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 @@ -675,7 +676,7 @@ class StorageServer(service.MultiService, Referenceable): ) if renew_leases: lease_info = self._make_lease_info(renew_secret, cancel_secret) - self._add_or_renew_leases(remaining_shares, lease_info) + self._add_or_renew_leases(remaining_shares.values(), lease_info) # all done self.add_latency("writev", self._get_current_time() - start) From 4defc641a2da2b20898f15eb1c9234dcc1cbeb38 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:36:05 -0400 Subject: [PATCH 076/916] Have ShareFile only write a new lease if there is room for it StorageServer passes available space down so it can make the decision. ShareFile has to do it because `add_or_renew_lease` only *sometimes* adds a lease and only ShareFile knows when that is. --- src/allmydata/storage/immutable.py | 20 ++++++++++++++++++-- src/allmydata/storage/server.py | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 55bcdda64..ad2d19f5f 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -21,6 +21,7 @@ from zope.interface import implementer from allmydata.interfaces import ( RIBucketWriter, RIBucketReader, ConflictingWriteError, DataTooLargeError, + NoSpace, ) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition @@ -249,14 +250,29 @@ class ShareFile(object): return raise IndexError("unable to renew non-existent lease") - def add_or_renew_lease(self, lease_info): + def add_or_renew_lease(self, available_space, lease_info): + """ + Renew an existing lease if possible, otherwise allocate a new one. + + :param int available_space: The maximum number of bytes of storage to + commit in this operation. If more than this number of bytes is + required, raise ``NoSpace`` instead. + + :param LeaseInfo lease_info: The details of the lease to renew or add. + + :raise NoSpace: If more than ``available_space`` bytes is required to + complete the operation. In this case, no lease is added. + + :return: ``None`` + """ try: self.renew_lease(lease_info.renew_secret, lease_info.expiration_time) except IndexError: + if lease_info.immutable_size() > available_space: + raise NoSpace() self.add_lease(lease_info) - def cancel_lease(self, cancel_secret): """Remove a lease with the given cancel_secret. If the last lease is cancelled, the file will be removed. Return the number of bytes that diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 21c612a59..66d9df998 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -618,7 +618,7 @@ class StorageServer(service.MultiService, Referenceable): :param LeaseInfo lease_info: The lease to put on the shares. """ for share in shares: - share.add_or_renew_lease(lease_info) + share.add_or_renew_lease(self.get_available_space(), lease_info) def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, From e0ed04c1033f995aa5cf90f829b63d127cd290af Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 14:27:20 -0400 Subject: [PATCH 077/916] use SyncTestCase to get `expectThat` --- src/allmydata/test/test_storage.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 329953e99..5b5cfa89d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -25,6 +25,10 @@ import shutil from functools import partial from uuid import uuid4 +from testtools.matchers import ( + HasLength, +) + from twisted.trial import unittest from twisted.internet import defer @@ -64,6 +68,7 @@ from .common import ( LoggingServiceParent, ShouldFailMixin, FakeDisk, + SyncTestCase, ) from .common_util import FakeCanary from .common_storage import ( @@ -118,7 +123,7 @@ class FakeStatsProvider(object): pass -class LeaseInfoTests(unittest.TestCase): +class LeaseInfoTests(SyncTestCase): """ Tests for ``LeaseInfo``. """ @@ -137,7 +142,10 @@ class LeaseInfoTests(unittest.TestCase): ``LeaseInfo.to_immutable_data``. """ info = LeaseInfo(*initializer_args) - self.assertEqual(len(info.to_immutable_data()), info.immutable_size()) + self.expectThat( + info.to_immutable_data(), + HasLength(info.immutable_size()), + ) class Bucket(unittest.TestCase): From dd1ab2afe8299f8b96651112e4117ffb267ad054 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 14:27:45 -0400 Subject: [PATCH 078/916] Add a helper to compute the size of a lease in a mutable share --- src/allmydata/storage/lease.py | 14 ++++++++++++-- src/allmydata/test/test_storage.py | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index d3b3eef88..3453c1ecc 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -16,6 +16,9 @@ import struct, time # struct format for representation of a lease in an immutable share IMMUTABLE_FORMAT = ">L32s32sL" +# struct format for representation of a lease in a mutable share +MUTABLE_FORMAT = ">LL32s32s20s" + class LeaseInfo(object): def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, expiration_time=None, nodeid=None): @@ -53,6 +56,13 @@ class LeaseInfo(object): """ return struct.calcsize(IMMUTABLE_FORMAT) + def mutable_size(self): + """ + :return int: The size, in bytes, of the representation of this lease in a + mutable share file. + """ + return struct.calcsize(MUTABLE_FORMAT) + def to_immutable_data(self): return struct.pack(IMMUTABLE_FORMAT, self.owner_num, @@ -60,7 +70,7 @@ class LeaseInfo(object): int(self.expiration_time)) def to_mutable_data(self): - return struct.pack(">LL32s32s20s", + return struct.pack(MUTABLE_FORMAT, self.owner_num, int(self.expiration_time), self.renew_secret, self.cancel_secret, @@ -70,5 +80,5 @@ class LeaseInfo(object): (self.owner_num, self.expiration_time, self.renew_secret, self.cancel_secret, - self.nodeid) = struct.unpack(">LL32s32s20s", data) + self.nodeid) = struct.unpack(MUTABLE_FORMAT, data) return self diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 5b5cfa89d..9ce6482ea 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -140,12 +140,19 @@ class LeaseInfoTests(SyncTestCase): """ ``LeaseInfo.immutable_size`` returns the length of the result of ``LeaseInfo.to_immutable_data``. + + ``LeaseInfo.mutable_size`` returns the length of the result of + ``LeaseInfo.to_mutable_data``. """ info = LeaseInfo(*initializer_args) self.expectThat( info.to_immutable_data(), HasLength(info.immutable_size()), ) + self.expectThat( + info.to_mutable_data(), + HasLength(info.mutable_size()), + ) class Bucket(unittest.TestCase): From f789339a79c995d617e09010563e6f418e815067 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 15:16:56 -0400 Subject: [PATCH 079/916] Have MutableShare file only write a new lease if there is room for it This is analagous to the earlier ShareFile change. --- src/allmydata/storage/mutable.py | 25 ++++++++++++--- src/allmydata/test/common_storage.py | 32 +++++++++++++++++++ src/allmydata/test/test_storage.py | 47 ++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index cdb4faeaf..74c0d1051 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -13,7 +13,10 @@ if PY2: import os, stat, struct -from allmydata.interfaces import BadWriteEnablerError +from allmydata.interfaces import ( + BadWriteEnablerError, + NoSpace, +) from allmydata.util import idlib, log from allmydata.util.assertutil import precondition from allmydata.util.hashutil import timing_safe_compare @@ -289,7 +292,19 @@ class MutableShareFile(object): except IndexError: return - def add_lease(self, lease_info): + def add_lease(self, available_space, lease_info): + """ + Add a new lease to this share. + + :param int available_space: The maximum number of bytes of storage to + commit in this operation. If more than this number of bytes is + required, raise ``NoSpace`` instead. + + :raise NoSpace: If more than ``available_space`` bytes is required to + complete the operation. In this case, no lease is added. + + :return: ``None`` + """ precondition(lease_info.owner_num != 0) # 0 means "no lease here" with open(self.home, 'rb+') as f: num_lease_slots = self._get_num_lease_slots(f) @@ -297,6 +312,8 @@ class MutableShareFile(object): if empty_slot is not None: self._write_lease_record(f, empty_slot, lease_info) else: + if lease_info.mutable_size() > available_space: + raise NoSpace() self._write_lease_record(f, num_lease_slots, lease_info) def renew_lease(self, renew_secret, new_expire_time): @@ -321,13 +338,13 @@ class MutableShareFile(object): msg += " ." raise IndexError(msg) - def add_or_renew_lease(self, lease_info): + def add_or_renew_lease(self, available_space, lease_info): precondition(lease_info.owner_num != 0) # 0 means "no lease here" try: self.renew_lease(lease_info.renew_secret, lease_info.expiration_time) except IndexError: - self.add_lease(lease_info) + self.add_lease(available_space, lease_info) def cancel_lease(self, cancel_secret): """Remove any leases with the given cancel_secret. If the last lease diff --git a/src/allmydata/test/common_storage.py b/src/allmydata/test/common_storage.py index f020a8146..529ebe586 100644 --- a/src/allmydata/test/common_storage.py +++ b/src/allmydata/test/common_storage.py @@ -31,3 +31,35 @@ def upload_immutable(storage_server, storage_index, renew_secret, cancel_secret, for shnum, writer in writers.items(): writer.remote_write(0, shares[shnum]) writer.remote_close() + + +def upload_mutable(storage_server, storage_index, secrets, shares): + """ + Synchronously upload some mutable shares to a ``StorageServer``. + + :param allmydata.storage.server.StorageServer storage_server: The storage + server object to use to perform the upload. + + :param bytes storage_index: The storage index for the immutable shares. + + :param secrets: A three-tuple of a write enabler, renew secret, and cancel + secret. + + :param dict[int, bytes] shares: A mapping from share numbers to share data + to upload. + + :return: ``None`` + """ + test_and_write_vectors = { + sharenum: ([], [(0, data)], None) + for sharenum, data + in shares.items() + } + read_vector = [] + + storage_server.remote_slot_testv_and_readv_and_writev( + storage_index, + secrets, + test_and_write_vectors, + read_vector, + ) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9ce6482ea..e03c07203 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -73,6 +73,7 @@ from .common import ( from .common_util import FakeCanary from .common_storage import ( upload_immutable, + upload_mutable, ) from .strategies import ( offsets, @@ -698,9 +699,9 @@ class Server(unittest.TestCase): def test_reserved_space_immutable_lease(self): """ - If there is not enough available space to store an additional lease then - ``remote_add_lease`` fails with ``NoSpace`` when an attempt is made to - use it to create a new lease. + If there is not enough available space to store an additional lease on an + immutable share then ``remote_add_lease`` fails with ``NoSpace`` when + an attempt is made to use it to create a new lease. """ disk = FakeDisk(total=1024, used=0) self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) @@ -722,6 +723,46 @@ class Server(unittest.TestCase): with self.assertRaises(interfaces.NoSpace): ss.remote_add_lease(storage_index, renew_secret, cancel_secret) + def test_reserved_space_mutable_lease(self): + """ + If there is not enough available space to store an additional lease on a + mutable share then ``remote_add_lease`` fails with ``NoSpace`` when an + attempt is made to use it to create a new lease. + """ + disk = FakeDisk(total=1024, used=0) + self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) + + ss = self.create("test_reserved_space_mutable_lease") + + renew_secrets = iter( + "{}{}".format("r" * 31, i).encode("ascii") + for i + in range(5) + ) + + storage_index = b"x" * 16 + write_enabler = b"w" * 32 + cancel_secret = b"c" * 32 + secrets = (write_enabler, next(renew_secrets), cancel_secret) + shares = {0: b"y" * 500} + upload_mutable(ss, storage_index, secrets, shares) + + # use up all the available space + disk.use(disk.available) + + # The upload created one lease. There is room for three more leases + # in the share header. Even if we're out of disk space, on a boring + # enough filesystem we can write these. + for i in range(3): + ss.remote_add_lease(storage_index, next(renew_secrets), cancel_secret) + + # Having used all of the space for leases in the header, we would have + # to allocate storage for the next lease. Since there is no space + # available, this must fail instead. + with self.assertRaises(interfaces.NoSpace): + ss.remote_add_lease(storage_index, next(renew_secrets), cancel_secret) + + def test_reserved_space(self): reserved = 10000 allocated = 0 From 6449ad03de20db407dc96ba2a6651b9d80ff797a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 13:38:37 -0400 Subject: [PATCH 080/916] Do not record corruption advisories if there is no available space --- src/allmydata/storage/server.py | 86 ++++++++++++++++++++++++------ src/allmydata/test/test_storage.py | 21 ++++++++ 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 66d9df998..3ee494786 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -737,24 +737,80 @@ class StorageServer(service.MultiService, Referenceable): # protocol backwards compatibility reasons. assert isinstance(share_type, bytes) assert isinstance(reason, bytes), "%r is not bytes" % (reason,) - fileutil.make_dirs(self.corruption_advisory_dir) - now = time_format.iso_utc(sep="T") + si_s = si_b2a(storage_index) - # windows can't handle colons in the filename - fn = os.path.join( - self.corruption_advisory_dir, - ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") - ) - with open(fn, "w") as f: - f.write("report: Share Corruption\n") - f.write("type: %s\n" % bytes_to_native_str(share_type)) - f.write("storage_index: %s\n" % bytes_to_native_str(si_s)) - f.write("share_number: %d\n" % shnum) - f.write("\n") - f.write(bytes_to_native_str(reason)) - f.write("\n") + log.msg(format=("client claims corruption in (%(share_type)s) " + "%(si)s-%(shnum)d: %(reason)s"), share_type=share_type, si=si_s, shnum=shnum, reason=reason, level=log.SCARY, umid="SGx2fA") + + fileutil.make_dirs(self.corruption_advisory_dir) + now = time_format.iso_utc(sep="T") + + report = render_corruption_report(share_type, si_s, shnum, reason) + if len(report) > self.get_available_space(): + return None + + report_path = get_corruption_report_path( + self.corruption_advisory_dir, + now, + si_s, + shnum, + ) + with open(report_path, "w") as f: + f.write(report) + return None + +CORRUPTION_REPORT_FORMAT = """\ +report: Share Corruption +type: {type} +storage_index: {storage_index} +share_number: {share_number} + +{reason} + +""" + +def render_corruption_report(share_type, si_s, shnum, reason): + """ + Create a string that explains a corruption report using freeform text. + + :param bytes share_type: The type of the share which the report is about. + + :param bytes si_s: The encoded representation of the storage index which + the report is about. + + :param int shnum: The share number which the report is about. + + :param bytes reason: The reason given by the client for the corruption + report. + """ + return CORRUPTION_REPORT_FORMAT.format( + type=bytes_to_native_str(share_type), + storage_index=bytes_to_native_str(si_s), + share_number=shnum, + reason=bytes_to_native_str(reason), + ) + +def get_corruption_report_path(base_dir, now, si_s, shnum): + """ + Determine the path to which a certain corruption report should be written. + + :param str base_dir: The directory beneath which to construct the path. + + :param str now: The time of the report. + + :param str si_s: The encoded representation of the storage index which the + report is about. + + :param int shnum: The share number which the report is about. + + :return str: A path to which the report can be written. + """ + # windows can't handle colons in the filename + return os.path.join( + base_dir, + ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + ) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e03c07203..314069ce2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1006,6 +1006,27 @@ class Server(unittest.TestCase): self.failUnlessEqual(set(b.keys()), set([0,1,2])) self.failUnlessEqual(b[0].remote_read(0, 25), b"\x00" * 25) + def test_reserved_space_advise_corruption(self): + """ + If there is no available space then ``remote_advise_corrupt_share`` does + not write a corruption report. + """ + disk = FakeDisk(total=1024, used=1024) + self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) + + workdir = self.workdir("test_reserved_space_advise_corruption") + ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) + ss.setServiceParent(self.sparent) + + si0_s = base32.b2a(b"si0") + ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, + b"This share smells funny.\n") + + self.assertEqual( + [], + os.listdir(ss.corruption_advisory_dir), + ) + def test_advise_corruption(self): workdir = self.workdir("test_advise_corruption") ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) From 5837841c090d110e1ec772f0aed137642a7d6aaa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 14:15:47 -0400 Subject: [PATCH 081/916] mention corruption advisories in the news fragment too --- newsfragments/LFS-01-005.security | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newsfragments/LFS-01-005.security b/newsfragments/LFS-01-005.security index 135b2487c..ba2bbd741 100644 --- a/newsfragments/LFS-01-005.security +++ b/newsfragments/LFS-01-005.security @@ -1,3 +1,4 @@ -The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information. +The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information and recording corruption advisories. Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. Now this operation will fail with an exception and the lease will not be created. +Similarly, if there is no space available, corruption advisories will be logged but not written to disk. From 8d15d61ff2600b3a4f560e3b55f14a13bc3138e5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 15:58:48 -0400 Subject: [PATCH 082/916] put the news fragment in the right place --- newsfragments/{LFS-01-005.security => 3823.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-005.security => 3823.security} (100%) diff --git a/newsfragments/LFS-01-005.security b/newsfragments/3823.security similarity index 100% rename from newsfragments/LFS-01-005.security rename to newsfragments/3823.security From 194499aafe42399185cbc4185fa078f09adfb608 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 16:09:54 -0400 Subject: [PATCH 083/916] remove unused import --- src/allmydata/storage/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 3ee494786..30fa5adc2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -15,7 +15,6 @@ else: from typing import Dict import os, re, struct, time -import six from foolscap.api import Referenceable from foolscap.ipb import IRemoteReference From cb675df48d08f4f0a42061d9261a3f5d47ac1673 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 16:10:24 -0400 Subject: [PATCH 084/916] remove unused encoding of storage index --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 314069ce2..70cad7db2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1018,7 +1018,6 @@ class Server(unittest.TestCase): ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) ss.setServiceParent(self.sparent) - si0_s = base32.b2a(b"si0") ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") From ea202ba61b90545ab78127f3340a8bb4bac18612 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 14:51:37 -0400 Subject: [PATCH 085/916] news fragment --- newsfragments/3824.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3824.security diff --git a/newsfragments/3824.security b/newsfragments/3824.security new file mode 100644 index 000000000..b29b2acc8 --- /dev/null +++ b/newsfragments/3824.security @@ -0,0 +1 @@ +The storage server implementation no longer records corruption advisories about storage indexes for which it holds no shares. From 470657b337ca199418aee6777866281769d8f38c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 14:56:09 -0400 Subject: [PATCH 086/916] Drop corruption advisories if we don't have a matching share --- src/allmydata/storage/server.py | 32 ++++++++++++++++++++++++++---- src/allmydata/test/test_storage.py | 23 +++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 30fa5adc2..c81d88bfc 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -77,9 +77,9 @@ class StorageServer(service.MultiService, Referenceable): sharedir = os.path.join(storedir, "shares") fileutil.make_dirs(sharedir) self.sharedir = sharedir - # we don't actually create the corruption-advisory dir until necessary self.corruption_advisory_dir = os.path.join(storedir, "corruption-advisories") + fileutil.make_dirs(self.corruption_advisory_dir) self.reserved_space = int(reserved_space) self.no_storage = discard_storage self.readonly_storage = readonly_storage @@ -730,6 +730,21 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("readv", self._get_current_time() - start) return datavs + def _share_exists(self, storage_index, shnum): + """ + Check local share storage to see if a matching share exists. + + :param bytes storage_index: The storage index to inspect. + :param int shnum: The share number to check for. + + :return bool: ``True`` if a share with the given number exists at the + given storage index, ``False`` otherwise. + """ + for existing_sharenum, ignored in self._get_bucket_shares(storage_index): + if existing_sharenum == shnum: + return True + return False + def remote_advise_corrupt_share(self, share_type, storage_index, shnum, reason): # This is a remote API, I believe, so this has to be bytes for legacy @@ -739,18 +754,27 @@ class StorageServer(service.MultiService, Referenceable): si_s = si_b2a(storage_index) + if not self._share_exists(storage_index, shnum): + log.msg( + format=( + "discarding client corruption claim for %(si)s/%(shnum)d " + "which I do not have" + ), + si=si_s, + shnum=shnum, + ) + return + log.msg(format=("client claims corruption in (%(share_type)s) " + "%(si)s-%(shnum)d: %(reason)s"), share_type=share_type, si=si_s, shnum=shnum, reason=reason, level=log.SCARY, umid="SGx2fA") - fileutil.make_dirs(self.corruption_advisory_dir) - now = time_format.iso_utc(sep="T") - report = render_corruption_report(share_type, si_s, shnum, reason) if len(report) > self.get_available_space(): return None + now = time_format.iso_utc(sep="T") report_path = get_corruption_report_path( self.corruption_advisory_dir, now, diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 70cad7db2..9889a001a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1018,6 +1018,7 @@ class Server(unittest.TestCase): ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) ss.setServiceParent(self.sparent) + upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") @@ -1032,6 +1033,7 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) si0_s = base32.b2a(b"si0") + upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") @@ -1070,6 +1072,27 @@ class Server(unittest.TestCase): self.failUnlessIn(b"share_number: 1", report) self.failUnlessIn(b"This share tastes like dust.", report) + def test_advise_corruption_missing(self): + """ + If a corruption advisory is received for a share that is not present on + this server then it is not persisted. + """ + workdir = self.workdir("test_advise_corruption_missing") + ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) + ss.setServiceParent(self.sparent) + + # Upload one share for this storage index + upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + + # And try to submit a corruption advisory about a different share + si0_s = base32.b2a(b"si0") + ss.remote_advise_corrupt_share(b"immutable", b"si0", 1, + b"This share smells funny.\n") + + self.assertEqual( + [], + os.listdir(ss.corruption_advisory_dir), + ) class MutableServer(unittest.TestCase): From 0ada9d93f794efed68d185b8f68bcb31f6394ee0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 23 Oct 2021 07:43:22 -0400 Subject: [PATCH 087/916] remove unused local --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9889a001a..06b8d7957 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1085,7 +1085,6 @@ class Server(unittest.TestCase): upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) # And try to submit a corruption advisory about a different share - si0_s = base32.b2a(b"si0") ss.remote_advise_corrupt_share(b"immutable", b"si0", 1, b"This share smells funny.\n") From b51f0ac8ff60a39b34c51d5f8b4c4b7aad232c37 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 23 Oct 2021 08:04:19 -0400 Subject: [PATCH 088/916] storage_index is a byte string and Python 3 cares --- src/allmydata/test/test_storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 06b8d7957..738e218eb 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1018,7 +1018,7 @@ class Server(unittest.TestCase): ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) ss.setServiceParent(self.sparent) - upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + upload_immutable(ss, b"si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") @@ -1033,7 +1033,7 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) si0_s = base32.b2a(b"si0") - upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + upload_immutable(ss, b"si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") @@ -1082,7 +1082,7 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) # Upload one share for this storage index - upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + upload_immutable(ss, b"si0", b"r" * 32, b"c" * 32, {0: b""}) # And try to submit a corruption advisory about a different share ss.remote_advise_corrupt_share(b"immutable", b"si0", 1, From 0b4e6754a34ee9ba8d7d71f6f16e7e29f4fd8ec8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Oct 2021 20:47:35 -0400 Subject: [PATCH 089/916] news fragment --- newsfragments/3827.security | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 newsfragments/3827.security diff --git a/newsfragments/3827.security b/newsfragments/3827.security new file mode 100644 index 000000000..4fee19c76 --- /dev/null +++ b/newsfragments/3827.security @@ -0,0 +1,4 @@ +The SFTP server no longer accepts password-based credentials for authentication. +Public/private key-based credentials are now the only supported authentication type. +This removes plaintext password storage from the SFTP credentials file. +It also removes a possible timing side-channel vulnerability which might have allowed attackers to discover an account's plaintext password. From 5878a64890ba0a395f61432a9b5dd534daa9a64a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Oct 2021 20:50:19 -0400 Subject: [PATCH 090/916] Remove password-based authentication from the SFTP frontend --- docs/frontends/FTP-and-SFTP.rst | 14 +- src/allmydata/frontends/auth.py | 122 ++++++++++------ src/allmydata/test/test_auth.py | 244 +++++++++++++++++++++++--------- 3 files changed, 255 insertions(+), 125 deletions(-) diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index 9d4f1dcec..ede719e26 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -47,8 +47,8 @@ servers must be configured with a way to first authenticate a user (confirm that a prospective client has a legitimate claim to whatever authorities we might grant a particular user), and second to decide what directory cap should be used as the root directory for a log-in by the authenticated user. -A username and password can be used; as of Tahoe-LAFS v1.11, RSA or DSA -public key authentication is also supported. +As of Tahoe-LAFS v1.17, +RSA/DSA public key authentication is the only supported mechanism. Tahoe-LAFS provides two mechanisms to perform this user-to-cap mapping. The first (recommended) is a simple flat file with one account per line. @@ -59,20 +59,14 @@ Creating an Account File To use the first form, create a file (for example ``BASEDIR/private/accounts``) in which each non-comment/non-blank line is a space-separated line of -(USERNAME, PASSWORD, ROOTCAP), like so:: +(USERNAME, KEY-TYPE, PUBLIC-KEY, ROOTCAP), like so:: % cat BASEDIR/private/accounts - # This is a password line: username password cap - alice password URI:DIR2:ioej8xmzrwilg772gzj4fhdg7a:wtiizszzz2rgmczv4wl6bqvbv33ag4kvbr6prz3u6w3geixa6m6a - bob sekrit URI:DIR2:6bdmeitystckbl9yqlw7g56f4e:serp5ioqxnh34mlbmzwvkp3odehsyrr7eytt5f64we3k9hhcrcja - # This is a public key line: username keytype pubkey cap # (Tahoe-LAFS v1.11 or later) carol ssh-rsa AAAA... URI:DIR2:ovjy4yhylqlfoqg2vcze36dhde:4d4f47qko2xm5g7osgo2yyidi5m4muyo2vjjy53q4vjju2u55mfa -For public key authentication, the keytype may be either "ssh-rsa" or "ssh-dsa". -To avoid ambiguity between passwords and public key types, a password cannot -start with "ssh-". +The key type may be either "ssh-rsa" or "ssh-dsa". Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in the next sections. diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index b61062334..312a9da1a 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -32,65 +32,93 @@ class FTPAvatarID(object): @implementer(checkers.ICredentialsChecker) class AccountFileChecker(object): - credentialInterfaces = (credentials.IUsernamePassword, - credentials.IUsernameHashedPassword, - credentials.ISSHPrivateKey) + credentialInterfaces = (credentials.ISSHPrivateKey,) + def __init__(self, client, accountfile): self.client = client - self.passwords = BytesKeyDict() - pubkeys = BytesKeyDict() - self.rootcaps = BytesKeyDict() - with open(abspath_expanduser_unicode(accountfile), "rb") as f: - for line in f: - line = line.strip() - if line.startswith(b"#") or not line: - continue - name, passwd, rest = line.split(None, 2) - if passwd.startswith(b"ssh-"): - bits = rest.split() - keystring = b" ".join([passwd] + bits[:-1]) - key = keys.Key.fromString(keystring) - rootcap = bits[-1] - pubkeys[name] = [key] - else: - self.passwords[name] = passwd - rootcap = rest - self.rootcaps[name] = rootcap + path = abspath_expanduser_unicode(accountfile) + with open_account_file(path) as f: + self.rootcaps, pubkeys = load_account_file(f) self._pubkeychecker = SSHPublicKeyChecker(InMemorySSHKeyDB(pubkeys)) def _avatarId(self, username): return FTPAvatarID(username, self.rootcaps[username]) - def _cbPasswordMatch(self, matched, username): - if matched: - return self._avatarId(username) - raise error.UnauthorizedLogin - def requestAvatarId(self, creds): if credentials.ISSHPrivateKey.providedBy(creds): d = defer.maybeDeferred(self._pubkeychecker.requestAvatarId, creds) d.addCallback(self._avatarId) return d - elif credentials.IUsernameHashedPassword.providedBy(creds): - return self._checkPassword(creds) - elif credentials.IUsernamePassword.providedBy(creds): - return self._checkPassword(creds) - else: - raise NotImplementedError() + raise NotImplementedError() - def _checkPassword(self, creds): - """ - Determine whether the password in the given credentials matches the - password in the account file. +def open_account_file(path): + """ + Open and return the accounts file at the given path. + """ + return open(path, "rt", encoding="utf-8") - Returns a Deferred that fires with the username if the password matches - or with an UnauthorizedLogin failure otherwise. - """ - try: - correct = self.passwords[creds.username] - except KeyError: - return defer.fail(error.UnauthorizedLogin()) +def load_account_file(lines): + """ + Load credentials from an account file. - d = defer.maybeDeferred(creds.checkPassword, correct) - d.addCallback(self._cbPasswordMatch, creds.username) - return d + :param lines: An iterable of account lines to load. + + :return: See ``create_account_maps``. + """ + return create_account_maps( + parse_accounts( + content_lines( + lines, + ), + ), + ) + +def content_lines(lines): + """ + Drop empty and commented-out lines (``#``-prefixed) from an iterator of + lines. + + :param lines: An iterator of lines to process. + + :return: An iterator of lines including only those from ``lines`` that + include content intended to be loaded. + """ + for line in lines: + line = line.strip() + if line and not line.startswith("#"): + yield line + +def parse_accounts(lines): + """ + Parse account lines into their components (name, key, rootcap). + """ + for line in lines: + name, passwd, rest = line.split(None, 2) + if not passwd.startswith("ssh-"): + raise ValueError( + "Password-based authentication is not supported; " + "configure key-based authentication instead." + ) + + bits = rest.split() + keystring = " ".join([passwd] + bits[:-1]) + key = keys.Key.fromString(keystring) + rootcap = bits[-1] + yield (name, key, rootcap) + +def create_account_maps(accounts): + """ + Build mappings from account names to keys and rootcaps. + + :param accounts: An iterator if (name, key, rootcap) tuples. + + :return: A tuple of two dicts. The first maps account names to rootcaps. + The second maps account names to public keys. + """ + rootcaps = BytesKeyDict() + pubkeys = BytesKeyDict() + for (name, key, rootcap) in accounts: + name_bytes = name.encode("utf-8") + rootcaps[name_bytes] = rootcap.encode("utf-8") + pubkeys[name_bytes] = [key] + return rootcaps, pubkeys diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index d5198d326..19c2f7c01 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -8,7 +8,17 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from future.builtins import str # noqa: F401 + from future.builtins import str, open # noqa: F401 + +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + text, + characters, + tuples, + lists, +) from twisted.trial import unittest from twisted.python import filepath @@ -38,25 +48,184 @@ dBSD8940XU3YW+oeq8e+p3yQ2GinHfeJ3BYQyNQLuMAJ -----END RSA PRIVATE KEY----- """) -DUMMY_ACCOUNTS = u"""\ -alice herpassword URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 -bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222 +DUMMY_KEY_DSA = keys.Key.fromString("""\ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABsQAAAAdzc2gtZH +NzAAAAgQDKMh/ELaiP21LYRBuPbUy7dUhv/XZwV7aS1LzxSP+KaJvtDOei8X76XEAfkqX+ +aGh9eup+BLkezrV6LlpO9uPzhY8ChlKpkvw5PZKv/2agSrVxZyG7yEzHNtSBQXE6qNMwIk +N/ycXLGCqyAhQSzRhLz9ETNaslRDLo7YyVWkiuAQAAABUA5nTatFKux5EqZS4EarMWFRBU +i1UAAACAFpkkK+JsPixSTPyn0DNMoGKA0Klqy8h61Ds6pws+4+aJQptUBshpwNw1ypo7MO ++goDZy3wwdWtURTPGMgesNdEfxp8L2/kqE4vpMK0myoczCqOiWMeNB/x1AStbSkBI8WmHW +2htgsC01xbaix/FrA3edK8WEyv+oIxlbV1FkrPkAAACANb0EpCc8uoR4/32rO2JLsbcLBw +H5wc2khe7AKkIa9kUknRIRvoCZUtXF5XuXXdRmnpVEm2KcsLdtZjip43asQcqgt0Kz3nuF +kAf7bI98G1waFUimcCSPsal4kCmW2HC11sg/BWOt5qczX/0/3xVxpo6juUeBq9ncnFTvPX +5fOlEAAAHoJkFqHiZBah4AAAAHc3NoLWRzcwAAAIEAyjIfxC2oj9tS2EQbj21Mu3VIb/12 +cFe2ktS88Uj/imib7QznovF++lxAH5Kl/mhofXrqfgS5Hs61ei5aTvbj84WPAoZSqZL8OT +2Sr/9moEq1cWchu8hMxzbUgUFxOqjTMCJDf8nFyxgqsgIUEs0YS8/REzWrJUQy6O2MlVpI +rgEAAAAVAOZ02rRSrseRKmUuBGqzFhUQVItVAAAAgBaZJCvibD4sUkz8p9AzTKBigNCpas +vIetQ7OqcLPuPmiUKbVAbIacDcNcqaOzDvoKA2ct8MHVrVEUzxjIHrDXRH8afC9v5KhOL6 +TCtJsqHMwqjoljHjQf8dQErW0pASPFph1tobYLAtNcW2osfxawN3nSvFhMr/qCMZW1dRZK +z5AAAAgDW9BKQnPLqEeP99qztiS7G3CwcB+cHNpIXuwCpCGvZFJJ0SEb6AmVLVxeV7l13U +Zp6VRJtinLC3bWY4qeN2rEHKoLdCs957hZAH+2yPfBtcGhVIpnAkj7GpeJAplthwtdbIPw +VjreanM1/9P98VcaaOo7lHgavZ3JxU7z1+XzpRAAAAFQC7360pZLbv7PFt4BPFJ8zAHxAe +QwAAAA5leGFya3VuQGJhcnlvbgECAwQ= +-----END OPENSSH PRIVATE KEY----- +""") -# dennis password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 +ACCOUNTS = u"""\ +# dennis {key} URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333 """.format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii") +# Python str.splitlines considers NEXT LINE, LINE SEPARATOR, and PARAGRAPH +# separator to be line separators, too. However, file.readlines() does not... +LINE_SEPARATORS = ( + '\x0a', # line feed + '\x0b', # vertical tab + '\x0c', # form feed + '\x0d', # carriage return +) + +class AccountFileParserTests(unittest.TestCase): + """ + Tests for ``load_account_file`` and its helper functions. + """ + @given(lists( + text(alphabet=characters( + blacklist_categories=( + # Surrogates are an encoding trick to help out UTF-16. + # They're not necessary to represent any non-surrogate code + # point in unicode. They're also not legal individually but + # only in pairs. + 'Cs', + ), + # Exclude all our line separators too. + blacklist_characters=("\n", "\r"), + )), + )) + def test_ignore_comments(self, lines): + """ + ``auth.content_lines`` filters out lines beginning with `#` and empty + lines. + """ + expected = set() + + # It's not clear that real files and StringIO behave sufficiently + # similarly to use the latter instead of the former here. In + # particular, they seem to have distinct and incompatible + # line-splitting rules. + bufpath = self.mktemp() + with open(bufpath, "wt", encoding="utf-8") as buf: + for line in lines: + stripped = line.strip() + is_content = stripped and not stripped.startswith("#") + if is_content: + expected.add(stripped) + buf.write(line + "\n") + + with auth.open_account_file(bufpath) as buf: + actual = set(auth.content_lines(buf)) + + self.assertEqual(expected, actual) + + def test_parse_accounts(self): + """ + ``auth.parse_accounts`` accepts an iterator of account lines and returns + an iterator of structured account data. + """ + alice_key = DUMMY_KEY.public().toString("openssh").decode("utf-8") + alice_cap = "URI:DIR2:aaaa:1111" + + bob_key = DUMMY_KEY_DSA.public().toString("openssh").decode("utf-8") + bob_cap = "URI:DIR2:aaaa:2222" + self.assertEqual( + list(auth.parse_accounts([ + "alice {} {}".format(alice_key, alice_cap), + "bob {} {}".format(bob_key, bob_cap), + ])), + [ + ("alice", DUMMY_KEY.public(), alice_cap), + ("bob", DUMMY_KEY_DSA.public(), bob_cap), + ], + ) + + def test_parse_accounts_rejects_passwords(self): + """ + The iterator returned by ``auth.parse_accounts`` raises ``ValueError`` + when processing reaches a line that has what looks like a password + instead of an ssh key. + """ + with self.assertRaises(ValueError): + list(auth.parse_accounts(["alice apassword URI:DIR2:aaaa:1111"])) + + def test_create_account_maps(self): + """ + ``auth.create_account_maps`` accepts an iterator of structured account + data and returns two mappings: one from account name to rootcap, the + other from account name to public keys. + """ + alice_cap = "URI:DIR2:aaaa:1111" + alice_key = DUMMY_KEY.public() + bob_cap = "URI:DIR2:aaaa:2222" + bob_key = DUMMY_KEY_DSA.public() + accounts = [ + ("alice", alice_key, alice_cap), + ("bob", bob_key, bob_cap), + ] + self.assertEqual( + auth.create_account_maps(accounts), + ({ + b"alice": alice_cap.encode("utf-8"), + b"bob": bob_cap.encode("utf-8"), + }, + { + b"alice": [alice_key], + b"bob": [bob_key], + }), + ) + + def test_load_account_file(self): + """ + ``auth.load_account_file`` accepts an iterator of serialized account lines + and returns two mappings: one from account name to rootcap, the other + from account name to public keys. + """ + alice_key = DUMMY_KEY.public().toString("openssh").decode("utf-8") + alice_cap = "URI:DIR2:aaaa:1111" + + bob_key = DUMMY_KEY_DSA.public().toString("openssh").decode("utf-8") + bob_cap = "URI:DIR2:aaaa:2222" + + accounts = [ + "alice {} {}".format(alice_key, alice_cap), + "bob {} {}".format(bob_key, bob_cap), + "# carol {} {}".format(alice_key, alice_cap), + ] + + self.assertEqual( + auth.load_account_file(accounts), + ({ + b"alice": alice_cap.encode("utf-8"), + b"bob": bob_cap.encode("utf-8"), + }, + { + b"alice": [DUMMY_KEY.public()], + b"bob": [DUMMY_KEY_DSA.public()], + }), + ) + + class AccountFileCheckerKeyTests(unittest.TestCase): """ Tests for key handling done by allmydata.frontends.auth.AccountFileChecker. """ def setUp(self): self.account_file = filepath.FilePath(self.mktemp()) - self.account_file.setContent(DUMMY_ACCOUNTS) + self.account_file.setContent(ACCOUNTS) abspath = abspath_expanduser_unicode(str(self.account_file.path)) self.checker = auth.AccountFileChecker(None, abspath) - def test_unknown_user_ssh(self): + def test_unknown_user(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with UnauthorizedLogin if called with an SSHPrivateKey object with a @@ -67,67 +236,6 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) - def test_unknown_user_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - UnauthorizedLogin if called with an SSHPrivateKey object with a - username not present in the account file. - - We use a commented out user, so we're also checking that comments are - skipped. - """ - key_credentials = credentials.UsernamePassword(b"dennis", b"password") - d = self.checker.requestAvatarId(key_credentials) - return self.assertFailure(d, error.UnauthorizedLogin) - - def test_password_auth_user_with_ssh_key(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - UnauthorizedLogin if called with an SSHPrivateKey object for a username - only associated with a password in the account file. - """ - key_credentials = credentials.SSHPrivateKey( - b"alice", b"md5", None, None, None) - avatarId = self.checker.requestAvatarId(key_credentials) - return self.assertFailure(avatarId, error.UnauthorizedLogin) - - def test_password_auth_user_with_correct_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - the user if the correct password is given. - """ - key_credentials = credentials.UsernamePassword(b"alice", b"herpassword") - d = self.checker.requestAvatarId(key_credentials) - def authenticated(avatarId): - self.assertEqual( - (b"alice", - b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), - (avatarId.username, avatarId.rootcap)) - return d - - def test_password_auth_user_with_correct_hashed_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - the user if the correct password is given in hashed form. - """ - key_credentials = credentials.UsernameHashedPassword(b"alice", b"herpassword") - d = self.checker.requestAvatarId(key_credentials) - def authenticated(avatarId): - self.assertEqual( - (b"alice", - b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), - (avatarId.username, avatarId.rootcap)) - return d - - def test_password_auth_user_with_wrong_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - UnauthorizedLogin if the wrong password is given. - """ - key_credentials = credentials.UsernamePassword(b"alice", b"WRONG") - avatarId = self.checker.requestAvatarId(key_credentials) - return self.assertFailure(avatarId, error.UnauthorizedLogin) - def test_unrecognized_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with From 3de481ab6bbabde6943648c80722cdacabb1d3e1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Oct 2021 20:52:35 -0400 Subject: [PATCH 091/916] remove unused imports --- src/allmydata/frontends/auth.py | 2 +- src/allmydata/test/test_auth.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 312a9da1a..b6f9c2b7e 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -12,7 +12,7 @@ if PY2: from zope.interface import implementer from twisted.internet import defer -from twisted.cred import error, checkers, credentials +from twisted.cred import checkers, credentials from twisted.conch.ssh import keys from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index 19c2f7c01..bfe717f79 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -16,7 +16,6 @@ from hypothesis import ( from hypothesis.strategies import ( text, characters, - tuples, lists, ) From 9764ac740ada46b8ee23b3060e951e0cd5dab9a9 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 26 Oct 2021 11:22:32 +0100 Subject: [PATCH 092/916] test kwargs overlap with params in start_action Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 1fbb9ec8d..bab37243c 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -330,3 +330,27 @@ class LogCallDeferredTests(TestCase): msg = logger.messages[0] assertContainsFields(self, msg, {"args": (10, 2)}) assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) + + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_keyword_args_dont_overlap_with_start_action(self, logger): + """ + Check that both keyword and positional arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(base, exp, kwargs, args): + return base ** exp + self.assertThat( + f(10, 2, kwargs={"kwarg_1": "value_1", "kwarg_2": 2}, args=(1, 2, 3)), + succeeded(Equals(100)), + ) + msg = logger.messages[0] + assertContainsFields(self, msg, {"args": (10, 2)}) + assertContainsFields( + self, + msg, + {"kwargs": {"args": [1, 2, 3], "kwargs": {"kwarg_1": "value_1", "kwarg_2": 2}}}, + ) From 5b9997f388ccca089081d8f5939f0c84edea3542 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Oct 2021 07:16:24 -0400 Subject: [PATCH 093/916] update the integration tests to reflect removal of sftp password auth --- integration/conftest.py | 29 ++++++++++++++++---------- integration/test_sftp.py | 45 +++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 39ff3b42b..ef5c518a8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -353,10 +353,23 @@ def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, nodes.append(process) return nodes +@pytest.fixture(scope="session") +def alice_sftp_client_key_path(temp_dir): + # The client SSH key path is typically going to be somewhere else (~/.ssh, + # typically), but for convenience sake for testing we'll put it inside node. + return join(temp_dir, "alice", "private", "ssh_client_rsa_key") @pytest.fixture(scope='session') @log_call(action_type=u"integration:alice", include_args=[], include_result=False) -def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request): +def alice( + reactor, + temp_dir, + introducer_furl, + flog_gatherer, + storage_nodes, + alice_sftp_client_key_path, + request, +): process = pytest_twisted.blockon( _create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", @@ -387,19 +400,13 @@ accounts.file = {accounts_path} """.format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path)) generate_ssh_key(host_ssh_key_path) - # 3. Add a SFTP access file with username/password and SSH key auth. - - # The client SSH key path is typically going to be somewhere else (~/.ssh, - # typically), but for convenience sake for testing we'll put it inside node. - client_ssh_key_path = join(process.node_dir, "private", "ssh_client_rsa_key") - generate_ssh_key(client_ssh_key_path) + # 3. Add a SFTP access file with an SSH key for auth. + generate_ssh_key(alice_sftp_client_key_path) # Pub key format is "ssh-rsa ". We want the key. - ssh_public_key = open(client_ssh_key_path + ".pub").read().strip().split()[1] + ssh_public_key = open(alice_sftp_client_key_path + ".pub").read().strip().split()[1] with open(accounts_path, "w") as f: f.write("""\ -alice password {rwcap} - -alice2 ssh-rsa {ssh_public_key} {rwcap} +alice-key ssh-rsa {ssh_public_key} {rwcap} """.format(rwcap=rwcap, ssh_public_key=ssh_public_key)) # 4. Restart the node with new SFTP config. diff --git a/integration/test_sftp.py b/integration/test_sftp.py index 6171c7413..3fdbb56d7 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -19,6 +19,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import os.path from posixpath import join from stat import S_ISDIR @@ -33,7 +34,7 @@ import pytest from .util import generate_ssh_key, run_in_thread -def connect_sftp(connect_args={"username": "alice", "password": "password"}): +def connect_sftp(connect_args): """Create an SFTP client.""" client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy) @@ -60,24 +61,24 @@ def connect_sftp(connect_args={"username": "alice", "password": "password"}): @run_in_thread def test_bad_account_password_ssh_key(alice, tmpdir): """ - Can't login with unknown username, wrong password, or wrong SSH pub key. + Can't login with unknown username, any password, or wrong SSH pub key. """ - # Wrong password, wrong username: - for u, p in [("alice", "wrong"), ("someuser", "password")]: + # Any password, wrong username: + for u, p in [("alice-key", "wrong"), ("someuser", "password")]: with pytest.raises(AuthenticationException): connect_sftp(connect_args={ "username": u, "password": p, }) - another_key = join(str(tmpdir), "ssh_key") + another_key = os.path.join(str(tmpdir), "ssh_key") generate_ssh_key(another_key) - good_key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key")) + good_key = RSAKey(filename=os.path.join(alice.node_dir, "private", "ssh_client_rsa_key")) bad_key = RSAKey(filename=another_key) # Wrong key: with pytest.raises(AuthenticationException): connect_sftp(connect_args={ - "username": "alice2", "pkey": bad_key, + "username": "alice-key", "pkey": bad_key, }) # Wrong username: @@ -86,13 +87,24 @@ def test_bad_account_password_ssh_key(alice, tmpdir): "username": "someoneelse", "pkey": good_key, }) +def sftp_client_key(node): + return RSAKey( + filename=os.path.join(node.node_dir, "private", "ssh_client_rsa_key"), + ) + +def test_sftp_client_key_exists(alice, alice_sftp_client_key_path): + """ + Weakly validate the sftp client key fixture by asserting that *something* + exists at the supposed key path. + """ + assert os.path.exists(alice_sftp_client_key_path) @run_in_thread def test_ssh_key_auth(alice): """It's possible to login authenticating with SSH public key.""" - key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key")) + key = sftp_client_key(alice) sftp = connect_sftp(connect_args={ - "username": "alice2", "pkey": key + "username": "alice-key", "pkey": key }) assert sftp.listdir() == [] @@ -100,7 +112,10 @@ def test_ssh_key_auth(alice): @run_in_thread def test_read_write_files(alice): """It's possible to upload and download files.""" - sftp = connect_sftp() + sftp = connect_sftp(connect_args={ + "username": "alice-key", + "pkey": sftp_client_key(alice), + }) with sftp.file("myfile", "wb") as f: f.write(b"abc") f.write(b"def") @@ -117,7 +132,10 @@ def test_directories(alice): It's possible to create, list directories, and create and remove files in them. """ - sftp = connect_sftp() + sftp = connect_sftp(connect_args={ + "username": "alice-key", + "pkey": sftp_client_key(alice), + }) assert sftp.listdir() == [] sftp.mkdir("childdir") @@ -148,7 +166,10 @@ def test_directories(alice): @run_in_thread def test_rename(alice): """Directories and files can be renamed.""" - sftp = connect_sftp() + sftp = connect_sftp(connect_args={ + "username": "alice-key", + "pkey": sftp_client_key(alice), + }) sftp.mkdir("dir") filepath = join("dir", "file") From 69d335c1e1503544850e4e3014ca1a6d1d89180b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 26 Oct 2021 13:14:26 +0100 Subject: [PATCH 094/916] update test overlap function docstring Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index bab37243c..61e0a6958 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -338,7 +338,7 @@ class LogCallDeferredTests(TestCase): ) def test_keyword_args_dont_overlap_with_start_action(self, logger): """ - Check that both keyword and positional arguments are logged when using ``log_call_deferred`` + Check that kwargs passed to decorated functions don't overlap with params in ``start_action`` """ @log_call_deferred(action_type=u"the-action") def f(base, exp, kwargs, args): From 28cc3cad66e0367da10ee97326d100c686f78d10 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Oct 2021 14:10:29 -0400 Subject: [PATCH 095/916] news fragment --- newsfragments/3829.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3829.minor diff --git a/newsfragments/3829.minor b/newsfragments/3829.minor new file mode 100644 index 000000000..e69de29bb From 7ec7cd45dd41b0f828a581865ab3b7bb15a655be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Oct 2021 14:10:41 -0400 Subject: [PATCH 096/916] Use "concurrency groups" to auto-cancel redundant builds --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45b2986a3..8209108bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,23 @@ on: - "master" pull_request: +# Control to what degree jobs in this workflow will run concurrently with +# other instances of themselves. +# +# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#concurrency +concurrency: + # We want every revision on master to run the workflow completely. + # "head_ref" is not set for the "push" event but it is set for the + # "pull_request" event. If it is set then it is the name of the branch and + # we can use it to make sure each branch has only one active workflow at a + # time. If it is not set then we can compute a unique string that gives + # every master/push workflow its own group. + group: "${{ github.head_ref || format('{0}-{1}', github.run_number, github.run_attempt) }}" + + # Then, we say that if a new workflow wants to start in the same group as a + # running workflow, the running workflow should be cancelled. + cancel-in-progress: true + env: # Tell Hypothesis which configuration we want it to use. TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" From eddfd244a761e006c70b80326edc987b16ef2c6a Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 26 Oct 2021 13:37:26 -0600 Subject: [PATCH 097/916] code and tests to check RSA key sizes --- src/allmydata/crypto/rsa.py | 12 ++++++ .../test/data/pycryptopp-rsa-1024-priv.txt | 1 + .../test/data/pycryptopp-rsa-32768-priv.txt | 1 + src/allmydata/test/test_crypto.py | 38 +++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt create mode 100644 src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index b5d15ad4a..d290388da 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -77,6 +77,18 @@ def create_signing_keypair_from_string(private_key_der): password=None, backend=default_backend(), ) + if not isinstance(priv_key, rsa.RSAPrivateKey): + raise ValueError( + "Private Key did not decode to an RSA key" + ) + if priv_key.key_size < 2048: + raise ValueError( + "Private Key is smaller than 2048 bits" + ) + if priv_key.key_size > (2048 * 8): + raise ValueError( + "Private Key is unreasonably large" + ) return priv_key, priv_key.public_key() diff --git a/src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt b/src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt new file mode 100644 index 000000000..6f5e67950 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt @@ -0,0 +1 @@ +MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAJLEAfZueLuT4vUQ1+c8ZM9dJ/LA29CYgA5toaMklQjbVQ2Skywvw1wEkRjhMpjQAx5+lpLTE2xCtqtfkHooMRNnquOxoh0o1Xya60jUHze7VB5QMV7BMKeUTff1hQqpIgw/GLvJRtar53cVY+SYf4SXx2/slDbVr8BI3DPwdeNtAgERAoGABzHD3GTJrteQJRxu+cQ3I0NPwx2IQ/Nlplq1GZDaIQ/FbJY+bhZrdXOswnl4cOcPNjNhu+c1qHGznv0ntayjCGgJ9dDySGqknDau+ezZcBO1JrIpPOABS7MVMst79mn47vB2+t8w5krrBYahAVp/L5kY8k+Pr9AU+L9mbevFW9MCQQDA+bAeMRNBfGc4gvoVV8ecovE1KRksFDlkaDVEOc76zNW6JZazHhQF/zIoMkV81rrg5UBntw3WR3R8A3l9osgDAkEAwrLQICJ3zjsJBt0xEkCBv9tK6IvSIc7MUQIc4J2Y1hiSjqsnTRACRy3UMsODfx/Lg7ITlDbABCLfv3v4D39jzwJBAKpFuYQNLxuqALlkgk8RN6hTiYlCYYE/BXa2TR4U4848RBy3wTSiEarwO1Ck0+afWZlCwFuDZo/kshMSH+dTZS8CQQC3PuIAIHDCGXHoV7W200zwzmSeoba2aEfTxcDTZyZvJi+VVcqi4eQGwbioP4rR/86aEQNeUaWpijv/g7xK0j/RAkBbt2U9bFFcja10KIpgw2bBxDU/c67h4+38lkrBUnM9XVBZxjbtQbnkkeAfOgQDiq3oBDBrHF3/Q8XM0CzZJBWS \ No newline at end of file diff --git a/src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt b/src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt new file mode 100644 index 000000000..d949f3f60 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt @@ -0,0 +1 @@ +MIJIQQIBADANBgkqhkiG9w0BAQEFAASCSCswgkgnAgEAAoIQAQC3x9r2dfYoTp7oIMsPdOhyNK5CB3TOtiaxhf3EkGAIaLWTXUVbxvOkiSu3Tca9VqFVnN7EkbT790uDjh4rviGeZF8oplVN+FDxKfcg5tXWv4ec9LnOUUAVRUnrUQA2azkOT+ozXQwZnJwUYr210VoV8D0MkrvOzNgGpb8aErDhW8SwrJcoYkObIE7n3C3zEMaEIyA1OFWSJDiXNGnBDvO54t1/y+/o4IuzLWWG7TPx8hnV+jcHRoxJTX2MZusJ7kugvxhgB0+avwXFTQr6ogvPNcUXak0+aLInLRtkYJ+0DYqo1hLAh8EBY/cLrhZM5LGGC4BAwGgUwsx3KKeOeduNnob3s/1rZpvZGwbGtfiWYQwDB8q68j3Ypf2Qvn7hPwicdOr0Dwe4TXJQ4yRHPeQaToOBUjtTJnrHsKDZET6i+jQ9e07Ct+yYrUwZjiaSXJYU/gCyPCui7L37NasXBJ00f1Ogm3gt4uxl3abO8mO1nKSWM+HbFBEyyO0apT+sSwYj6IL7cyCSJtWYMD4APdW5rXSArhyiaHV+xNbVUXAdBrZSNuwet925hTOf4IQD9uqfzeV3HIoiUCxn5GKYPZy01Kft+DExuDbJjMmES2GhfPWRIFB5MN0UdjlagDHLzFraQUcLTDKlxL0iZ+uV4Itv5dQyaf93Szu2LD1jnkvZOV5GN1RxTmZCH1FIPYCNwS6mIRG/4aPWA0HCZX8HzSMOBshAS6wECaoLWxv8D3K4Tm1rp/EgP7NZRxTj2ToOostJtjzTrVb3f3+zaT5svxD1Exw8tA1fZNRThIDKZXVSSLDYaiRDAUg7xEMD2eDCvNQasjAwX5Tnw7R4M/CZoZhgYVwIE+vHQTh8H+M/J8CNLxPT4N3fuXCqT8YoJVUOmKHe0kE5Rtd87X2BQY5SSx6LFMRRSVdBBpWB6cwLo8egehYAScEDQh0ht/ssaraWZ2LGt5hZL0I5V58iS/6C4IOu+1ry75g6mecWoHD0fBQELB3Q3Qi6c6Hik/jgTLQHb5UMqKj/MDSdTWuxwH2dYU5H4EGAkbfufBoxw9hIpdeS7/aDulvRKtFVPfi/pxmrd1lxQCBA4ionRe4IOY0E9i419TOgMtGgZxNlEXtp445MbeIlurxIDIX8N+RGHWljGR/9K6sjbgtGKyKLUxg51DZeuDKQGdyKXtIIkZ+Od9HN+3Mv0Ch5B9htIRV9hE6oLWLT+grqJCFAOD3olGgrRXByDsd8YouahYfjqb4KNCOyFPS3j5MdUpq+fiLrG3O98/L/xtmXxw+ekl95EGAnlwiCwULsjzVjHJDzSc68cldMnzNqLwhwWXpc0iswCWCQVFce/d1KlWqrtwq2ThH2pX3BJ5Pnu+KMISNNC/tagLe9vjmrh6ZhEks7hefn0srytJdivGDFqMs/ISmcld0U/0ZqE05b7BpErpfVrG9kb5QxWBTpaEb2O0pRsaYRcllFuNF6Nl/jPDBnn4BMYnOFnn9OKGPEDUeV/6CYP9x+Wi96M5Ni6vtv+zw9Xg8drslS5DJazXQFbJ0aqW3EgalUJVV0NgykB6Hr4pxTzrwo0+R/ro32DEj5OfjjU7TB4fYie0eax8tpdvzcWJRZ/c5b/Dg1yK+hbiMg9aTctHAsYJkOvMpxvull20IuV2sErWZ7KZhId19AFOnEQ6ILlHRwUf35AyEVmUL5BqLl137EeEVShEmage4+E/N6PdKzJdJGl1AQGyb7NTD86m0Jj2+8qu6zsBgyUfiJqZ17fixKV6l9HGJKSmY9If2XrX/IhNZ5dvqSmODJ1ZRGC5gjJcxcdHp2Q1179SlNmXiR/7DMcprL/+iVhRyxzM2GEJ78q9jS6j/Z+0vLzdNOPo1KxD191ogYjl5ck9gnHAkbaiANaK4rrfMytDkNm0JRua4p0mVyVHWZWwatoMhJxVl3+9x37OkF24ICTJZ4LSKDLJxi9WCQbhgACIA1mjcW0P+4AszpbuSXOQkPtT+MQ0IxHMzX261yHAIPbGsbSzoTy+PWJywFdMDy5afXDTNpmMfpzWkw2fhBQasNoGHl2CwFftJdr4WWxuN6mSwhNVHJTw1xe4A5fa6bjip5kmrLQK85YF4Ron0OIOofjcCzvjKCkNkGVKBhRiqBoqV6Pzz1XauVHFhFgZZNWXI+le+Fg9SJojeDtFQp5w6dZKBJMxV2uNPqV0U4VOtvAas2+Ul4zIJDB/FJyDX8POrsR+VkW7via64xM1hQlOZ5ispEOUvmO/NWkAsJM0n3S7qgud6NaFqOofQZcbh5r1z2uIrXwUIb85m2t/sPJBI1J/Dql4dmzgfn/q6Siqi8FeDoma/lQBZWyEeGz+/ckHdw/BGPx5FZlc8xLegNrQj4sVkUZXVAjNoUguA5HT9GcAmE5FeOHdHtD0bdTaNFkQbKdi3yUlGA1GZeyPThwfBaizgX3i6oOtGguX3HQMQtExip5xR2vsiYJsbWXuzlKEws8GwXoiJo8xEh+TPavxxtZ7dDdnJY1mUhKTVGLBCqCrJ+uhWdWuHKvC9x++V5NO6WQrUiG/o8oOwkpWyH7GC/VtulpxkoJlxAej3JxlHn91cN4PstDo4goOhQBi9k2A5rsmvjGG75BOKlqvhaQ6BPOa+9F5D5H0RhT0hw43TZmJri+0Ba2WT3FigcHHYGtx4UJfyqfg7d+WXvpIynC7i3SIN3N7atg3EsWwPuzDKE6ycjWTD6ToKmYLMnDgl4PzOEBFstG12OdcuQwhk2Dy5uEdxqGfViy3fV+Muev0yAkE/pRwutgQjQdw0OPXyGoqchYx33/cHq1fDWmkXZab8wuVThcx3He30UI4rr3MMff0gxdnJt3e6YcHHF0R8fGwkVC03zWXI2hfqHq+rNQkBnIbbRnepKvJylmcHn8KVJ13Nm2iHRTw7B8r6fE6LsmUJndh/M2Poa1AtxfGBniMIfqtV0RuT7UR1nDI0C8Lnx7E2KTw1MXCLh4xzGr5wZ+4T5FTeUnzd6yc7EEduLxktqh7RpmnBBPRNIufI9ztPTmRPXgF7r9PxI8MI09Sr2HQq2ZmEs6G0w8l8WMiABvlG/YQd+UHGn29acrzSYp6AfggjuUV7PrCC4flKk5IGBNdUtUqFxBRUuvn0ln7HayAAYLJuVMNv9daBwqMpp3Faor/0K+jC0FhIan3R6wBpKSuJo/6jZJoSlSCLGCkFqM9ks3sgD5cDvxahV7HNOv7AisDws2LsVATHbF0HFeoEA7lp6NzjK5dgqd+9rA95U0c7w31E1E9GbmzLADC/0eSDKEkdKGIJ4mP1erpBOc+cdJ2tVP5e6cZ7KNhzjYf19tORINCTrPAp9/aLXnoHgtLp3ozkFS/dGowLZ6Q5XInPBchgiI4TVHDDxGpwMAZp3G3yM1QDptd3pxRSv4m97QIOa7ma9l3TCK8RA/bs/akYoZnxM92GvG/3FQdws1y3Lz2NjoikVSaX0TS1t16TupL3PQioaeRJLnTZu0WGR20WLL6kEBz6cHJC3ZN9Zilnoje8lEm/7/WYOCt490+w4KS24aJcgDPzV7Z1npXy19p3ywEY0AJND8uurWeTEHIBJNxMPU2OMGd0bGa2S0yr/dfbIz3FmD06noX7/XKMjQ+gW8EBXAA7s8TA2RE0HbD8IGKlg3CCIaYsS4BbvK0B71qHhe/yM8qnUo5+vv1UpbioYVBI77UfiqqUDUAIIg+apIKJjU352GqXiEovXGR6Jeag+ufzPkPq9BqvyIfW0+3r2/wp4nIu7Z9XM6iU1Lj1j/wM1goktBnDfY6hbjHA0acQFCgUrzeGqyzYSe9kufDTSw7ePbx2rLG+fXa9qwqVwY0iBjJ8Hu6xIFmvesHwq0ySH0IqyI/Y53ee2hhju0xWAz8GishuMv4/apVLWQ4MbmG788ybGRxePWqYx/KI8M1fUvZGRXmtwAqEIaakewUVpL3QhawB4eR074Yhl5gY/ElwlcxNboUVayqJwgh4BO+/2tAutTDCtkzdLMjH4JoDpMNsf4GiLVvlSahU76B+oOlttcIm69oRB5BklrgbPCwqbQldsvvP3nHuFxBAlunefMMGZFbTd59JbO5UAkAHQ7XRw3MWDq8B3V1uCF59r4uXc+kvYFS/y8DTpQGKtO0RQx5yIonoNCbJjYWtx+zMACXoXWkrH03IQJMKmPM3IMbtMDMxIdqjD1hdaQ4dAnVcCq7ZvcbIThtCHX0+Vqo9eHoqA2kBtZLRq5rq4GG8Jm7o9mrpuVTLvym0goJuK2KQbF39CxlTG8eIIRKFQNhKC1XtuTGiIQzd14UsHWHhqhWo8uXHGhAvkl3ga8+5bDuJRhJ3ndsNE/tnq/VlJf329ATseDCLmVEDRiqe7CJeeyvMLgN0oE0lGZkmf2iYfRpB0zdkj6EpVdVZs2f/vRTp7S0ldwvV0pTDj5dzboY+nhd2hzR1+EnLPuUbVGqotTz8BWkxo9DpoGkA//5ZMeCkqFtKh3f7/UAWC5EyBZpjoPN3JGtEOdBRLX9pKrvY6tqpwaiGAHA85LywmB3UoudiGyifKe3ydIlMltsSpgc8IESwQaku2+ZlvZklm8N8KVl+ctF+n58bYS0ex63FfYoJEbUzJMcyC8Gse7zfC5MFX7nVQPWRrJ6waRu+r33KKllmKp1pqtTH1SO0N3WTP8W/npELnG6A9RnnsbtXO1WhN1HuyT5yv9KRaVPq+2EkoweAEq/Q1SGtJBX0hxWaK2UDRb4VRMHC1uDF/CVMCcfvTOQ8/ihWgrZtroDQ8J8TU0ICZVCdz3duvw5/C0eCLB5szT1EsMY2x1hKpnfS21Y7SCpG3SYv2Ii47kCex1A35Et/7MMwilelxgrwDCsXyObkepVwdrBwV6YF2qd+jMj+H4mCfhempxwCSlhXgwhS0svSPmPPAJOU4gSmcVktfs/CyqCKLzpGxHXjdcA41/gWVCeYDdjOEirh9rUIy8KlIspI+3y+XNdWrRfH9UkYQsjH7mwvixOQfc3NUvMLOSnCe4bLZ1gR4mIiaGwR15YT+Tl3AkfHu3Ic062iPlWON5Sn6ZOBE1FnGi25YOiBCdDkF1vGdzPb2SLBnucVnEqKfBB3/0KcMrT6bDApKrPxfVQfx7YJnKO6T8nddFdPne2sr2Joz+QJ4DR7nnSBvu0VEZTXLAr+K7OOSJwlE76WYT/oHDHM4LivUit0ChnsUegNFwD7zO6nz3OWYzDaB+XzVr0c5wtpZP1IYRCs20L5jOc2P1dzV7WHErHJ8/VhDZ76d//2SCCdjv5kTfwXXHsfWRK8jMV+TZSmKlKgq+pDd9Um8Ao5ShvGqMz6TThFihNrXUL2xCEXJ1ki7xL3fTTCgK/SlMt7NYeOv5xqIdQdc7tSjYt9y76UbY6bVe+i1H3ppaYh2+oBaSDyzbInglXpHEWS4yJfh7kJxXV5P2u+LeOIzmz3xpZJJCiRjdW/Bl6jbAgERAoIQABPRyc9I9OY6rL6uNAQtPDR5Idnxvsr/kLjKr3IPkeLKCYrfZFezkr7rp9oK5b8V5DjrRTNQ9+j6CqdJDUr96ocK0wvpx/HR/rCYmqave3QFmKoGUEXvqgxVRrd+sjgQlTY/1X4CgU4OYSVV8VJaV4TgLr2XWoc+P3Qq+QBNT0+E4IF8BkMZp+sVDYdvloYib8L0urBn9SZZPVGPsQ1KZZQL6rXwWJ4iQUMCYsrJRFjWWB6a++UtQVMzBgKXpeV2j69z+xlqM0Bf5QO1fCoWfsOFzHh8Z7PoJ0p/2EmR8xryZsvu7fGgNXEXVF4fUrf6i52DwAb7ptUP/PPAnp5sg5lP11byyIGLEM6hCEKbJ1uC77oNY6q/xWowBMHOROYYXcqZKGWdOo7bLPSlC3EYPj8SgaIGW7spy/xv6TCB3BaYeRWwb2VQEfxjAK1sMVYPASBhqr3jWgoKeOFdoYJ7el2BLqprHod1Vbqr+2ahq2Fjt2WIGt3mjmdb8WnGht3f7xfzbX+CYGATPzEKOOHojQJ0lpptITSm336cwdW//4qo4XdMMo/cnO5cKzbjgbAdI1eCIEaSIvmpRgs0PNQuzSKPZ3GBqvPLFPeePeOZsq+IdNXs5YqPTw7BdJ3Wm/VZzZACBSbdjP3Mbr/yG+qEIx2i0x6I690twqy+fxdKy/HHcRGcjiBMODROq+cpxRROjxHqd9/8udNQqjqcg6j/iMzOiQv0FQ9+iEyEzk/jjF8rmFlp9FtSKe4FJ+ZgNfKFAdhDVt+cu5MpW5NZJ1wKkOM2xEzSKZlYrXx1MQbEqsUb6uopkHWoS435jsGrkzgjbDUTN2SW21o/xaiSJn7/27oUiezK7sKqK70Sf2ixdqXQXwBC6sBItE6aK/VFR+r8YcU0ysxzj7WhJB+CDNatv4d4M0oFZkXB9wZ7GIPD282KqAUM+TUOqMnpLKftZAEpRGC5ck/keBU+J7/vGO//HUKOjtPsqYPPV6qY1Pc6jrUn5RkIxzc+qo5lSoae3DL/e/7a+SCKN97Elac/bOtTRy/of4jYf8HgNQVd56NxQeoy+fUboH11jwuz3BSrHmBLnbljxz42gglBRFY4Zw0Vh35KISziV9yXqj+a+72dj1iOXCc0w/27E3gQERaex5m+8eGTxKb1R32HKV9Ww94UYDdkLZwW3g7sG6uXO9+tjJY2uZk8GHFxyYlCUB8a0URVNVMYdKDHqTuhrFLOv/CWjCBg92VB19bwSGFWEfwUroQlZa9nU6FHp0a9SgpLvq2VSeReOppoSngAuft8vxNUDXeDRfZfwf4jtUdp14zLE3QvSU83RKy+Wv/4jC/Y2ro7SqZ6wAWIlYr9Js1ixbOyeXu7e99D8sjWZbB3QMD5zYpsW416jOxZ0OXKrRZ9om+B6CtGgugjxZri8us9VpZXw9Q5TDcW88Ym6Dersajy71qnndzvo0K2FJBW7EMi64J/2lr70yAJADNU9z90B3BK0X5junIBbp88MfJNKVjrm7VV4DVVk5YdmpMqxWUVW/xj51ARIxmu2boXSpUxHs9ZXAoF1C/OoIVcM/7/tOtOERzUFFRClGsw6yeTEPvPlYY6eKnKQJputuCMD/+qbhj6kpxjclAnfEJMr+Wa/QnOLp+0/Lvz9gh5hyMdgYCBIaPe1rJ7TglrqsdcoIjHObvMm2OjeYdZUAHB+Hgozu0H82XC+OD57wax1n4fw+YktMtgobt2YRENRAcyYReehwfMKM0ahR6GVIdRCXQ4RggEbyQUoTArKSS13JpliMLNEhwocFsahqxazDm//tadLKCPEjnuKrWGXEwiHpJBOLas/J2HhQEQ3XKMDCAGz+QIfkjxGvbhYARpBTgf2AWNoj1BzWwPWn1vUQk8v7osEoP0s2kaSencOFlPfRzkVowKJAnR5IZ/xv6lau7bjqsOnMutoKjJ3lWUzvjhuvAHUh7AG/t/Uubn0ZdZalVIvDR4xcjcRdQSsyxcVKg5cw9V7e8fOFocHlb/JKYUqWaG7edondhueTNK9n4YAwjgykPhcj7+aJOWJAP6tTlqIt10lC09mHIkgfGdEU7gGmODgXMj6C5bW51TGKi38mtAs4YwCiUJ/m1x+yGFP3LBsB0jswMxSIL1/5B9djzeqbYRoZAUoBuS/qPzDtSNqOO7ZLmCb2YL6vV1x9nCEUkmIvEyDNB83MxZeMMv3cIp8VXPx8X5U78sLfqTHlq8dZnhvGs9zwVOUk729bfGLuk9ZQxHuFwoodFOUMLTdgJGPaXWjEaY/rdzKnuN5GDhtJ7MDqipVFd4O7PUNCjeqQo9hJAbPRaCXh7cweIWcBkVl/0df+Y4vGtmvQEyt4wvQyYYCCVE3J5m1UK60Uf/DB3OtM08Xcr/DiRG6zdIUVcdpQzRBRIJLUoP5vDp/jj4qpoh+bsR4uIQpvU1ityWixGiAAMVZuuvnJ+G/A7mc5naLN+hH6wELoqRxDbUqNerfxulkEKIpPwiZ3l5AI5O8yLiG2Pu9tPj0QoTz5neBDDNyx2EyAlQh6Be7hSZyWqOuS5YWbs+h+XVmsNdQaY0CKDsX5NjgmtYeh1KF+RPYTs44982RosMVUnijKP5LrtM945zk38/RZ5qR/Wn66Qm2ToKEiTnw5wQFFx86/lZPeFDQKpsxx+qi9rf7pxVALvl+p7vehLrNajnFDAh5DvsNlWkID/jgipuNSFIN6TsLuMvRAbqWWJBpOOVaE9Mj174Lv+/C75EJPVMUAkzvBpr2scTNl9sSixXgdFsc1TZ3zXs+vV4AKuYjw3Gq6dmnAj6Qu0XaYfgnGZqz4lzYJIff2mP1AAPHN7rCfnlza03cAppazc1WvTqIC22Gx1Sn906cdcG8LUobdx08sXTVxi6wgyqfQUuU+JbCpH4eoHFpUMifXmGHRHciQCytE/UIOKTPX1JNFnRKmEM5DYhfD8/wi5nHgNS/L6zHqpsrWfu5UyvumZJ7XA/djiZ37x7JdpTVj/8EgIn146AYRoVlS+V1xWDOz6c1BG9BUN8ZWdpY/Y4W65owEN19CNg9eKWizEQD8TH7X5rz874WVlrsEuBOTN9feYylhT0uyJCAPWX/ARhwX2iTSVsIemAGwI8tvoqq9u8vXU/j0+EtiFYjBm+GTo/E/GqLjSsEIc+B7RnARWTjfMNqNu49DoGVLUtvQWAoZlYqGLGpvis7PlO1tNIRbhaXcSXasBbO6DpASLBZwGTfZzpm3D2OC60v52f22uwJx/2tHRUILWXgbmc7/kWnkb1FZbpUSfrkxiLcX6cK+3RLT//Pnbk9wva+noJ/aVFb9ldBkkAk4iX5XYHSTWf2IdPe5Lz1bBB2Y3WtFo0MR1LKf46yQncL+FbzWTLRSHPY3UeRhVg3FHkH6MnXYpov8hHwZ4FrJaT7LMmdj13DL3HF5lwwYzvkclyUJ2taQCwnXPlgXvWRgmYfNblc98/yn3m3wWzx5rS4gGFHqBkJYwTqW2cGuRDVZ0V3t3+UfzqIJmK8nXpm0GKjZT50PfMjsS6+uVgTHaQ38HDFvpBM/1z2Sh2fcGfbkxVBWt8Wwl0Xntt6tYYamFGfqR+8W6VRVQJitb6uZZiA+wcbO+kfZOw55VGHld/USRiRv8QuxGe95TZV47f1CcCJzZhWqiaNH65DLsLAja7DeNwxd6CHaDAik6S6rD0FyZ9PQPaICPPI4/xAo/0ZVnd/yEc8OI+3yM4Ks+YgQ02Gnrl1z9lv2Y9zytEPBDFy8iWYtiyXZ8i4U7AXOGd5i4h3jKPlW7h0OkRKiSSh4TgO7dD+5Sxk5kAMUo9nxumcCmTBWL6i6yRnsKmS0nkIyZI4wuEihk4Icof6JsPqrvXxc9VgQ6QWQ0FgAeubKbqIFgV58l2JK4Qfv3JKYrKMS/n/BCjRVZh3DfkTcZzQg+m9Ytcze7bv52bN0S2xrDITaw4q0IKPgmXI5Nwb4HA2t4p0iBHgoqtMbU2tkoVyh16EVnCwnS/IhHi4HTlcKSNDCWp52NXf0cWGjgxDV2ds37QYD6JoLz6Jf+NIUElPQ/CySdVnfcTHK6h1xjG3K5OoeIboMqJ0WxKdRm+Eu/2OpC2T/x4i0YxM6pthPXUQ+tYnjYd4csTbjE9aAVexoM+ARW6WJj/utUp0VvRQOiFRTLDVNJfzG1YUDXq3u0cAWkezq9q8bny97HBHP5vnjzymajF89NHP+bjZrvPNigJOXSPybJPPFLhTPZGjryD+78fT0VrvMHkXutC/Yqa2OEXe+jYXOhx5phxknCngScLmIudX2c/fXXxxoLeJHD9Hjv2ASlDszSEuBFDawPEMuQaNf6sjTi3PLgOaVZDID+NAh9sw3RqcnQjMcyR6ojGxkDpzxj5VBNHxbPXNuAUXPNkl8KfkAgwbP1qBWbyHAzUBg0+rBcRBjnD+WHkhiJRqKW7RMyyGMgpk7E2p75ZsdtjDX1uzxJ99QT+q3qEoM8qfAMniuUoxeVX4WWaL+eS3aDhE9hJtz2qVJjx/oYu+X6tSjSoY/3OHlum80NLM5h/tVBXi8kSFmtV9NkiGPXT3OVpEodhhCXBZOblOTOkolbawoROX1tJNXpNAJCxz5d7jkjPM/VUoBrvtXcfMBJOGyAgrfCu/qZ787tsi49ZwMKPjW7SAWzgzsVVynVS3SyPfUs69um4QESoW5rMqbnh0jTRCiCGAjK/2jDjhqpA3r395j0TDlQh9goCzwzYfEyFEAPspF73GcEcR2eb64S0bRjT/SUrPrRFUSV0MhFefwXwd+mv2VcF7Zr8GzlR9fOpngy3xrC7GkyeSz2jNSwIkpssLpvXPbG4mzXs4WBFDcDb0hZmFHvU+fLI1+Do9lQ3KbSyCXxA3VoveSEv7spX+9EGJpHjesN8cPcjChjVozfOzGWDXw9xRAFVbE/eLLrik+ftGqzmqm1zNSbXInJqfFmgeJAH95eS7j6r/kqO6b38rKtMIRMWj/2xtArTtpqmEbF7JgQNM56dIsKgf+Iea3XeV2A5wa/d1EMj7omPTUezw5beqBExgShFc5xkibXHuSTLD/ibQTya42F514GH+1CpmXJ2MtoQMBv5mxJ5l+HynS6i11kfku33m6CMPzv9H7vsO+0OMgK9zf7qOIPIN6tpOkHXJPy6ytHkPNJoQ1SStUawwwddGGOVu0u/IfaCp47sLMqIoUAF1kZSt3laLGeW0Y3/Mbdb5j5NwK+36XuWUvJs+eHIKRvc7KqcW8Ww+ReglXFdc9HGmUOHV6t7hQ6YT059ThcDZQf0JasLJwFPAo9BfHL2sgBUdF4rRt0jLBVNaXbcwO+tg374KIf7dHcKKkPQ9HT0fzkBu0+SlsEJfpqMklksImd6Ls1clJSORvKAnzcPvSbxA2vcGg++Lu2vdqSzQXD+2BegqE95A7h0Dd7VH6AvuqosfLpuarI5Hs+FX4H6vpxMa9lb8RTIi2lAI70CgggBALr8nb9910Az4BdF02PCn0uM5oa1W94D2wQN9sW88ivd2pXMRlht4y0546P96ud8Daxtv1acT2henrCw1S3I9CpR/0HDoKywEzPgN3JQsJhDfsvEhRCrKnU9miwvjCe38nlkMG9PVZmVTjlvt5UWihzbTnjv9nBSnQ6fhz4QqqRBAi8Lcmc6IKuz7CuROsY4lNCHW1xLcVoKJOTOMV1DUKCXn36K4bkiYE0lhWCtAZQBVHkJWupZpogjd5mr9qy8IfXF91iIPKw02XLgNiclPX6q4r3m98aMD0c/slvsIH0r5fphjLdoQHYPt4Mp+Vum1cGk+ogmpcwSJnBJ1qbrFvlBmcGb5LoMd9z4qhvWwWVOKw565kyWkaB5WO4v1KFx67KVdPszzAUF8u2Ac5RIPY+4Db8hvTCovDH2y3q3mBynYJX2FjHS+3Q02E66thuzHfbxHIKHSazq5gJWzr+hYfal+5kZxOfydFMIC+jdRmFajNmoKFM2LOUlZMVAHPVTK40DshixVjakvEMUCJyDHURyydgDbs9W0ElSYq9mVMXF/2m11KY0Eptzvuh1LkFHIfDOdUCjKOrsd7JeUqF860WPgxHUnAas5HKBTM2xNXEyAsQXtQk1jU/CxKgLr3WDLF4eQ76a/BO3SeGhytpasDKUMQiqXyN7v1gJeBQoyiFitC1oHUVVTg7EgJfN0B0dFWKL8iyYItWB7xKtXHPsedU9EWRfghBAxoAqf8GLW0905DMHdnIQKg/43iaKWNqmNqCVRMKQnShA6GN6tOxtvaVV4WRNtwtEuOP2U42cNA702e0qFtmWDBjARuee1qhJCuklkYdDFKrzn0MXT/5xxNCtGVLeZCFPWw0uDUQu+HjD8Izc42fnVGS8fLwGLjj0Ajnn/MtVusCHvUFJSPLG8qsCXBuhsywmtpZGKKe2EP+KKphBFfExQQJWXR9tbBGIcygK9c6wj3Tnrwii8D3oIGvEgnNYWUL0pRVSs6tpRwzXwK1el1wAoU7rUQ16UoJQx01tWEvxN7wTsbo/V3IHp8F/UAMNnK1GQDZqn/NDR1Ln70yT56kqXsNf88WI38eox55vtOCePiFmpHddvRuMZrmSu9FFQtd2rK4eDMrDuGxFJh63+n53iLFlCBbNcc1XV5CP99B3STPSzYHPS9n0aCoiDL5kJ96LelFEkFqr9gOhG/3JpW7rGw30Mv1rFN4dFKn58dSyfi2tHbz2geuIVG5BEhujxvhYg53CC8v1agYd2zlSPQnCKU2efI47iXbGw66l1ACwLWsI21pR/HVt4YyjKwy8IWJoNPPN0AjcDq1Czis6kUXfmLRDks7DciEdhOqT49zQyn4hNebkFg+VCs3Y1JfMilRYdCH5aJJn6g6w9wqE/qCx6wQuq/7Y5ImEpKEYme40uqJMjO2oekz1FhsZ8PWSku+d+Srus0pQkB8MMjHoFrAtXi0QWY1y0wo6Ci1kM6T9wbVLmF8hXkfqhEdB+RcyNqQeGquNxM6rU2JKvy/HLwO+zTD53CQC1ToYV2+5MCRr9+N2/CbcifMUN4VIEn1Eej0zwHF/yN2Dc+UYWiyEQtlG14z2hlkDP0CPGq4tt8VdftJ+HvCw8DXvTWTnLnn1Zp8JOcQmEeP99YAYcjKhKnol+34BK6OqlAPxBhpdin+TRG05T1CoGS4qDFCdS/mIdCVFv9g2/QS1SdUQIS52zaRHnQQCSCWEa+ZSTfRHd58wlVwt58M3tCbGyNiM6wA90GWFA+zPn5OSuWleAC/cHp8uaJ5p1tC2CPYxbU19N/pQmg+fwNTBO24wUN+3zJXC++eGtiFofpQjnDWXLH27+oIG+YuutaWh1jf4Jsf3HybnAmDBUf4D39zprOur24+buf+h5uDddADdFHnQ8GHo7txQ0pEU1Q5L6tUw7JY4zVLZ7PF04Bl/XLIRHwb9hGoAEGsblcahUXa6SWq6oQmyoNO5l91ZDZk2ovSdq0kMrEB543Y6Uo8UvPIDgOwvcVhjrx2BDy7H0YG8rMIerCI+4mXi+xrU5Akhyom5b8TFqsEmZN5lvrsdcNtYc4/d7qnkbVYBZlx2MyeDC+ch5f1yVBY1cLnpjFFHFUXZFmpzUrhXPc20vgeXnQQgqQtV5fbQDYUGz5KIe8d1wVGIVMut1rmRa9/dspSJMmE24mNe/K11eSymPBI+oSmwmo2KobIOb4otMXXGiNmwVSN8Yv22FoF3u2zgpx6esCfGLLScnsXOpCf0f7aP4aqwqN5yeypzAlhF3+yakuuv0m/dHUEhxuOqStrxEG8ShJv5tkHsM3V1WLRkpBAadXPy6gSysA265grR8BX4LbUZnFqvoDDNrvRSweNv2HddvI2fcgltJ/fIcEu8Qk/WNLUUWJXdMRbaUwO9IPvhQULFEUCLdqvK5bB5oDnUQQ3FTq7Lspp/naoolLMn7k6K5gx2IxQnpq9+iTCzU/vrKL+O7Mi86AHJxPCr9tk/MPEqzaH1SjA8zrPqdZdtyngTEMn5ZPiHV2zMUuWPJ2xXT2zrpyx7mVXJdl0SE2gbnOTs2/5wFPy9aTKynFtKxZB1y1iWEAlBWsTnoS8FE+6CBZH01xww9GRjoMi9xDee+wXV/olDo/dROj4RPYSvIeB3tIxorxRR17YjyzZPssKDGTvfzKM8kqYYNE/BqEKBKLCz0bhPCCWxu3JaVJomTVJTrFy9JzmBMy2O3sgLRDl6X7vkqOm1AoIIAQD7nE/lfkcKEttlB0HLSKT+yDGo8kJAR4zKmi5fZpVgWYK30Aib5HFTA9BHVZElnhTeNyvYMdSO1FdtNsa7tQ1/0rD985d/GLXe/f25PAbEsmgnFMmc9zSmpLIZ5vTxIC7Bk73mqwwgZZxSNvpqurbUO+787vMn2wKC74fJHC6NF5FMFrCypu4B5RLs6C9fGjRKab1vW2mi2967gCZrB1celCcgkBzN6XA7tvjDozDz7JU+x7ugmBx+6MKpsLc/FPrRgEhwWdPIsV6R+vOqRugeTBtr+NyvFhAa639l/e9EQwpEbVJgbNg5okOZliYDF4UM7YADgv0aKJtir+4xN5Cka7Jb8vyYIAcchy4cjz8IDNK3SuvhRmPTbEOs/xwZpoN3YqUiARI0RvYznaByKpOJSpxzqqP1W/026K6n0KagIjyQht6p5ElpsXlIgcH0fwpXseNYl2pQAzj0jAGFaJNYSBdgyQdZkoiUDprKUm9dZfDL8m9FFpoDV+BuJmxDe2XUpLfDhTnF5n/F9wYjmd4Vhfui0HA6kh0dLvOS0EZEvz4mT6zD7Sxx+T4uyZJE9nq1KOEpQTW27mzJad4jXJkiYe5C33DSEdOpwVAu8pIYFxmcj9uuNHoK2hpYcst/wYuNzgAHB9LuJaJRFLZSXN+IVyBWU2S8iejIVYzAKhm7Pj72hIE25Z7oQE/MniMQeUgmoIlqxbSpnWho+K4koZGNIyiGv3N9XFTjN9YCdWSC4AVuyfyKa8c8Wl1cWggnOwhj1CkFeMCK+f02a64kupllLUL5I2bzC2drmjpdEGB8m7KaCWl+W86pWKHKltns7u6Z0TlEPCk2Y2+ypD7GEicZSbMwAPt5jpTfxoMk2h9ICzgDbFPaJTtAsYNMiAYz9Sa+w0ELdSYoGD1OqN/ZkPE/sGRcXfAk4efEkfRDbCU0hiH2HMbKFLhH63/RfGSbgeYSGDHTs66JOJ3htSh1arYOmkwBB5v33cnVCmRiUGgE4QijTnMmYLKH42txfzD6fU1TJKUr2woazXiPvpS53tgSbO/zmBUE6fiFIaOGpT0iHXhx38sDX21VPVY4zwkYvmFNKliwgnZTZiThCNF8e1r4W5SlOyoCm+cc6UnPB1XOYx/Nd1W7Njm46rL4rsfZ2w18vATLl4ofn+6M1dgN39FO6ueKvZzxHUH1Gp2J3Z1cphfke3+O8NKi0BmIe+TjfuTzCt6l/rkr0UjKqXqYF1OedZe0kwkIRDmY6cY+gQlIdIFOaefF/3bBu95mAozWMTtZZGAPrf1QM52AJ/0fZKjoBvvZTVbeP6TnuulOcahtZVGDs3Q2Io9d5Y/c/adXwEyizH19Z8dV/ImY9JdmXDB80wDodoo0/uL8Ig/2NslKCu4KtxjzLwgKHhsz2wWgjagn3AGkD6nlVdElCPwRMdHW0v1Ld5RzZG+oXD88tXe91cLH7YY6k44pB86gD2EauwqDPSk1Q0TPy+Fj8sLEWwg/prsVZWMvwLvGCRRCCUWiDJhuWT1dzOxHTcbLJSAqSTaRDccvIrFR9YdqqmZtinnSwzByzOG0xY4uO3j4EhK3GVpi6L8zgoEqP4F1vU1EwPn/W7VfsLggBBRhG06yk+R4zOBtUNOHi3Ra/P/D7smXKmgR5hnz8tfObTgCO6FdIZAnP7DbS4bw1eykk55rG9x/k76Kd9iB6PtlnTl2gqaCcx/JX09lhWNbXL0NL9J1T+aEyJiHZyViVcHBKjXaUSlf8yYbuFMSV82iT/LgYLSmEb+tsS3bm6Sa1r4uoOrET40Dky88Oru7hoZ49f1HJrGLhoRlDO4rCnXV7QABqwAE5qJCDZ0Kx1Vvs0WrK1yypHAjbmK9O4+98Ih+65HhdXoR5Ds2Yj1ovv+d9NWBMEQpLEpOdtEoZ6xqAr1DDgdPVg5wSPtEavKOEfQWfPERqCQC/oqcO9rMbwEZGx3wcJyIZZ6jbupWGcHmSu3bvb0sJjdX69wQGL9Gl5WzR3xrqMYDX/ObNKml0QM0//SX0+j3FhMzMzwzqDc79a0FnXjjMBloIRVWsFdGqt5ZF8fXSEkHejycJDbyXZ2amxtPN9LgOZ6GvboFEnoEpslW4shx2+zO3Q/u0YYbaLGZu5zKumObpau92s8clYwC37htg/IT/JYLUVvSx6HaWj3GaVfvFlQ2/oH+Pk3MOVAyx1GXpZoOtjcs44/U1fKVIIAn0jX4g//wcsdt9jdbdU1PD6UpH5VlH8xJ3fNWxr37R8nIw8HzBnbrgm6PWH1wiWzbZSR5dAn5WUv8MS8JxMKC+QyNjZ6/kgfO1Yt0PV1EPJ4ji6A0F+akKWlYVXdbgGVQyISsje66u4fncZOMHgVwlF3X2sNe+ybRMUTysPsTAmRm2YUvIX6b0IGL+CcSWMKM7PeCyX+utfIn2IWZ0Wa5mjN56TRFBx0b9Xdnq9gLbx+HaUHSLERJloYg8jfeshmUIha6qfb7ywtLBixXcJTQUYtlXkQJ5pzXyYNWqv5gKShjAxsOMxvg/AvXw1g2TKjq/vZs7X+lIbghfEilIu8UUn1r2Lkwak0AI4si1prjsqNCaxduiZGGjeKiOlDA9c+72AmrGj8hbgCyzOq8mAYTlvadCUH2GRmQQnGVvw2pxoHpFFFBx1ZPWmmU44lnBjlWxPfQ2Ic9u1yLYHEnUVYTxDKHK5bT8940F86YFfjozWK67PFKWju0iuriL7cvbi8yxyeiTwKCCABX/mhaHRoAGGl0XRgu8izYQk5dgoWVp3YgBpI+74EFlZQKQgL8b/JvosV6WV97/iSNYNKDHGGahuFEFvroXpEE20rxxXjJvEVFlrCuRBbePeFQ1PNTI19GOxtgFmASsOqTenElUoKioJ1INJKggxPRWCTtnhmeRP6deD+kvIyJiAEHFHISdbUFgdiM+QyZhAnLiv3RFHGTyInVFbzgmCXxOEsOX3lIEC1RexGW6AC+Hr5XE3YT7fQD1HSEjSjJwfHdEd3PTyucVRsI4ftdtyv/X3nCxwswQekSeFPvBbTvnC/9WxULA+IZcM7UT/zf1go9AlfHmbdvF5meQN17ueyxiEhbHC9mnHSkOMiFkjzkYQUz/ZmNdAhLhGYVvCfTgOdjGSf9vgWoAsysADZj5cKd/EK0TBzLmrLqVgVm7PxJuC1zvxmA28GgGN5DKrANCP8Ky9EuXchRX3tMZRX/03llAtDAhJjln0XMuH4TOvPxlAYMEuXMzjM+qC9r4e+CgX3oAb04y+xV8ytq3EBJpxzU6rlWmDQlVgeqCKbpIRjViloToNyKctuUcrQxKBXEXbWef0Y8iQQyUSlE4RfThhRc+D2uCbLV9wIXxGBgy9zp+Wq2ob6a7AZDpvMh52GgtjL/HU0OZw02dF8AxJuyDI8m3FNPXzvUdngpbd4nmrl5H2PZIe+oKCS7p8QLM6064IKIulPYwBBkeWFyM3bNI/0ZDa3U4aaePJmluaWIQZRhoGtjTs5Ty18WkztdbkfubFXxNy9qnmgS8V5M7nNCFYZr7C3U2UcUXJM+GZC7HFS7voSr15JIRpxH4gM/0kblyAUibAg/pxjI6x3FOCWk6j6AUXVULGta+CrZBpzUys9H47x+hhCpXc1clO9ninAazS45Xhyb7Bul5YY81zFjMHIyW3ajl2NgEjfOPyIwziYd5qqiAILL2vFqgv6lYKtTi4F8QWSdgEOCTuj1AWH/A9MFiabM3kgfgi+RkFSM5j+NkrUGSqGUtQCdm+noOZA9UzCc6CmNJjhYgb0MWgsIfBK1aRaYBmfZEgAZm5aQmCGQbSVRNosibkq2S0WKIkswx+V3vBjiLFl5IT5WSjrfyZnAvYWPqB90dBUGpLq5xYP2tyD/ZaMOVl5xmPS/b70VVkdTFpK8dF6u+coe+COx3G1BAPbwLyHSI4Ta8xbBQd0u4meGfQKOjMFv+nJZI1UdOtyMOWK+ch1Cq9HCVeLJMRisWttYTRJWwD3v4thf+wS3lZRXNcJe8fVRs/5hDPVlEj331ZDxQ9kjT3ZInw1kb/GrmBRCOmoQMQncJJ4iSXBRiNl9wTVODt5y8p9wW/l4/tUjGGs6vJuGpjd7tqD4RiMzsVT8JATcZdxMOSImx300FwrXxh14zDJcUjLSR/MTibbiZe4VvnXyBef3XlervqD9sdrN6p9/0d6qyq65j1LhbyauEt2AFVl+nkhCkGNQG1AVXFSJ4NOgnAt4D7Plm4mK8d6hgQqnlbIynRFSMoGXqrRSuBYf4VGAdZTFpvruKZKO7bxNX/wuzpTG/l8I+nR69L1oIDmGNnit4cfvxWO3GoTJp6b81gsVKLexavCW2e5wFYOoK/9yHTu8j4AZYY3VIX9Ic3uWInWJe1O2laC0wDW9eQTuL/3g8X3yqqAB2tWyDebSn6e67cr4x5NhBLqASgWimpECey0adDrVCgSggA+dZRV6fA2niJpsAhSonzO+P7/ScTc6b/SYGjao1gQpq7nh/vioPphvcMOQvYRt0eH4Z5Xwjk9ZmzfpvxNGrdkVaBpXrXWs/+JGAJRwFrylg6uRxSs/xuxL9PBFtmegv5x3Z4Tx5SojnYKoTCiSzyFPCuF7uAEeeReGmGlY5m999oVwwcDwxKjiShh44IIbNSXTuOjgJgi8voJhFKq+rZyC7Y3MosnbCdLe0oX5cXgDSiAx4emb0L70D63dhNdBSMRAzIfrKilZGtk5CqcJs5vmJBTTDC7OOZBDVQ2fELUj2hc4p2F3S8ro1oC1hfbx8FEBDoioatCFGOPID+bXZlK285umMC93t2jQhlM6C4GtHSUEp7r7S/PvRq6pLpwwiGw7CKAKc4BXfPa81igg3qEjCRfeRywkkUpd7P6Yh9cUZKh0JawCXY7bi4WLCjzbEvq6M7BXU34O/uqgQJAtv3mYLLMkc4RRPytT4TzIUxuN5uKuJOkx9yZViprAy3Nb/kyzoPuIOFgzPIrhO54w1bqvWMMv6MW4cw7sf/G5vIuz+aNfRS5HqGlgSL2cFoEllTrxeU6JQRQqy8t/kD5nhJIA55++zc9j8yAHk/sJY0DzJulv33tQYstVofdkSEmUFmmAYMrNVB8BnguDd2fKLOpeyfSw1stu7y5DsBjNrzi+/q2wZr2naA+Fly3FEXGHySjJUGwWz9LuCYgGevfZyUT9aTsi5eufmlIG1/PJoQFy5Xud4TGAVGPB2BMs9/b1DZpbMcYW54M5Dq2eqrsfCgTLZ+jNIwopJJuDzLybSC+EA3RvbzYRrMdCCvgzQbgU7t6+9WTggPoS39Fcq7LSFqB277kIzIXQm6hD+zKECmHmPN9ruEvZ5EWdalz5ZCj+NSe2xXjW7+Pd8HYg3Sx81IllU2azy+C26QDiGjbYqbvNU7DOLvY3rQjUAXVJWkIusxfVsQmO8biXxE9iNoDPNEARvQzqhNyExrr5kMmVDbgbD5+c9/BeI2tmV7SUp9cEkQKCCAEA3gJknVFNvZgq/soq/qmChnRoDYp2sTAS0OJlJwApcyHNsT8Wp6tzDQNdbB5S5PTlPIsIkZVhMrtcMzBU//oa+FB+DUBYfzPrxMH9/cuNgGEuuRJXin/FC4JCy4+M1MILI0YgB8QZwjuJ7jCCmmiDM7xpdcPHfYUCN0vSKeuwmpxTBubYJSnhELsQsur8nzU9MpmJB+c/Fzp5PAepbX7yhGSa/p1Gl5G9Yd2uUkSyuRwLN2Tw2P6vu0XY8BRlc+VVx+mpVBMGKY1xj91tlj6QkzQYMhfRx6oONd7Z0nal8O/b4gYbgkHr9p47paKaArpmVrNw9AoqnpxM3ps7lNaszU/3uosbHND3N0oZoLqhBxpfkquE1dSyb0Fo4/An2mW/SzjsDvHi4tUzlvR+gtpF8ZwvsVpUbxTue74/wT+iFNLqJSu1aLpe5MnFXhgjm38nPlGqe1hs3TAFFAMQZqeREakFkaJRx4FLVXZMWCqef5Yu0hIl8aqH5NURUiHnDl3SUjb8f1dvNiW8CQcjiNMPQCrtFzBjBoDsgyltgYqYWsbcfCgvBzquvur6ocDqeRW3kMm3nN8vZSy6V11pprsdtOz/aC6QuVsGDkEooeUXfqr4exWFmbXVGKJTezgc+EFdBKa0uujJLHuPOHuv7lHyaT3RPRxn8abcdIe4bVJS8II3jjiuP39P+hqgw5qXaON75djxuJBUHTCJTZAhL2FiT1tB4E6TFEJpBLjL5A06kZh9Q6MqH8iCnqoWJE9wmxX4WBWNm2qLxeujMASotv7/0b6GY1t49JGXfQ+c6LQY5mtDPJ7knKtb/tW77v2THFpaD0AjeHFRile86OtGcoh82hPaV4hla0GSaxiR1TjubL6a1dgNwHs0SCQojtJf0331AqxIc4V8BUKQcpUBv/hcZV9nnMtba9ZjFtsi0hQg0/3huwpVDKje1gwHXnzRPesWTDN3QlM/pkEDxydf7yHr7sRhLhXF2rSjB0Vnogq2Imw0zFRHfDc0HYxt3J1nc5u8ssX7JrI2F6Y9M4oKwh764xTTuNF79UbqV1nqo/s18OzTr8V25Nu60r2mblxTUhFk6bvz5wmzsv/GL/i41z+qnudlCkNDL3qAoQoT8uhaxSpJPNK1DplB/YPLF6lG7WbtyGmp4NEBZzLDbTUoDD3060e9Pi7VxbBnX8wwptKZ6FZRUSGsyWsUNU40pZp+qp0kXfqIOBz9vUAxK0o+/qsrqe9Jn1SPf8O6Wb82c2LL9KMIrpmuY2jwUJa1LNUS2xxhixxUwop2GZb0YgUqozqzJxU4ko+I4jgoF8MKGAnu9x0pzo9IbABgYeisHVhIXHx/2vCq9i5klyofDn12h36FIthMGGiYEKSqKcOzuyFIMkXhGINwXhpwgWXbxFfXyeZnMjqYCTr/UeJPIK2THjsEckGyUaW/OKPqDQYZrgmHxZ5+sGgrJKBQQlIuyXb7U9I2c8yNxZW1L9IDG/RRgBQWVkfSQA4qV0+0vcvGlJ7E+GV3cGzbyzxYAq4Jwk3vNF63rSpGVsRCGyPv9LR4fsV6jMpX7NLlRSbIv2Gm+QDjVkOL/Ot7h82BByj5wj2eh/WRSrpUvdgp/iG3oPn6JRkU4w8gYHR+aIobX1e0f7STwZ3jWxZTIor6pxUTTUOsf1nZjAsFdjOVLtrf3IJAfKAc6QnkXA9krtyhleUUlb6S65LBsa5zO3WyBVHT/JOblK/phDiGlZc/GofnMfgRZkec+k8Dgd7f4wIt6ZHWTYKBRzzWTfav/gHNeZBNdG/eNL6pmb4ano4tLP46arruihMVIMH8WSmG2q7gcXbDxTyHi9qPKzkwNq/h+SW18WJ+9/qBEDQ5AVKsAfJaUd7qIUmJ040lL/xUTV075bnpkBuHb5+M29JAFJe2P2vULBtv3Jc56pq/lri35sSME9eniAzUexzUp/iT4Y8fFib8TJ+ZLQ5ezHDs4o8yngXg7xDUF+V8IGazHUMDICtl+IpeuViut68EH4jR7KldLsO5syRkJU+2lpaeh+7HUwXzBPRbm0iO42h8PW5rIDxp1iQKruVtnS+e5B/0P0OLD/JFReX2TWAEWMWGBHm29Quil/VHc4XQ8sMODvUb+hEVLUsv/iv9iVXx48ERGTiotz3e9zgv84SEZFbYjM5DhG2+CWwCS24OEmgYWM2P8G7OSuwa0RmmDPshoBQVf4+ZzuBxFBPVRLC0pvvdJMow2DpTRcKCq9CS4MG0QS1AH2QRCuT9VsrYTueWGxi75+Sq6tOcSR0CEM/MkLgz/KPeNcu6r8ywuSKbIYDZtoAvwOrZ0swTEE4F05yeVJ0CmTxaQa2GkpLPaxPSpMOWCHNF9Bp9RTeeGNAVzEcEIf5L5TK/ayA6eN4MGob3PjByTlNtxOTn5cIHkYSd1mROIyh14hMeZ4gPTXNqWwZG3G8tHz1GKDTflZkb9a6Wm0iUd2xPaiStKFpQSlm7zxyRfm7b1K6hbIQvs8ulciXVr1dz4w9Y62+cGyDbox4JikfONtmKEcsroixC2JVSgqVIHYvHoMR4mXX2Mft2Occ04gE+iCbE5wcIheYFncStlNeLvFGSjCQvw9y9PJ6wLI482gAaaivJIFgGxsvu6DRDu4XrwF1ISoH6KALeSRlMJ+ZdKQUAxFDJLnGPXew7GFoGXNygE6IexiWV/swbq/VLl2BM4IvboDzAhtERI3zLRPMLfEg4OOjAO9zmvGMCgggBAI0U6B4zfbor2UG5zkmlHcBbOc+a3/N4PNZLwcWfMcS6hzsU8v7fgM1sOz03K4EEPr1ULSI/Tq71XsIcaGPt124quX2O6wzplYsDYy40MBeeKry7xsaLnGo5UCqvCprelYx2zGUY/fuz2UJxbeMyM9m9uTBZ8h3rOuioGQgmRDhI+ACcti4kMKg1W0nqd2pZ69tgCEGt3H2puq9SmukNm41xYE3YkMvo5e7yjlWVcdQ93K3x3dPP8mtr6ckkKMhOoxDB3tsd69LTxXc3ebhD1u/pGhqyAvpXcPaN0TqjhNMKdnn+G+g7BfOjmO0FsF4ElRO5d/O7KrUs/E6vfvE4m46KeWlE1plG8C6Ukx/Af6UwCHtWTMQihLfskuIMz67o/YDOnJ7miGb146yd3E1nOjydRwUoSeVPYzLCL4R7aO8DCdKbmVnQyh/xUBSM1m+MWH/UyqFQMx+vFMseDoPjx/+G2ZvKa/GXNRoThXonVpAFFXUzEU2DzIzxa75FWUNU4Nhc9h3HLsYCG4hYYb2ab45cQD3uOjHIS1VB6tXKLbwBfIQFH9bi3wnUdmGBnRHAU3NEvflxmNFCejBZoLsbqp/niVr1BIzvmmZHOR0di07sVkKdoRGBFuLuS53UPOBndoQJre+SESEyVNwdN8jnDFsCQ3k4KZbS9d85MgoCagtNA9XaZ0kvQtwP7zBqVAwEeCn/cJG2yKbVMXOstGGW4TexTHiGlSCT0Q9lSAYPLJqT96x8vL5JoUeGaIL1h7b1hdwR14LZgp0nmROKzCKovASkuMaPvSDv8kA3TLG9mJD0flp8cB2y3+njj3j8O8aY/RHx3qNJwIR8djGnmcpw5hjzFA6rbx0zj0UCc69ogNTbdeh9Ia5Z9RMdsEUkBLj1+AABk5AV90xv8wAUjxzpflhR+fz51wsvAL9CIPwvJIvbzSHZEPOgKiW1zwOkO8NOrG1GdyPYgD2JseLxfZQ3pivqfcOekLJ+X8ZT3VN+wqojz99lXFUDA4IU5WSRYVROmWypZm5LfhulX13+REGgDs0sGNmjqCNcsQ6UW6NFFIK6dh6OmVnuKW1+lSG097xlhv64IDabYM/wf9kH34QLyyZvI0OVVoUnKuiebZMExAZJ8NzxTyM6ol/J8wIHRuSHXwu812AVgUdIDGdswNF2PjapNrb/6TRXZeP6BtizlHWoMlpJp9QaNOqhNPj3uONB7P8EJrS0u23SXunTz+GIKzGP4x/a1hDburtYmoKUrls+rF2eufbTypANSJf5u7niVnXaQn2Mpy07FeeeptyYi2hWgXOrWjtsUy9OLRgR4TKyzKtj3rJX+jRJ/SgNv59VQta83JN0Xw+4qIJPWhYHvgSAdp1EugnSK70PvoLN7T3OX3Ox3HjtKcZR/ClR9w2hpoWbGEMmqeiUug64aYFQ6UyBeKWEUpPT3rd7Cusu57WiSoj7OsXX5vWlUz23Dmz/UqJq91qo7UorjlueIkyMgPpaKfEeF6FM5i/lkBBPlB0rD8l5RaJ7c6EgD/6ahcyM7bteQIpL/7P8G9VcWD/45D1HqOhc0DXenSJuZBnA50IMLaT77bomcEtEigMCiBjKpTCSjNJ/CL9aJe+1EOQpYL8kEH7ZHrUlQtO8tnOCM3tQ+0d72g0zo35pPTbwgEOdH7BAqs4z/EEdjmEaE15VdmVeDXYUUnl2XSX4TO45G2l5O7wzLwEvYWRx+cez6ro+Hv4f5MkPeQvyqLzKwuwocOG6GD4TzsUl3w/h3Tw+kEkbzPW2UijeygLSYad44jmfwTQCwee/DzPbiGXv8a3Zo1KrT0+RLgKQ4K5/RLqLFfcHZtgKqSIFKbPaNPoBs9YWNkR82Be75bYtyAWuaNj/taw9h06hHBlN9JAlXE46wUjZ8ScG9Lw+pI9SxW+k5sWzrOjCv0rH6wGF2XwEjU7zTXe9njj4zPj+Jgsc1Q01ThYUNfAXG0M1cm9SteVEovgRXT14nv3yqgyOMW2Q/REGqNuyRvrbxjfwfk7ZbvVF6mDR5ayB0qdnH5YlEbDfE/MmbEQ1UQvEkbMZsyrCzNjUoG/DxThsuCARZt1P9OpDSYmcG1LL4TgFfSZIF40QfHeJJjZhwotCwrBSWCkThF/TAHO6MFYaUvX0iofIMzIjdojuf7eTLU2dVaxDLoYWorvKl18T1zo9ESws0Ro453sXTzvQbyGJaDhYQbAhkYwvzX3D1tq4r4iBDqTlJKGsX59z2G1m5K48dIAqpjysknMyDeCz2MyfpKbj1ja0GzuNbtv0X48PMR+6PTMc25zatNU93aDR30fE1BEtjRgUrUuZzMSC0FkAkuqWTuN0mK7kYZZ8Uv3fSa0pOGh/uEyuIZ2+slOobCeqiG9hgmOnjvPAY/DXRTu+sSsRyeICSfLaawja3ZGhpz/fTFKygSY8O8Iolyg1MeyPrIz3eNndkd7RlBifbN+RZD8pNHJhljHnBRvO579Kn5eBey9cih0/DCXrqiJrxz2/rulNezKuLsY3m+l//IqzA38kpR5sbHEDoO+0HZcNTpU7hsc+3yj806eZ0SvJdDLxjiOoebLBLo6JebfOmaBAjplam8GLLuoJfH0DlwJkAEUEQvcx4Y0AbUAL3CmQUHWHiGrlCrWml7nlIyEhLj7Uj32z9lRXxBBrH5obgwl8RWpmCAti7K4ryFSveRMo0A67wR3APYYvF1DoSbIRABn2ikQVvPrcjiXDNwkx \ No newline at end of file diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 0aefa757f..052ddfbd7 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -60,6 +60,28 @@ class TestRegression(unittest.TestCase): # The public key corresponding to `RSA_2048_PRIV_KEY`. RSA_2048_PUB_KEY = b64decode(f.read().strip()) + with RESOURCE_DIR.child('pycryptopp-rsa-1024-priv.txt').open('r') as f: + # Created using `pycryptopp`: + # + # from base64 import b64encode + # from pycryptopp.publickey import rsa + # priv = rsa.generate(1024) + # priv_str = b64encode(priv.serialize()) + # pub_str = b64encode(priv.get_verifying_key().serialize()) + RSA_TINY_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_TINY_PRIV_KEY, native_bytes) + + with RESOURCE_DIR.child('pycryptopp-rsa-32768-priv.txt').open('r') as f: + # Created using `pycryptopp`: + # + # from base64 import b64encode + # from pycryptopp.publickey import rsa + # priv = rsa.generate(32768) + # priv_str = b64encode(priv.serialize()) + # pub_str = b64encode(priv.get_verifying_key().serialize()) + RSA_HUGE_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_HUGE_PRIV_KEY, native_bytes) + def test_old_start_up_test(self): """ This was the old startup test run at import time in `pycryptopp.cipher.aes`. @@ -232,6 +254,22 @@ class TestRegression(unittest.TestCase): priv_key, pub_key = rsa.create_signing_keypair_from_string(self.RSA_2048_PRIV_KEY) rsa.verify_signature(pub_key, self.RSA_2048_SIG, b'test') + def test_decode_tiny_rsa_keypair(self): + ''' + An unreasonably small RSA key is rejected ("unreasonably small" + means less that 2048 bits) + ''' + with self.assertRaises(ValueError): + rsa.create_signing_keypair_from_string(self.RSA_TINY_PRIV_KEY) + + def test_decode_huge_rsa_keypair(self): + ''' + An unreasonably _large_ RSA key is rejected ("unreasonably large" + means 32768 or more bits) + ''' + with self.assertRaises(ValueError): + rsa.create_signing_keypair_from_string(self.RSA_HUGE_PRIV_KEY) + def test_encrypt_data_not_bytes(self): ''' only bytes can be encrypted From 2336cae78c02adec4fe9516f75f506c5f20ce075 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 28 Oct 2021 08:26:13 +0100 Subject: [PATCH 098/916] remove step, release checklist Signed-off-by: fenn-cs --- .github/CONTRIBUTING.rst | 6 ++++++ README.rst | 6 ++++++ docs/release-checklist.rst | 43 +++++++++++++++++++++++++------------- newsfragments/3816.minor | 0 4 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 newsfragments/3816.minor diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index b59385aa4..fa3d66ffe 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -18,3 +18,9 @@ Examples of contributions include: Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standards `_ and the `Contributor Code of Conduct <../docs/CODE_OF_CONDUCT.md>`_. + + +🥳 First Contribution? +====================== + +If you are committing to Tahoe for the very first time, consider adding your name to our contributor list in `CREDITS <../CREDITS>`__ diff --git a/README.rst b/README.rst index 705ed11bb..20748a8db 100644 --- a/README.rst +++ b/README.rst @@ -97,6 +97,12 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. +🥳 First Contribution? +---------------------- + +If you are committing to Tahoe for the very first time, consider adding your name to our contributor list in `CREDITS `__ + + 🤝 Supporters -------------- diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index f943abb5d..dc060cd8d 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -3,9 +3,8 @@ Release Checklist ================= -These instructions were produced while making the 1.15.0 release. They -are based on the original instructions (in old revisions in the file -`docs/how_to_make_a_tahoe-lafs_release.org`). +This release checklist specifies a series of checks that anyone engaged in +releasing a version of Tahoe should follow. Any contributor can do the first part of the release preparation. Only certain contributors can perform other parts. These are the two main @@ -13,9 +12,12 @@ sections of this checklist (and could be done by different people). A final section describes how to announce the release. +This checklist is based on the original instructions (in old revisions in the file +`docs/how_to_make_a_tahoe-lafs_release.org`). + Any Contributor ---------------- +``````````````` Anyone who can create normal PRs should be able to complete this portion of the release process. @@ -33,12 +35,29 @@ Tuesday if you want to get anything in"). - Create a ticket for the release in Trac - Ticket number needed in next section +Get a clean checkout +```````````````````` + +The release proccess involves compressing source files and putting them in formats +suitable for distribution such as ``.tar.gz`` and ``zip``. That said, it's neccesary to +the release process begins with a clean checkout to avoid making a release with +previously generated files. + +- Inside the tahoe root dir run ``git clone . ../tahoe-release-x.x.x`` where (x.x.x is the release number such as 1.16.0). + +*The above command would create a new directory at the same level as your original clone named +``tahoe-release-x.x.x``. You could name the folder however you want but it would be a good +practice to give it the release name. You MAY also discard this directory once the release +process is complete.* + +- ``cd into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]```` + Create Branch and Apply Updates ``````````````````````````````` -- Create a branch for release-candidates (e.g. `XXXX.release-1.15.0.rc0`) -- run `tox -e news` to produce a new NEWS.txt file (this does a commit) +- Create a branch for the release (e.g. `XXXX.release-1.16.0`) +- run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) - create the news for the release - newsfragments/.minor @@ -46,7 +65,7 @@ Create Branch and Apply Updates - manually fix NEWS.txt - - proper title for latest release ("Release 1.15.0" instead of "Release ...post1432") + - proper title for latest release ("Release 1.16.0" instead of "Release ...post1432") - double-check date (maybe release will be in the future) - spot-check the release notes (these come from the newsfragments files though so don't do heavy editing) @@ -54,7 +73,7 @@ Create Branch and Apply Updates - update "relnotes.txt" - - update all mentions of 1.14.0 -> 1.15.0 + - update all mentions of 1.16.0 -> 1.16.x - update "previous release" statement and date - summarize major changes - commit it @@ -63,12 +82,6 @@ Create Branch and Apply Updates - change the value given for `version` from `OLD.post1` to `NEW.post1` -- update "CREDITS" - - - are there any new contributors in this release? - - one way: git log release-1.14.0.. | grep Author | sort | uniq - - commit it - - update "docs/known_issues.rst" if appropriate - update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github @@ -125,7 +138,7 @@ they will need to evaluate which contributors' signatures they trust. Privileged Contributor ------------------------ +`````````````````````` Steps in this portion require special access to keys or infrastructure. For example, **access to tahoe-lafs.org** to upload diff --git a/newsfragments/3816.minor b/newsfragments/3816.minor new file mode 100644 index 000000000..e69de29bb From 972790cdebe6056ae93c59452975c7c1b67c7c9c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 09:47:47 -0400 Subject: [PATCH 099/916] news fragment --- newsfragments/3830.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3830.minor diff --git a/newsfragments/3830.minor b/newsfragments/3830.minor new file mode 100644 index 000000000..e69de29bb From 70fb5d563abfcd809ce627b5ed35c0b09d55d684 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 09:48:26 -0400 Subject: [PATCH 100/916] Get rid of the public expiration_time attribute LeaseInfo now has a getter and a setter for this attribute. LeaseInfo is now also immutable by way of `attrs`. LeaseInfo is now also comparable by way of `attrs`. --- src/allmydata/scripts/debug.py | 8 +-- src/allmydata/storage/immutable.py | 6 +-- src/allmydata/storage/lease.py | 72 ++++++++++++++++++++------ src/allmydata/storage/mutable.py | 7 ++- src/allmydata/test/test_storage.py | 23 +++----- src/allmydata/test/test_storage_web.py | 2 +- 6 files changed, 73 insertions(+), 45 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 2d6ba4602..4d3f4cb21 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -170,7 +170,7 @@ def dump_immutable_lease_info(f, out): leases = list(f.get_leases()) if leases: for i,lease in enumerate(leases): - when = format_expiration_time(lease.expiration_time) + when = format_expiration_time(lease.get_expiration_time()) print(" Lease #%d: owner=%d, expire in %s" \ % (i, lease.owner_num, when), file=out) else: @@ -223,7 +223,7 @@ def dump_mutable_share(options): print(file=out) print(" Lease #%d:" % leasenum, file=out) print(" ownerid: %d" % lease.owner_num, file=out) - when = format_expiration_time(lease.expiration_time) + when = format_expiration_time(lease.get_expiration_time()) print(" expires in %s" % when, file=out) print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) @@ -730,7 +730,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): m = MutableShareFile(abs_sharefile) WE, nodeid = m._read_write_enabler_and_nodeid(f) data_length = m._read_data_length(f) - expiration_time = min( [lease.expiration_time + expiration_time = min( [lease.get_expiration_time() for (i,lease) in m._enumerate_leases(f)] ) expiration = max(0, expiration_time - now) @@ -811,7 +811,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) - expiration_time = min( [lease.expiration_time + expiration_time = min( [lease.get_expiration_time() for lease in sf.get_leases()] ) expiration = max(0, expiration_time - now) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index b8b18f140..81470eed8 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -156,9 +156,9 @@ class ShareFile(object): for i,lease in enumerate(self.get_leases()): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: + if new_expire_time > lease.get_expiration_time(): # yes - lease.expiration_time = new_expire_time + lease = lease.renew(new_expire_time) with open(self.home, 'rb+') as f: self._write_lease_record(f, i, lease) return @@ -167,7 +167,7 @@ class ShareFile(object): def add_or_renew_lease(self, lease_info): try: self.renew_lease(lease_info.renew_secret, - lease_info.expiration_time) + lease_info.get_expiration_time()) except IndexError: self.add_lease(lease_info) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 187f32406..594d61cf5 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -13,24 +13,64 @@ if PY2: import struct, time +import attr + +@attr.s(frozen=True) class LeaseInfo(object): - def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, - expiration_time=None, nodeid=None): - self.owner_num = owner_num - self.renew_secret = renew_secret - self.cancel_secret = cancel_secret - self.expiration_time = expiration_time - if nodeid is not None: - assert isinstance(nodeid, bytes) - assert len(nodeid) == 20 - self.nodeid = nodeid + """ + Represent the details of one lease, a marker which is intended to inform + the storage server how long to store a particular share. + """ + owner_num = attr.ib(default=None) + + # Don't put secrets into the default string representation. This makes it + # slightly less likely the secrets will accidentally be leaked to + # someplace they're not meant to be. + renew_secret = attr.ib(default=None, repr=False) + cancel_secret = attr.ib(default=None, repr=False) + + _expiration_time = attr.ib(default=None) + + nodeid = attr.ib(default=None) + + @nodeid.validator + def _validate_nodeid(self, attribute, value): + if value is not None: + if not isinstance(value, bytes): + raise ValueError( + "nodeid value must be bytes, not {!r}".format(value), + ) + if len(value) != 20: + raise ValueError( + "nodeid value must be 20 bytes long, not {!r}".format(value), + ) + return None def get_expiration_time(self): - return self.expiration_time + # type: () -> float + """ + Retrieve a POSIX timestamp representing the time at which this lease is + set to expire. + """ + return self._expiration_time + + def renew(self, new_expire_time): + # type: (float) -> LeaseInfo + """ + Create a new lease the same as this one but with a new expiration time. + + :param new_expire_time: The new expiration time. + + :return: The new lease info. + """ + return attr.assoc( + self, + _expiration_time=new_expire_time, + ) def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period - return self.expiration_time - 31*24*60*60 + return self._expiration_time - 31*24*60*60 def get_age(self): return time.time() - self.get_grant_renew_time_time() @@ -39,7 +79,7 @@ class LeaseInfo(object): (self.owner_num, self.renew_secret, self.cancel_secret, - self.expiration_time) = struct.unpack(">L32s32sL", data) + self._expiration_time) = struct.unpack(">L32s32sL", data) self.nodeid = None return self @@ -47,18 +87,18 @@ class LeaseInfo(object): return struct.pack(">L32s32sL", self.owner_num, self.renew_secret, self.cancel_secret, - int(self.expiration_time)) + int(self._expiration_time)) def to_mutable_data(self): return struct.pack(">LL32s32s20s", self.owner_num, - int(self.expiration_time), + int(self._expiration_time), self.renew_secret, self.cancel_secret, self.nodeid) def from_mutable_data(self, data): (self.owner_num, - self.expiration_time, + self._expiration_time, self.renew_secret, self.cancel_secret, self.nodeid) = struct.unpack(">LL32s32s20s", data) return self diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 2ef0c3215..53a38fae9 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -304,9 +304,9 @@ class MutableShareFile(object): for (leasenum,lease) in self._enumerate_leases(f): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: + if new_expire_time > lease.get_expiration_time(): # yes - lease.expiration_time = new_expire_time + lease = lease.renew(new_expire_time) self._write_lease_record(f, leasenum, lease) return accepting_nodeids.add(lease.nodeid) @@ -324,7 +324,7 @@ class MutableShareFile(object): precondition(lease_info.owner_num != 0) # 0 means "no lease here" try: self.renew_lease(lease_info.renew_secret, - lease_info.expiration_time) + lease_info.get_expiration_time()) except IndexError: self.add_lease(lease_info) @@ -454,4 +454,3 @@ def create_mutable_sharefile(filename, my_nodeid, write_enabler, parent): ms.create(my_nodeid, write_enabler) del ms return MutableShareFile(filename, parent) - diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index d18960a1e..8123be2c5 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -835,7 +835,7 @@ class Server(unittest.TestCase): # Start out with single lease created with bucket: renewal_secret, cancel_secret = self.create_bucket_5_shares(ss, b"si0") [lease] = ss.get_leases(b"si0") - self.assertEqual(lease.expiration_time, 123 + DEFAULT_RENEWAL_TIME) + self.assertEqual(lease.get_expiration_time(), 123 + DEFAULT_RENEWAL_TIME) # Time passes: clock.advance(123456) @@ -843,7 +843,7 @@ class Server(unittest.TestCase): # Adding a lease with matching renewal secret just renews it: ss.remote_add_lease(b"si0", renewal_secret, cancel_secret) [lease] = ss.get_leases(b"si0") - self.assertEqual(lease.expiration_time, 123 + 123456 + DEFAULT_RENEWAL_TIME) + self.assertEqual(lease.get_expiration_time(), 123 + 123456 + DEFAULT_RENEWAL_TIME) def test_have_shares(self): """By default the StorageServer has no shares.""" @@ -1230,17 +1230,6 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(a.cancel_secret, b.cancel_secret) self.failUnlessEqual(a.nodeid, b.nodeid) - def compare_leases(self, leases_a, leases_b): - self.failUnlessEqual(len(leases_a), len(leases_b)) - for i in range(len(leases_a)): - a = leases_a[i] - b = leases_b[i] - self.failUnlessEqual(a.owner_num, b.owner_num) - self.failUnlessEqual(a.renew_secret, b.renew_secret) - self.failUnlessEqual(a.cancel_secret, b.cancel_secret) - self.failUnlessEqual(a.nodeid, b.nodeid) - self.failUnlessEqual(a.expiration_time, b.expiration_time) - def test_leases(self): ss = self.create("test_leases") def secrets(n): @@ -1321,11 +1310,11 @@ class MutableServer(unittest.TestCase): self.failUnlessIn("I have leases accepted by nodeids:", e_s) self.failUnlessIn("nodeids: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' .", e_s) - self.compare_leases(all_leases, list(s0.get_leases())) + self.assertEqual(all_leases, list(s0.get_leases())) # reading shares should not modify the timestamp read(b"si1", [], [(0,200)]) - self.compare_leases(all_leases, list(s0.get_leases())) + self.assertEqual(all_leases, list(s0.get_leases())) write(b"si1", secrets(0), {0: ([], [(200, b"make me bigger")], None)}, []) @@ -1359,7 +1348,7 @@ class MutableServer(unittest.TestCase): "shares", storage_index_to_dir(b"si1")) s0 = MutableShareFile(os.path.join(bucket_dir, "0")) [lease] = s0.get_leases() - self.assertEqual(lease.expiration_time, 235 + DEFAULT_RENEWAL_TIME) + self.assertEqual(lease.get_expiration_time(), 235 + DEFAULT_RENEWAL_TIME) # Time passes... clock.advance(835) @@ -1367,7 +1356,7 @@ class MutableServer(unittest.TestCase): # Adding a lease renews it: ss.remote_add_lease(b"si1", renew_secret, cancel_secret) [lease] = s0.get_leases() - self.assertEqual(lease.expiration_time, + self.assertEqual(lease.get_expiration_time(), 235 + 835 + DEFAULT_RENEWAL_TIME) def test_remove(self): diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index b3f5fac98..e905b240d 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -490,7 +490,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): # current lease has), so we have to reach inside it. for i,lease in enumerate(sf.get_leases()): if lease.renew_secret == renew_secret: - lease.expiration_time = new_expire_time + lease = lease.renew(new_expire_time) f = open(sf.home, 'rb+') sf._write_lease_record(f, i, lease) f.close() From 76caf4634710344f839b8b8e58bfc424a124fdcc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:23:58 -0400 Subject: [PATCH 101/916] make the alternate LeaseInfo constructors into class methods --- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/lease.py | 46 +++++++++++++++++++++--------- src/allmydata/storage/mutable.py | 2 +- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 81470eed8..0042673f5 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -144,7 +144,7 @@ class ShareFile(object): for i in range(num_leases): data = f.read(self.LEASE_SIZE) if data: - yield LeaseInfo().from_immutable_data(data) + yield LeaseInfo.from_immutable_data(data) def add_lease(self, lease_info): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 594d61cf5..191edbe1a 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -75,13 +75,22 @@ class LeaseInfo(object): def get_age(self): return time.time() - self.get_grant_renew_time_time() - def from_immutable_data(self, data): - (self.owner_num, - self.renew_secret, - self.cancel_secret, - self._expiration_time) = struct.unpack(">L32s32sL", data) - self.nodeid = None - return self + @classmethod + def from_immutable_data(cls, data): + # type: (bytes) -> cls + """ + Create a new instance from the encoded data given. + + :param data: A lease serialized using the immutable-share-file format. + """ + names = [ + "owner_num", + "renew_secret", + "cancel_secret", + "expiration_time", + ] + values = struct.unpack(">L32s32sL", data) + return cls(nodeid=None, **dict(zip(names, values))) def to_immutable_data(self): return struct.pack(">L32s32sL", @@ -96,9 +105,20 @@ class LeaseInfo(object): self.renew_secret, self.cancel_secret, self.nodeid) - def from_mutable_data(self, data): - (self.owner_num, - self._expiration_time, - self.renew_secret, self.cancel_secret, - self.nodeid) = struct.unpack(">LL32s32s20s", data) - return self + @classmethod + def from_mutable_data(cls, data): + # (bytes) -> cls + """ + Create a new instance from the encoded data given. + + :param data: A lease serialized using the mutable-share-file format. + """ + names = [ + "owner_num", + "expiration_time", + "renew_secret", + "cancel_secret", + "nodeid", + ] + values = struct.unpack(">LL32s32s20s", data) + return cls(**dict(zip(names, values))) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 53a38fae9..e6f24679b 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -253,7 +253,7 @@ class MutableShareFile(object): f.seek(offset) assert f.tell() == offset data = f.read(self.LEASE_SIZE) - lease_info = LeaseInfo().from_mutable_data(data) + lease_info = LeaseInfo.from_mutable_data(data) if lease_info.owner_num == 0: return None return lease_info From 3514995068b7b132d3f7c590b4ecb8347f36655e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:26:30 -0400 Subject: [PATCH 102/916] some versions of mypy don't like this so nevermind --- src/allmydata/storage/lease.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 191edbe1a..17683a888 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -77,7 +77,6 @@ class LeaseInfo(object): @classmethod def from_immutable_data(cls, data): - # type: (bytes) -> cls """ Create a new instance from the encoded data given. @@ -107,7 +106,6 @@ class LeaseInfo(object): @classmethod def from_mutable_data(cls, data): - # (bytes) -> cls """ Create a new instance from the encoded data given. From 125c937d466db13e27ab06cc40e5d68aa5d93d28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Oct 2021 10:49:08 -0400 Subject: [PATCH 103/916] Switch to HTTP header scheme. --- docs/proposed/http-storage-node-protocol.rst | 38 ++++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index d5b6653be..fd1db5c4c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -450,16 +450,22 @@ A lease is also created for the shares. Details of the buckets to create are encoded in the request body. For example:: - {"renew-secret": "efgh", "cancel-secret": "ijkl", - "upload-secret": "xyzf", - "share-numbers": [1, 7, ...], "allocated-size": 12345} + {"share-numbers": [1, 7, ...], "allocated-size": 12345} + +The request must include ``WWW-Authenticate`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. +For example:: + + WWW-Authenticate: x-tahoe-renew-secret + WWW-Authenticate: x-tahoe-cancel-secret + WWW-Authenticate: x-tahoe-upload-secret The response body includes encoded information about the created buckets. For example:: {"already-have": [1, ...], "allocated": [7, ...]} -The uplaod secret is an opaque _byte_ string. +The upload secret is an opaque _byte_ string. It will be generated by hashing a combination of:b 1. A tag. @@ -521,9 +527,9 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). -The request body looks this, with data and upload secret being bytes:: +The request must include a ``Authorization`` header that includes the upload secret:: - { "upload-secret": "xyzf", "data": "thedata" } + Authorization: x-tahoe-upload-secret Responses: @@ -727,9 +733,11 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: POST /v1/immutable/AAAAAAAAAAAAAAAA - {"renew-secret": "efgh", "cancel-secret": "ijkl", - "upload-secret": "xyzf", - "share-numbers": [1, 7], "allocated-size": 48} + WWW-Authenticate: x-tahoe-renew-secret efgh + WWW-Authenticate: x-tahoe-cancel-secret jjkl + WWW-Authenticate: x-tahoe-upload-secret xyzf + + {"share-numbers": [1, 7], "allocated-size": 48} 200 OK {"already-have": [1], "allocated": [7]} @@ -738,22 +746,22 @@ Immutable Data PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 0-15/48 - - {"upload-secret": b"xyzf", "data": "first 16 bytes!!" + Authorization: x-tahoe-upload-secret xyzf + 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 16-31/48 - - {"upload-secret": "xyzf", "data": "second 16 bytes!" + Authorization: x-tahoe-upload-secret xyzf + 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 32-47/48 - - {"upload-secret": "xyzf", "data": "final 16 bytes!!" + Authorization: x-tahoe-upload-secret xyzf + 201 CREATED From f635aec5bebda81ad1c073efa598e3546861ebf9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:53:29 -0400 Subject: [PATCH 104/916] news fragment --- newsfragments/3832.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3832.minor diff --git a/newsfragments/3832.minor b/newsfragments/3832.minor new file mode 100644 index 000000000..e69de29bb From 65d3ab614256a21430dc77b2982137de1cccfd8b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:53:52 -0400 Subject: [PATCH 105/916] move backdating logic into mutable/immutable share files --- src/allmydata/storage/immutable.py | 15 +++++++++++++-- src/allmydata/storage/mutable.py | 15 +++++++++++++-- src/allmydata/test/test_storage_web.py | 12 +----------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0042673f5..7712e568a 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -152,11 +152,22 @@ class ShareFile(object): self._write_lease_record(f, num_leases, lease_info) self._write_num_leases(f, num_leases+1) - def renew_lease(self, renew_secret, new_expire_time): + def renew_lease(self, renew_secret, new_expire_time, allow_backdate=False): + # type: (bytes, int, bool) -> None + """ + Update the expiration time on an existing lease. + + :param allow_backdate: If ``True`` then allow the new expiration time + to be before the current expiration time. Otherwise, make no + change when this is the case. + + :raise IndexError: If there is no lease matching the given renew + secret. + """ for i,lease in enumerate(self.get_leases()): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.get_expiration_time(): + if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes lease = lease.renew(new_expire_time) with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index e6f24679b..de840b89a 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -298,13 +298,24 @@ class MutableShareFile(object): else: self._write_lease_record(f, num_lease_slots, lease_info) - def renew_lease(self, renew_secret, new_expire_time): + def renew_lease(self, renew_secret, new_expire_time, allow_backdate=False): + # type: (bytes, int, bool) -> None + """ + Update the expiration time on an existing lease. + + :param allow_backdate: If ``True`` then allow the new expiration time + to be before the current expiration time. Otherwise, make no + change when this is the case. + + :raise IndexError: If there is no lease matching the given renew + secret. + """ accepting_nodeids = set() with open(self.home, 'rb+') as f: for (leasenum,lease) in self._enumerate_leases(f): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.get_expiration_time(): + if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes lease = lease.renew(new_expire_time) self._write_lease_record(f, leasenum, lease) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index e905b240d..38e380223 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -485,17 +485,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): return d def backdate_lease(self, sf, renew_secret, new_expire_time): - # ShareFile.renew_lease ignores attempts to back-date a lease (i.e. - # "renew" a lease with a new_expire_time that is older than what the - # current lease has), so we have to reach inside it. - for i,lease in enumerate(sf.get_leases()): - if lease.renew_secret == renew_secret: - lease = lease.renew(new_expire_time) - f = open(sf.home, 'rb+') - sf._write_lease_record(f, i, lease) - f.close() - return - raise IndexError("unable to renew non-existent lease") + sf.renew_lease(renew_secret, new_expire_time, allow_backdate=True) def test_expire_age(self): basedir = "storage/LeaseCrawler/expire_age" From 54bf271fbebdb7d63642cc4d86dcf9507ae839df Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 11:12:08 -0400 Subject: [PATCH 106/916] news fragment --- newsfragments/3833.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3833.minor diff --git a/newsfragments/3833.minor b/newsfragments/3833.minor new file mode 100644 index 000000000..e69de29bb From 34d2f74ede88107d6e30c927fdce704be06e2c3d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Oct 2021 11:12:17 -0400 Subject: [PATCH 107/916] Tell RTD how to install Sphinx. --- .readthedocs.yaml | 5 +++++ docs/requirements.txt | 4 ++++ newsfragments/3831.minor | 0 tox.ini | 7 +------ 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt create mode 100644 newsfragments/3831.minor diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..65b390f26 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,5 @@ +version: 2 + +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..39c4c20f0 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +sphinx +docutils<0.18 # https://github.com/sphinx-doc/sphinx/issues/9788 +recommonmark +sphinx_rtd_theme diff --git a/newsfragments/3831.minor b/newsfragments/3831.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index 61a811b71..38cee1f9f 100644 --- a/tox.ini +++ b/tox.ini @@ -217,13 +217,8 @@ commands = # your web browser. [testenv:docs] -# we pin docutils because of https://sourceforge.net/p/docutils/bugs/301/ -# which asserts when it reads links to .svg files (e.g. about.rst) deps = - sphinx - docutils==0.12 - recommonmark - sphinx_rtd_theme + -r docs/requirements.txt # normal install is not needed for docs, and slows things down skip_install = True commands = From 66845c9a1786778e145aab65e30fb0068e2f8245 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 11:12:20 -0400 Subject: [PATCH 108/916] Add ShareFile.is_valid_header and use it instead of manual header inspection --- src/allmydata/scripts/debug.py | 2 +- src/allmydata/storage/immutable.py | 15 +++++++++++++++ src/allmydata/storage/server.py | 2 +- src/allmydata/test/test_system.py | 8 ++++---- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 4d3f4cb21..71e1ccb41 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -795,7 +795,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) - elif struct.unpack(">L", prefix[:4]) == (1,): + elif ShareFile.is_valid_header(prefix): # immutable class ImmediateReadBucketProxy(ReadBucketProxy): diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 7712e568a..407116038 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -57,6 +57,21 @@ class ShareFile(object): LEASE_SIZE = struct.calcsize(">L32s32sL") sharetype = "immutable" + @classmethod + def is_valid_header(cls, header): + # (bytes) -> bool + """ + Determine if the given bytes constitute a valid header for this type of + container. + + :param header: Some bytes from the beginning of a container. + + :return: ``True`` if the bytes could belong to this container, + ``False`` otherwise. + """ + (version,) = struct.unpack(">L", header[:4]) + return version == 1 + def __init__(self, filename, max_size=None, create=False): """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ precondition((max_size is not None) or (not create), max_size, create) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 041783a4e..f339c579b 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -378,7 +378,7 @@ class StorageServer(service.MultiService, Referenceable): # note: if the share has been migrated, the renew_lease() # call will throw an exception, with information to help the # client update the lease. - elif header[:4] == struct.pack(">L", 1): + elif ShareFile.is_valid_header(header): sf = ShareFile(filename) else: continue # non-sharefile diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 087a1c634..72ce4b6ec 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -22,7 +22,7 @@ from twisted.trial import unittest from twisted.internet import defer from allmydata import uri -from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.mutable import ShareFile, MutableShareFile from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode @@ -1290,9 +1290,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # are sharefiles here filename = os.path.join(dirpath, filenames[0]) # peek at the magic to see if it is a chk share - magic = open(filename, "rb").read(4) - if magic == b'\x00\x00\x00\x01': - break + with open(filename, "rb") as f: + if ShareFile.is_valid_header(f.read(32)): + break else: self.fail("unable to find any uri_extension files in %r" % self.basedir) From 1b46ac7a241e719cc0d7ddc4b66fa9fcdca5992d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 11:38:18 -0400 Subject: [PATCH 109/916] add MutableShareFile.is_valid_header and use it --- src/allmydata/scripts/debug.py | 287 ++++++++++++++--------------- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/mutable.py | 18 +- src/allmydata/storage/server.py | 2 +- src/allmydata/storage/shares.py | 3 +- src/allmydata/test/common.py | 2 +- src/allmydata/test/test_system.py | 3 +- 7 files changed, 163 insertions(+), 154 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 71e1ccb41..ab48b0fd0 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -15,15 +15,22 @@ try: except ImportError: pass - -# do not import any allmydata modules at this level. Do that from inside -# individual functions instead. import struct, time, os, sys + from twisted.python import usage, failure from twisted.internet import defer from foolscap.logging import cli as foolscap_cli -from allmydata.scripts.common import BaseOptions +from allmydata.scripts.common import BaseOptions +from allmydata import uri +from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.immutable import ShareFile +from allmydata.mutable.layout import unpack_share +from allmydata.mutable.layout import MDMFSlotReadProxy +from allmydata.mutable.common import NeedMoreDataError +from allmydata.immutable.layout import ReadBucketProxy +from allmydata.util import base32 +from allmydata.util.encodingutil import quote_output class DumpOptions(BaseOptions): def getSynopsis(self): @@ -56,13 +63,11 @@ def dump_share(options): # check the version, to see if we have a mutable or immutable share print("share filename: %s" % quote_output(options['filename']), file=out) - f = open(options['filename'], "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - return dump_mutable_share(options) - # otherwise assume it's immutable - return dump_immutable_share(options) + with open(options['filename'], "rb") as f: + if MutableShareFile.is_valid_header(f.read(32)): + return dump_mutable_share(options) + # otherwise assume it's immutable + return dump_immutable_share(options) def dump_immutable_share(options): from allmydata.storage.immutable import ShareFile @@ -712,125 +717,115 @@ def call(c, *args, **kwargs): return results[0] def describe_share(abs_sharefile, si_s, shnum_s, now, out): - from allmydata import uri - from allmydata.storage.mutable import MutableShareFile - from allmydata.storage.immutable import ShareFile - from allmydata.mutable.layout import unpack_share - from allmydata.mutable.common import NeedMoreDataError - from allmydata.immutable.layout import ReadBucketProxy - from allmydata.util import base32 - from allmydata.util.encodingutil import quote_output - import struct - - f = open(abs_sharefile, "rb") - prefix = f.read(32) - - if prefix == MutableShareFile.MAGIC: - # mutable share - m = MutableShareFile(abs_sharefile) - WE, nodeid = m._read_write_enabler_and_nodeid(f) - data_length = m._read_data_length(f) - expiration_time = min( [lease.get_expiration_time() - for (i,lease) in m._enumerate_leases(f)] ) - expiration = max(0, expiration_time - now) - - share_type = "unknown" - f.seek(m.DATA_OFFSET) - version = f.read(1) - if version == b"\x00": - # this slot contains an SMDF share - share_type = "SDMF" - elif version == b"\x01": - share_type = "MDMF" - - if share_type == "SDMF": - f.seek(m.DATA_OFFSET) - data = f.read(min(data_length, 2000)) - - try: - pieces = unpack_share(data) - except NeedMoreDataError as e: - # retry once with the larger size - size = e.needed_bytes - f.seek(m.DATA_OFFSET) - data = f.read(min(data_length, size)) - pieces = unpack_share(data) - (seqnum, root_hash, IV, k, N, segsize, datalen, - pubkey, signature, share_hash_chain, block_hash_tree, - share_data, enc_privkey) = pieces - - print("SDMF %s %d/%d %d #%d:%s %d %s" % \ - (si_s, k, N, datalen, - seqnum, str(base32.b2a(root_hash), "utf-8"), - expiration, quote_output(abs_sharefile)), file=out) - elif share_type == "MDMF": - from allmydata.mutable.layout import MDMFSlotReadProxy - fake_shnum = 0 - # TODO: factor this out with dump_MDMF_share() - class ShareDumper(MDMFSlotReadProxy): - def _read(self, readvs, force_remote=False, queue=False): - data = [] - for (where,length) in readvs: - f.seek(m.DATA_OFFSET+where) - data.append(f.read(length)) - return defer.succeed({fake_shnum: data}) - - p = ShareDumper(None, "fake-si", fake_shnum) - def extract(func): - stash = [] - # these methods return Deferreds, but we happen to know that - # they run synchronously when not actually talking to a - # remote server - d = func() - d.addCallback(stash.append) - return stash[0] - - verinfo = extract(p.get_verinfo) - (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, - offsets) = verinfo - print("MDMF %s %d/%d %d #%d:%s %d %s" % \ - (si_s, k, N, datalen, - seqnum, str(base32.b2a(root_hash), "utf-8"), - expiration, quote_output(abs_sharefile)), file=out) + with open(abs_sharefile, "rb") as f: + prefix = f.read(32) + if MutableShareFile.is_valid_header(prefix): + _describe_mutable_share(abs_sharefile, f, now, si_s, out) + elif ShareFile.is_valid_header(prefix): + _describe_immutable_share(abs_sharefile, now, si_s, out) else: - print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) + print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) - elif ShareFile.is_valid_header(prefix): - # immutable +def _describe_mutable_share(abs_sharefile, f, now, si_s, out): + # mutable share + m = MutableShareFile(abs_sharefile) + WE, nodeid = m._read_write_enabler_and_nodeid(f) + data_length = m._read_data_length(f) + expiration_time = min( [lease.get_expiration_time() + for (i,lease) in m._enumerate_leases(f)] ) + expiration = max(0, expiration_time - now) - class ImmediateReadBucketProxy(ReadBucketProxy): - def __init__(self, sf): - self.sf = sf - ReadBucketProxy.__init__(self, None, None, "") - def __repr__(self): - return "" - def _read(self, offset, size): - return defer.succeed(sf.read_share_data(offset, size)) + share_type = "unknown" + f.seek(m.DATA_OFFSET) + version = f.read(1) + if version == b"\x00": + # this slot contains an SMDF share + share_type = "SDMF" + elif version == b"\x01": + share_type = "MDMF" - # use a ReadBucketProxy to parse the bucket and find the uri extension - sf = ShareFile(abs_sharefile) - bp = ImmediateReadBucketProxy(sf) + if share_type == "SDMF": + f.seek(m.DATA_OFFSET) + data = f.read(min(data_length, 2000)) - expiration_time = min( [lease.get_expiration_time() - for lease in sf.get_leases()] ) - expiration = max(0, expiration_time - now) + try: + pieces = unpack_share(data) + except NeedMoreDataError as e: + # retry once with the larger size + size = e.needed_bytes + f.seek(m.DATA_OFFSET) + data = f.read(min(data_length, size)) + pieces = unpack_share(data) + (seqnum, root_hash, IV, k, N, segsize, datalen, + pubkey, signature, share_hash_chain, block_hash_tree, + share_data, enc_privkey) = pieces - UEB_data = call(bp.get_uri_extension) - unpacked = uri.unpack_extension_readable(UEB_data) + print("SDMF %s %d/%d %d #%d:%s %d %s" % \ + (si_s, k, N, datalen, + seqnum, str(base32.b2a(root_hash), "utf-8"), + expiration, quote_output(abs_sharefile)), file=out) + elif share_type == "MDMF": + fake_shnum = 0 + # TODO: factor this out with dump_MDMF_share() + class ShareDumper(MDMFSlotReadProxy): + def _read(self, readvs, force_remote=False, queue=False): + data = [] + for (where,length) in readvs: + f.seek(m.DATA_OFFSET+where) + data.append(f.read(length)) + return defer.succeed({fake_shnum: data}) - k = unpacked["needed_shares"] - N = unpacked["total_shares"] - filesize = unpacked["size"] - ueb_hash = unpacked["UEB_hash"] - - print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, - str(ueb_hash, "utf-8"), expiration, - quote_output(abs_sharefile)), file=out) + p = ShareDumper(None, "fake-si", fake_shnum) + def extract(func): + stash = [] + # these methods return Deferreds, but we happen to know that + # they run synchronously when not actually talking to a + # remote server + d = func() + d.addCallback(stash.append) + return stash[0] + verinfo = extract(p.get_verinfo) + (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, + offsets) = verinfo + print("MDMF %s %d/%d %d #%d:%s %d %s" % \ + (si_s, k, N, datalen, + seqnum, str(base32.b2a(root_hash), "utf-8"), + expiration, quote_output(abs_sharefile)), file=out) else: - print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) + print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) + + +def _describe_immutable_share(abs_sharefile, now, si_s, out): + class ImmediateReadBucketProxy(ReadBucketProxy): + def __init__(self, sf): + self.sf = sf + ReadBucketProxy.__init__(self, None, None, "") + def __repr__(self): + return "" + def _read(self, offset, size): + return defer.succeed(sf.read_share_data(offset, size)) + + # use a ReadBucketProxy to parse the bucket and find the uri extension + sf = ShareFile(abs_sharefile) + bp = ImmediateReadBucketProxy(sf) + + expiration_time = min( [lease.get_expiration_time() + for lease in sf.get_leases()] ) + expiration = max(0, expiration_time - now) + + UEB_data = call(bp.get_uri_extension) + unpacked = uri.unpack_extension_readable(UEB_data) + + k = unpacked["needed_shares"] + N = unpacked["total_shares"] + filesize = unpacked["size"] + ueb_hash = unpacked["UEB_hash"] + + print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, + str(ueb_hash, "utf-8"), expiration, + quote_output(abs_sharefile)), file=out) - f.close() def catalog_shares(options): from allmydata.util.encodingutil import listdir_unicode, quote_output @@ -933,34 +928,34 @@ def corrupt_share(options): f.write(d) f.close() - f = open(fn, "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - # mutable - m = MutableShareFile(fn) - f = open(fn, "rb") - f.seek(m.DATA_OFFSET) - data = f.read(2000) - # make sure this slot contains an SMDF share - assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" - f.close() + with open(fn, "rb") as f: + prefix = f.read(32) - (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, - ig_datalen, offsets) = unpack_header(data) + if MutableShareFile.is_valid_header(prefix): + # mutable + m = MutableShareFile(fn) + f = open(fn, "rb") + f.seek(m.DATA_OFFSET) + data = f.read(2000) + # make sure this slot contains an SMDF share + assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" + f.close() - assert version == 0, "we only handle v0 SDMF files" - start = m.DATA_OFFSET + offsets["share_data"] - end = m.DATA_OFFSET + offsets["enc_privkey"] - flip_bit(start, end) - else: - # otherwise assume it's immutable - f = ShareFile(fn) - bp = ReadBucketProxy(None, None, '') - offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) - start = f._data_offset + offsets["data"] - end = f._data_offset + offsets["plaintext_hash_tree"] - flip_bit(start, end) + (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, + ig_datalen, offsets) = unpack_header(data) + + assert version == 0, "we only handle v0 SDMF files" + start = m.DATA_OFFSET + offsets["share_data"] + end = m.DATA_OFFSET + offsets["enc_privkey"] + flip_bit(start, end) + else: + # otherwise assume it's immutable + f = ShareFile(fn) + bp = ReadBucketProxy(None, None, '') + offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) + start = f._data_offset + offsets["data"] + end = f._data_offset + offsets["plaintext_hash_tree"] + flip_bit(start, end) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 407116038..24465c1ed 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -59,7 +59,7 @@ class ShareFile(object): @classmethod def is_valid_header(cls, header): - # (bytes) -> bool + # type: (bytes) -> bool """ Determine if the given bytes constitute a valid header for this type of container. diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index de840b89a..1b29b4a65 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -67,6 +67,20 @@ class MutableShareFile(object): MAX_SIZE = MAX_MUTABLE_SHARE_SIZE # TODO: decide upon a policy for max share size + @classmethod + def is_valid_header(cls, header): + # type: (bytes) -> bool + """ + Determine if the given bytes constitute a valid header for this type of + container. + + :param header: Some bytes from the beginning of a container. + + :return: ``True`` if the bytes could belong to this container, + ``False`` otherwise. + """ + return header.startswith(cls.MAGIC) + def __init__(self, filename, parent=None): self.home = filename if os.path.exists(self.home): @@ -77,7 +91,7 @@ class MutableShareFile(object): write_enabler_nodeid, write_enabler, data_length, extra_least_offset) = \ struct.unpack(">32s20s32sQQ", data) - if magic != self.MAGIC: + if not self.is_valid_header(data): msg = "sharefile %s had magic '%r' but we wanted '%r'" % \ (filename, magic, self.MAGIC) raise UnknownMutableContainerVersionError(msg) @@ -388,7 +402,7 @@ class MutableShareFile(object): write_enabler_nodeid, write_enabler, data_length, extra_least_offset) = \ struct.unpack(">32s20s32sQQ", data) - assert magic == self.MAGIC + assert self.is_valid_header(data) return (write_enabler, write_enabler_nodeid) def readv(self, readv): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f339c579b..0f30dad6a 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -373,7 +373,7 @@ class StorageServer(service.MultiService, Referenceable): for shnum, filename in self._get_bucket_shares(storage_index): with open(filename, 'rb') as f: header = f.read(32) - if header[:32] == MutableShareFile.MAGIC: + if MutableShareFile.is_valid_header(header): sf = MutableShareFile(filename, self) # note: if the share has been migrated, the renew_lease() # call will throw an exception, with information to help the diff --git a/src/allmydata/storage/shares.py b/src/allmydata/storage/shares.py index ec6c0a501..59e7b1539 100644 --- a/src/allmydata/storage/shares.py +++ b/src/allmydata/storage/shares.py @@ -17,8 +17,7 @@ from allmydata.storage.immutable import ShareFile def get_share_file(filename): with open(filename, "rb") as f: prefix = f.read(32) - if prefix == MutableShareFile.MAGIC: + if MutableShareFile.is_valid_header(prefix): return MutableShareFile(filename) # otherwise assume it's immutable return ShareFile(filename) - diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 0f2dc7c62..97368ee92 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1068,7 +1068,7 @@ def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False): def _corrupt_mutable_share_data(data, debug=False): prefix = data[:32] - assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) + assert MutableShareFile.is_valid_header(prefix), "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) data_offset = MutableShareFile.DATA_OFFSET sharetype = data[data_offset:data_offset+1] assert sharetype == b"\x00", "non-SDMF mutable shares not supported" diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 72ce4b6ec..d859a0e00 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -22,7 +22,8 @@ from twisted.trial import unittest from twisted.internet import defer from allmydata import uri -from allmydata.storage.mutable import ShareFile, MutableShareFile +from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.immutable import ShareFile from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode From 8d202a4018bc5121800ea5551fd925d8432b2996 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:37:37 -0400 Subject: [PATCH 110/916] news fragment --- newsfragments/3835.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3835.minor diff --git a/newsfragments/3835.minor b/newsfragments/3835.minor new file mode 100644 index 000000000..e69de29bb From d0ee17d99efff05738f0eb3140c3a5c947c20b5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:39:01 -0400 Subject: [PATCH 111/916] some docstrings --- src/allmydata/test/no_network.py | 26 +++++++++++++++ src/allmydata/test/test_download.py | 52 ++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 7a84580bf..aa41ab6bc 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -479,6 +479,18 @@ class GridTestMixin(object): def set_up_grid(self, num_clients=1, num_servers=10, client_config_hooks={}, oneshare=False): + """ + Create a Tahoe-LAFS storage grid. + + :param num_clients: See ``NoNetworkGrid`` + :param num_servers: See `NoNetworkGrid`` + :param client_config_hooks: See ``NoNetworkGrid`` + + :param bool oneshare: If ``True`` then the first client node is + configured with ``n == k == happy == 1``. + + :return: ``None`` + """ # self.basedir must be set port_assigner = SameProcessStreamEndpointAssigner() port_assigner.setUp() @@ -557,6 +569,15 @@ class GridTestMixin(object): return sorted(shares) def copy_shares(self, uri): + # type: (bytes) -> Dict[bytes, bytes] + """ + Read all of the share files for the given capability from the storage area + of the storage servers created by ``set_up_grid``. + + :param bytes uri: A Tahoe-LAFS data capability. + + :return: A ``dict`` mapping share file names to share file contents. + """ shares = {} for (shnum, serverid, sharefile) in self.find_uri_shares(uri): with open(sharefile, "rb") as f: @@ -601,6 +622,11 @@ class GridTestMixin(object): f.write(corruptdata) def corrupt_all_shares(self, uri, corruptor, debug=False): + # type: (bytes, Callable[[bytes, bool], bytes] -> bytes), bool) -> None + """ + Apply ``corruptor`` to the contents of all share files associated with a + given capability and replace the share file contents with its result. + """ for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): with open(i_sharefile, "rb") as f: sharedata = f.read() diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index d61942839..6b8dc6a31 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -951,12 +951,52 @@ class Corruption(_Base, unittest.TestCase): self.corrupt_shares_numbered(imm_uri, [2], _corruptor) def _corrupt_set(self, ign, imm_uri, which, newvalue): + # type: (Any, bytes, int, int) -> None + """ + Replace a single byte share file number 2 for the given capability with a + new byte. + + :param imm_uri: Corrupt share number 2 belonging to this capability. + :param which: The byte position to replace. + :param newvalue: The new byte value to set in the share. + """ log.msg("corrupt %d" % which) def _corruptor(s, debug=False): return s[:which] + bchr(newvalue) + s[which+1:] self.corrupt_shares_numbered(imm_uri, [2], _corruptor) def test_each_byte(self): + """ + Test share selection behavior of the downloader in the face of certain + kinds of data corruption. + + 1. upload a small share to the no-network grid + 2. read all of the resulting share files out of the no-network storage servers + 3. for each of + + a. each byte of the share file version field + b. each byte of the immutable share version field + c. each byte of the immutable share data offset field + d. the most significant byte of the block_shares offset field + e. one of the bytes of one of the merkle trees + f. one of the bytes of the share hashes list + + i. flip the least significant bit in all of the the share files + ii. perform the download/check/restore process + + 4. add 2 ** 24 to the share file version number + 5. perform the download/check/restore process + + 6. add 2 ** 24 to the share version number + 7. perform the download/check/restore process + + The download/check/restore process is: + + 1. attempt to download the data + 2. assert that the recovered plaintext is correct + 3. assert that only the "correct" share numbers were used to reconstruct the plaintext + 4. restore all of the share files to their pristine condition + """ # Setting catalog_detection=True performs an exhaustive test of the # Downloader's response to corruption in the lsb of each byte of the # 2070-byte share, with two goals: make sure we tolerate all forms of @@ -1145,8 +1185,18 @@ class Corruption(_Base, unittest.TestCase): return d def _corrupt_flip_all(self, ign, imm_uri, which): + # type: (Any, bytes, int) -> None + """ + Flip the least significant bit at a given byte position in all share files + for the given capability. + """ def _corruptor(s, debug=False): - return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:] + # type: (bytes, bool) -> bytes + before_corruption = s[:which] + after_corruption = s[which+1:] + original_byte = s[which:which+1] + corrupt_byte = bchr(ord(original_byte) ^ 0x01) + return b"".join([before_corruption, corrupt_byte, after_corruption]) self.corrupt_all_shares(imm_uri, _corruptor) class DownloadV2(_Base, unittest.TestCase): From 8cb1f4f57cc6591e46573bd214ef0a7c43ad2c04 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 14:25:24 -0400 Subject: [PATCH 112/916] news fragment --- newsfragments/3527.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3527.minor diff --git a/newsfragments/3527.minor b/newsfragments/3527.minor new file mode 100644 index 000000000..e69de29bb From 54d80222c9cdf6662510f67d15de0cf7494a723e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 14:34:47 -0400 Subject: [PATCH 113/916] switch to monkey-patching from other sources This is not much of an improvement to the tests themselves, unfortunately. However, it does get us one step closer to dropping `mock` as a dependency. --- src/allmydata/test/cli/test_create.py | 144 ++++++++++++++++---------- src/allmydata/test/common.py | 25 +++++ 2 files changed, 115 insertions(+), 54 deletions(-) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 282f26163..609888fb3 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -11,16 +11,24 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os -import mock + +try: + from typing import Any, List, Tuple +except ImportError: + pass + from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python import usage from allmydata.util import configutil +from allmydata.util import tor_provider, i2p_provider from ..common_util import run_cli, parse_cli +from ..common import ( + disable_modules, +) from ...scripts import create_node from ... import client - def read_config(basedir): tahoe_cfg = os.path.join(basedir, "tahoe.cfg") config = configutil.get_config(tahoe_cfg) @@ -105,11 +113,12 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_hide_ip_no_i2p_txtorcon(self): - # hmm, I must be doing something weird, these don't work as - # @mock.patch decorators for some reason - txi2p = mock.patch('allmydata.util.i2p_provider._import_txi2p', return_value=None) - txtorcon = mock.patch('allmydata.util.tor_provider._import_txtorcon', return_value=None) - with txi2p, txtorcon: + """ + The ``create-client`` sub-command tells the user to install the necessary + dependencies if they have neither tor nor i2p support installed and + they request network location privacy with the ``--hide-ip`` flag. + """ + with disable_modules("txi2p", "txtorcon"): basedir = self.mktemp() rc, out, err = yield run_cli("create-client", "--hide-ip", basedir) self.assertTrue(rc != 0, out) @@ -118,8 +127,7 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_i2p_option_no_txi2p(self): - txi2p = mock.patch('allmydata.util.i2p_provider._import_txi2p', return_value=None) - with txi2p: + with disable_modules("txi2p"): basedir = self.mktemp() rc, out, err = yield run_cli("create-node", "--listen=i2p", "--i2p-launch", basedir) self.assertTrue(rc != 0) @@ -127,8 +135,7 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_tor_option_no_txtorcon(self): - txtorcon = mock.patch('allmydata.util.tor_provider._import_txtorcon', return_value=None) - with txtorcon: + with disable_modules("txtorcon"): basedir = self.mktemp() rc, out, err = yield run_cli("create-node", "--listen=tor", "--tor-launch", basedir) self.assertTrue(rc != 0) @@ -145,9 +152,7 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_hide_ip_no_txtorcon(self): - txtorcon = mock.patch('allmydata.util.tor_provider._import_txtorcon', - return_value=None) - with txtorcon: + with disable_modules("txtorcon"): basedir = self.mktemp() rc, out, err = yield run_cli("create-client", "--hide-ip", basedir) self.assertEqual(0, rc) @@ -295,11 +300,10 @@ class Config(unittest.TestCase): def test_node_slow_tor(self): basedir = self.mktemp() d = defer.Deferred() - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=d): - d2 = run_cli("create-node", "--listen=tor", basedir) - d.callback(({}, "port", "location")) - rc, out, err = yield d2 + self.patch(tor_provider, "create_config", lambda *a, **kw: d) + d2 = run_cli("create-node", "--listen=tor", basedir) + d.callback(({}, "port", "location")) + rc, out, err = yield d2 self.assertEqual(rc, 0) self.assertIn("Node created", out) self.assertEqual(err, "") @@ -308,11 +312,10 @@ class Config(unittest.TestCase): def test_node_slow_i2p(self): basedir = self.mktemp() d = defer.Deferred() - with mock.patch("allmydata.util.i2p_provider.create_config", - return_value=d): - d2 = run_cli("create-node", "--listen=i2p", basedir) - d.callback(({}, "port", "location")) - rc, out, err = yield d2 + self.patch(i2p_provider, "create_config", lambda *a, **kw: d) + d2 = run_cli("create-node", "--listen=i2p", basedir) + d.callback(({}, "port", "location")) + rc, out, err = yield d2 self.assertEqual(rc, 0) self.assertIn("Node created", out) self.assertEqual(err, "") @@ -353,6 +356,27 @@ class Config(unittest.TestCase): self.assertIn("is not empty", err) self.assertIn("To avoid clobbering anything, I am going to quit now", err) +def fake_config(testcase, module, result): + # type: (unittest.TestCase, Any, Any) -> List[Tuple] + """ + Monkey-patch a fake configuration function into the given module. + + :param testcase: The test case to use to do the monkey-patching. + + :param module: The module into which to patch the fake function. + + :param result: The return value for the fake function. + + :return: A list of tuples of the arguments the fake function was called + with. + """ + calls = [] + def fake_config(reactor, cli_config): + calls.append((reactor, cli_config)) + return result + testcase.patch(module, "create_config", fake_config) + return calls + class Tor(unittest.TestCase): def test_default(self): basedir = self.mktemp() @@ -360,12 +384,14 @@ class Tor(unittest.TestCase): tor_port = "ghi" tor_location = "jkl" config_d = defer.succeed( (tor_config, tor_port, tor_location) ) - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=config_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=tor", basedir)) - self.assertEqual(len(co.mock_calls), 1) - args = co.mock_calls[0][1] + + calls = fake_config(self, tor_provider, config_d) + rc, out, err = self.successResultOf( + run_cli("create-node", "--listen=tor", basedir), + ) + + self.assertEqual(len(calls), 1) + args = calls[0] self.assertIdentical(args[0], reactor) self.assertIsInstance(args[1], create_node.CreateNodeOptions) self.assertEqual(args[1]["listen"], "tor") @@ -380,12 +406,15 @@ class Tor(unittest.TestCase): tor_port = "ghi" tor_location = "jkl" config_d = defer.succeed( (tor_config, tor_port, tor_location) ) - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=config_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=tor", "--tor-launch", - basedir)) - args = co.mock_calls[0][1] + + calls = fake_config(self, tor_provider, config_d) + rc, out, err = self.successResultOf( + run_cli( + "create-node", "--listen=tor", "--tor-launch", + basedir, + ), + ) + args = calls[0] self.assertEqual(args[1]["listen"], "tor") self.assertEqual(args[1]["tor-launch"], True) self.assertEqual(args[1]["tor-control-port"], None) @@ -396,12 +425,15 @@ class Tor(unittest.TestCase): tor_port = "ghi" tor_location = "jkl" config_d = defer.succeed( (tor_config, tor_port, tor_location) ) - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=config_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=tor", "--tor-control-port=mno", - basedir)) - args = co.mock_calls[0][1] + + calls = fake_config(self, tor_provider, config_d) + rc, out, err = self.successResultOf( + run_cli( + "create-node", "--listen=tor", "--tor-control-port=mno", + basedir, + ), + ) + args = calls[0] self.assertEqual(args[1]["listen"], "tor") self.assertEqual(args[1]["tor-launch"], False) self.assertEqual(args[1]["tor-control-port"], "mno") @@ -434,12 +466,13 @@ class I2P(unittest.TestCase): i2p_port = "ghi" i2p_location = "jkl" dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) ) - with mock.patch("allmydata.util.i2p_provider.create_config", - return_value=dest_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=i2p", basedir)) - self.assertEqual(len(co.mock_calls), 1) - args = co.mock_calls[0][1] + + calls = fake_config(self, i2p_provider, dest_d) + rc, out, err = self.successResultOf( + run_cli("create-node", "--listen=i2p", basedir), + ) + self.assertEqual(len(calls), 1) + args = calls[0] self.assertIdentical(args[0], reactor) self.assertIsInstance(args[1], create_node.CreateNodeOptions) self.assertEqual(args[1]["listen"], "i2p") @@ -461,12 +494,15 @@ class I2P(unittest.TestCase): i2p_port = "ghi" i2p_location = "jkl" dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) ) - with mock.patch("allmydata.util.i2p_provider.create_config", - return_value=dest_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=i2p", "--i2p-sam-port=mno", - basedir)) - args = co.mock_calls[0][1] + + calls = fake_config(self, i2p_provider, dest_d) + rc, out, err = self.successResultOf( + run_cli( + "create-node", "--listen=i2p", "--i2p-sam-port=mno", + basedir, + ), + ) + args = calls[0] self.assertEqual(args[1]["listen"], "i2p") self.assertEqual(args[1]["i2p-launch"], False) self.assertEqual(args[1]["i2p-sam-port"], "mno") diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 0f2dc7c62..2e6da9801 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -26,8 +26,14 @@ __all__ = [ "PIPE", ] +try: + from typing import Tuple, ContextManager +except ImportError: + pass + import sys import os, random, struct +from contextlib import contextmanager import six import tempfile from tempfile import mktemp @@ -1213,6 +1219,25 @@ class ConstantAddresses(object): raise Exception("{!r} has no client endpoint.") return self._handler +@contextmanager +def disable_modules(*names): + # type: (Tuple[str]) -> ContextManager + """ + A context manager which makes modules appear to be missing while it is + active. + + :param *names: The names of the modules to disappear. + """ + missing = object() + modules = list(sys.modules.get(n, missing) for n in names) + for n in names: + sys.modules[n] = None + yield + for n, original in zip(names, modules): + if original is missing: + del sys.modules[n] + else: + sys.modules[n] = original class _TestCaseMixin(object): """ From 8d5727977b9a1a7865954db30f9d4771518b97c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 14:47:42 -0400 Subject: [PATCH 114/916] it doesn't typecheck, nevermind --- src/allmydata/test/common.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 2e6da9801..8e97fa598 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -26,11 +26,6 @@ __all__ = [ "PIPE", ] -try: - from typing import Tuple, ContextManager -except ImportError: - pass - import sys import os, random, struct from contextlib import contextmanager @@ -1221,7 +1216,6 @@ class ConstantAddresses(object): @contextmanager def disable_modules(*names): - # type: (Tuple[str]) -> ContextManager """ A context manager which makes modules appear to be missing while it is active. From f8655f149bb0754013adc985a6041738f18327f2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 15:04:19 -0400 Subject: [PATCH 115/916] fix the type annotations and such --- src/allmydata/test/no_network.py | 9 +++++++-- src/allmydata/test/test_download.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index aa41ab6bc..b9fa99005 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -25,6 +25,11 @@ if PY2: from past.builtins import unicode from six import ensure_text +try: + from typing import Dict, Callable +except ImportError: + pass + import os from base64 import b32encode from functools import ( @@ -622,7 +627,7 @@ class GridTestMixin(object): f.write(corruptdata) def corrupt_all_shares(self, uri, corruptor, debug=False): - # type: (bytes, Callable[[bytes, bool], bytes] -> bytes), bool) -> None + # type: (bytes, Callable[[bytes, bool], bytes], bool) -> None """ Apply ``corruptor`` to the contents of all share files associated with a given capability and replace the share file contents with its result. @@ -630,7 +635,7 @@ class GridTestMixin(object): for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): with open(i_sharefile, "rb") as f: sharedata = f.read() - corruptdata = corruptor(sharedata, debug=debug) + corruptdata = corruptor(sharedata, debug) with open(i_sharefile, "wb") as f: f.write(corruptdata) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 6b8dc6a31..aeea9642e 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -14,6 +14,11 @@ if PY2: # a previous run. This asserts that the current code is capable of decoding # shares from a previous version. +try: + from typing import Any +except ImportError: + pass + import six import os from twisted.trial import unittest From 78dbe7699403dbe38f94d574367f5d5e95916f4a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 15:20:44 -0400 Subject: [PATCH 116/916] remove unused import --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0f30dad6a..3e2d3b5c6 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -14,7 +14,7 @@ if PY2: else: from typing import Dict -import os, re, struct, time +import os, re, time import six from foolscap.api import Referenceable From 8b976b441e793f45b50e5d5ebcb4314beba889ee Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:05:34 -0400 Subject: [PATCH 117/916] add LeaseInfo.is_renew_secret and use it --- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/lease.py | 12 +++++++++ src/allmydata/storage/mutable.py | 2 +- src/allmydata/test/test_storage.py | 39 ++++++++++++++++++++++-------- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 24465c1ed..c9b8995b5 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -180,7 +180,7 @@ class ShareFile(object): secret. """ for i,lease in enumerate(self.get_leases()): - if timing_safe_compare(lease.renew_secret, renew_secret): + if lease.is_renew_secret(renew_secret): # yup. See if we need to update the owner time. if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 17683a888..2132048ce 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -15,6 +15,8 @@ import struct, time import attr +from allmydata.util.hashutil import timing_safe_compare + @attr.s(frozen=True) class LeaseInfo(object): """ @@ -68,6 +70,16 @@ class LeaseInfo(object): _expiration_time=new_expire_time, ) + def is_renew_secret(self, candidate_secret): + # type: (bytes) -> bool + """ + Check a string to see if it is the correct renew secret. + + :return: ``True`` if it is the correct renew secret, ``False`` + otherwise. + """ + return timing_safe_compare(self.renew_secret, candidate_secret) + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self._expiration_time - 31*24*60*60 diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 1b29b4a65..017f2dbb7 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -327,7 +327,7 @@ class MutableShareFile(object): accepting_nodeids = set() with open(self.home, 'rb+') as f: for (leasenum,lease) in self._enumerate_leases(f): - if timing_safe_compare(lease.renew_secret, renew_secret): + if lease.is_renew_secret(renew_secret): # yup. See if we need to update the owner time. if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 8123be2c5..005309f87 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -755,28 +755,28 @@ class Server(unittest.TestCase): # Create a bucket: rs0, cs0 = self.create_bucket_5_shares(ss, b"si0") - leases = list(ss.get_leases(b"si0")) - self.failUnlessEqual(len(leases), 1) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0])) + (lease,) = ss.get_leases(b"si0") + self.assertTrue(lease.is_renew_secret(rs0)) rs1, cs1 = self.create_bucket_5_shares(ss, b"si1") # take out a second lease on si1 rs2, cs2 = self.create_bucket_5_shares(ss, b"si1", 5, 0) - leases = list(ss.get_leases(b"si1")) - self.failUnlessEqual(len(leases), 2) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2])) + (lease1, lease2) = ss.get_leases(b"si1") + self.assertTrue(lease1.is_renew_secret(rs1)) + self.assertTrue(lease2.is_renew_secret(rs2)) # and a third lease, using add-lease rs2a,cs2a = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) ss.remote_add_lease(b"si1", rs2a, cs2a) - leases = list(ss.get_leases(b"si1")) - self.failUnlessEqual(len(leases), 3) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2, rs2a])) + (lease1, lease2, lease3) = ss.get_leases(b"si1") + self.assertTrue(lease1.is_renew_secret(rs1)) + self.assertTrue(lease2.is_renew_secret(rs2)) + self.assertTrue(lease3.is_renew_secret(rs2a)) # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None) + self.assertIsNone(ss.remote_add_lease(b"si18", b"", b"")) # check that si0 is readable readers = ss.remote_get_buckets(b"si0") @@ -3028,3 +3028,22 @@ class ShareFileTests(unittest.TestCase): sf = self.get_sharefile() with self.assertRaises(IndexError): sf.cancel_lease(b"garbage") + + def test_renew_secret(self): + """ + A lease loaded from a share file can have its renew secret verified. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + expiration_time = 2 ** 31 + + sf = self.get_sharefile() + lease = LeaseInfo( + owner_num=0, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + expiration_time=expiration_time, + ) + sf.add_lease(lease) + (loaded_lease,) = sf.get_leases() + self.assertTrue(loaded_lease.is_renew_secret(renew_secret)) From b5f882ffa60574f193a18e70e3c310077a2f097e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:21:22 -0400 Subject: [PATCH 118/916] introduce and use LeaseInfo.is_cancel_secret --- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/lease.py | 10 ++++++++++ src/allmydata/storage/mutable.py | 2 +- src/allmydata/test/test_storage.py | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index c9b8995b5..4f6a1c9c7 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -209,7 +209,7 @@ class ShareFile(object): leases = list(self.get_leases()) num_leases_removed = 0 for i,lease in enumerate(leases): - if timing_safe_compare(lease.cancel_secret, cancel_secret): + if lease.is_cancel_secret(cancel_secret): leases[i] = None num_leases_removed += 1 if not num_leases_removed: diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 2132048ce..ff96ebaf4 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -80,6 +80,16 @@ class LeaseInfo(object): """ return timing_safe_compare(self.renew_secret, candidate_secret) + def is_cancel_secret(self, candidate_secret): + # type: (bytes) -> bool + """ + Check a string to see if it is the correct cancel secret. + + :return: ``True`` if it is the correct cancel secret, ``False`` + otherwise. + """ + return timing_safe_compare(self.cancel_secret, candidate_secret) + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self._expiration_time - 31*24*60*60 diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 017f2dbb7..9480a3c03 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -371,7 +371,7 @@ class MutableShareFile(object): with open(self.home, 'rb+') as f: for (leasenum,lease) in self._enumerate_leases(f): accepting_nodeids.add(lease.nodeid) - if timing_safe_compare(lease.cancel_secret, cancel_secret): + if lease.is_cancel_secret(cancel_secret): self._write_lease_record(f, leasenum, blank_lease) modified += 1 else: diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 005309f87..aac40362c 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3047,3 +3047,22 @@ class ShareFileTests(unittest.TestCase): sf.add_lease(lease) (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_renew_secret(renew_secret)) + + def test_cancel_secret(self): + """ + A lease loaded from a share file can have its cancel secret verified. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + expiration_time = 2 ** 31 + + sf = self.get_sharefile() + lease = LeaseInfo( + owner_num=0, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + expiration_time=expiration_time, + ) + sf.add_lease(lease) + (loaded_lease,) = sf.get_leases() + self.assertTrue(loaded_lease.is_cancel_secret(cancel_secret)) From 696a260ddfc02be35a63ed1446eda0b5434cc86f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 29 Oct 2021 09:00:38 -0400 Subject: [PATCH 119/916] news fragment --- newsfragments/3836.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3836.minor diff --git a/newsfragments/3836.minor b/newsfragments/3836.minor new file mode 100644 index 000000000..e69de29bb From 892b4683654cd1281be8feeaa65a2ef946ed4f5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 29 Oct 2021 09:03:37 -0400 Subject: [PATCH 120/916] use the port assigner to assign a port for the main tub --- src/allmydata/test/common_system.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 9d14c8642..874c7f6ba 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -672,11 +672,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): """ iv_dir = self.getdir("introducer") if not os.path.isdir(iv_dir): - _, port_endpoint = self.port_assigner.assign(reactor) + _, web_port_endpoint = self.port_assigner.assign(reactor) + main_location_hint, main_port_endpoint = self.port_assigner.assign(reactor) introducer_config = ( u"[node]\n" u"nickname = introducer \N{BLACK SMILING FACE}\n" + - u"web.port = {}\n".format(port_endpoint) + u"web.port = {}\n".format(web_port_endpoint) + + u"tub.port = {}\n".format(main_port_endpoint) + + u"tub.location = {}\n".format(main_location_hint) ).encode("utf-8") fileutil.make_dirs(iv_dir) From ffe23452a4e83b6f912d2b6c94584f10235ed457 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 30 Oct 2021 13:32:42 +0100 Subject: [PATCH 121/916] gpg setup Signed-off-by: fenn-cs --- docs/release-checklist.rst | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index dc060cd8d..75a9e2f4a 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -56,7 +56,7 @@ process is complete.* Create Branch and Apply Updates ``````````````````````````````` -- Create a branch for the release (e.g. `XXXX.release-1.16.0`) +- Create a branch for the release/candidate (e.g. `XXXX.release-1.16.0`) - run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) - create the news for the release @@ -92,6 +92,27 @@ Create Branch and Apply Updates - Confirm CI runs successfully on all platforms +Preparing to Authenticate Release (Setting up GPG) +`````````````````````````````````````````````````` +*Skip the section if you already have GPG setup.* + +In other to keep releases authentic it's required that releases are signed before being +published. This ensure's that users of Tahoe are able to verify that the version of Tahoe +they are using is coming from a trusted or at the very least known source. + +The authentication is done using the ``GPG`` implementation of ``OpenGPG`` to be able to complete +the release steps you would have to download the ``GPG`` software and setup a key(identity). + +- `Download `__ and install GPG for your operating system. +- Generate a key pair using ``gpg --gen-key``. *Some questions would be asked to personalize your key configuration.* + +You might take additional steps including: + +- Setting up a revocation certificate (Incase you lose your secret key) +- Backing up your key pair +- Upload your fingerprint to a keyserver such as `openpgp.org `__ + + Create Release Candidate ```````````````````````` @@ -108,8 +129,10 @@ they will need to evaluate which contributors' signatures they trust. - (all steps above are completed) - sign the release - - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.15.0rc0" tahoe-lafs-1.15.0rc0 - - (replace the key-id above with your own) + - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` + +*Replace the key-id above with your own, which can simply be your email if's attached your fingerprint.* +*Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0`* - build all code locally - these should all pass: @@ -123,8 +146,7 @@ they will need to evaluate which contributors' signatures they trust. - build tarballs - tox -e tarballs - - confirm it at least exists: - - ls dist/ | grep 1.15.0rc0 + - Confirm that release tarballs exist by runnig: ``ls dist/ | grep 1.16.0rc0`` - inspect and test the tarballs @@ -133,8 +155,8 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2.py3-none-any.whl - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz + - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl`` + - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz`` Privileged Contributor From 882f1973062df9b3d1aec49d468621e46f72dfb6 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 30 Oct 2021 13:37:58 +0100 Subject: [PATCH 122/916] format updates Signed-off-by: fenn-cs --- docs/release-checklist.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 75a9e2f4a..403a6f933 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -50,13 +50,13 @@ previously generated files. practice to give it the release name. You MAY also discard this directory once the release process is complete.* -- ``cd into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]```` +- ``cd`` into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]`` Create Branch and Apply Updates ``````````````````````````````` -- Create a branch for the release/candidate (e.g. `XXXX.release-1.16.0`) +- Create a branch for the release/candidate (e.g. ``XXXX.release-1.16.0``) - run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) - create the news for the release @@ -73,7 +73,7 @@ Create Branch and Apply Updates - update "relnotes.txt" - - update all mentions of 1.16.0 -> 1.16.x + - update all mentions of ``1.16.0`` to new and higher release version for example ``1.16.1`` - update "previous release" statement and date - summarize major changes - commit it From 5ba636c7b10fd146c39eff9a60c34f9eb5943a9a Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 2 Nov 2021 10:36:32 +0100 Subject: [PATCH 123/916] removed deferred logger from basic function in test_logs Signed-off-by: fenn-cs --- src/allmydata/test/web/test_logs.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index fe0a0445d..81ec357c0 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -54,9 +54,7 @@ from ...web.logs import ( TokenAuthenticatedWebSocketServerProtocol, ) -from ...util.eliotutil import ( - log_call_deferred -) +from eliot import log_call class StreamingEliotLogsTests(SyncTestCase): """ @@ -110,7 +108,7 @@ class TestStreamingLogs(AsyncTestCase): messages.append(json.loads(msg)) proto.on("message", got_message) - @log_call_deferred(action_type=u"test:cli:some-exciting-action") + @log_call(action_type=u"test:cli:some-exciting-action") def do_a_thing(arguments): pass @@ -121,7 +119,7 @@ class TestStreamingLogs(AsyncTestCase): self.assertThat(len(messages), Equals(3)) self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) - self.assertThat(messages[0]["kwargs"]["arguments"], + self.assertThat(messages[0]["arguments"], Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) self.assertThat(messages[1]["action_type"], Equals("test:cli:some-exciting-action")) self.assertThat("started", Equals(messages[0]["action_status"])) From fcfc89e3ae4d2a73ba110b2b23eaf24001e78dd9 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 2 Nov 2021 14:32:20 +0100 Subject: [PATCH 124/916] moved new tests/update for eliotutils Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 73 ---------------------------- src/allmydata/util/eliotutil.py | 22 +-------- 2 files changed, 2 insertions(+), 93 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 61e0a6958..3f915ecd2 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -56,7 +56,6 @@ from eliot.testing import ( capture_logging, assertHasAction, swap_logger, - assertContainsFields, ) from twisted.internet.defer import ( @@ -282,75 +281,3 @@ class LogCallDeferredTests(TestCase): ), ), ) - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_gets_positional_arguments(self, logger): - """ - Check that positional arguments are logged when using ``log_call_deferred`` - """ - @log_call_deferred(action_type=u"the-action") - def f(a): - return a ** 2 - self.assertThat( - f(4), succeeded(Equals(16))) - msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (4,)}) - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_gets_keyword_arguments(self, logger): - """ - Check that keyword arguments are logged when using ``log_call_deferred`` - """ - @log_call_deferred(action_type=u"the-action") - def f(base, exp): - return base ** exp - self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) - msg = logger.messages[0] - assertContainsFields(self, msg, {"kwargs": {"base": 10, "exp": 2}}) - - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_gets_keyword_and_positional_arguments(self, logger): - """ - Check that both keyword and positional arguments are logged when using ``log_call_deferred`` - """ - @log_call_deferred(action_type=u"the-action") - def f(base, exp, message): - return base ** exp - self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) - msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2)}) - assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) - - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_keyword_args_dont_overlap_with_start_action(self, logger): - """ - Check that kwargs passed to decorated functions don't overlap with params in ``start_action`` - """ - @log_call_deferred(action_type=u"the-action") - def f(base, exp, kwargs, args): - return base ** exp - self.assertThat( - f(10, 2, kwargs={"kwarg_1": "value_1", "kwarg_2": 2}, args=(1, 2, 3)), - succeeded(Equals(100)), - ) - msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2)}) - assertContainsFields( - self, - msg, - {"kwargs": {"args": [1, 2, 3], "kwargs": {"kwarg_1": "value_1", "kwarg_2": 2}}}, - ) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index fe431568f..4e48fbb9f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,11 +87,7 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import ( - AnyBytesJSONEncoder, - bytes_to_unicode -) -import json +from .jsonbytes import AnyBytesJSONEncoder def validateInstanceOf(t): @@ -315,14 +311,6 @@ class _DestinationParser(object): _parse_destination_description = _DestinationParser().parse -def is_json_serializable(object): - try: - json.dumps(object) - return True - except (TypeError, OverflowError): - return False - - def log_call_deferred(action_type): """ Like ``eliot.log_call`` but for functions which return ``Deferred``. @@ -332,11 +320,7 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} - # Remove complex (unserializable) objects from positional args to - # prevent eliot from throwing errors when it attempts serialization - args = tuple(arg if is_json_serializable(arg) else str(arg) for arg in a) - with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): + with start_action(action_type=action_type).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) @@ -350,5 +334,3 @@ if PY2: capture_logging = eliot_capture_logging else: capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) - - From 39c4a2c4eb1963b2035644d97c1760b649c21278 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 2 Nov 2021 15:10:54 -0400 Subject: [PATCH 125/916] tidy up some corners --- src/allmydata/scripts/debug.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index ab48b0fd0..260cca55b 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -746,6 +746,13 @@ def _describe_mutable_share(abs_sharefile, f, now, si_s, out): if share_type == "SDMF": f.seek(m.DATA_OFFSET) + + # Read at least the mutable header length, if possible. If there's + # less data than that in the share, don't try to read more (we won't + # be able to unpack the header in this case but we surely don't want + # to try to unpack bytes *following* the data section as if they were + # header data). Rather than 2000 we could use HEADER_LENGTH from + # allmydata/mutable/layout.py, probably. data = f.read(min(data_length, 2000)) try: @@ -810,8 +817,8 @@ def _describe_immutable_share(abs_sharefile, now, si_s, out): sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) - expiration_time = min( [lease.get_expiration_time() - for lease in sf.get_leases()] ) + expiration_time = min(lease.get_expiration_time() + for lease in sf.get_leases()) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) @@ -934,9 +941,10 @@ def corrupt_share(options): if MutableShareFile.is_valid_header(prefix): # mutable m = MutableShareFile(fn) - f = open(fn, "rb") - f.seek(m.DATA_OFFSET) - data = f.read(2000) + with open(fn, "rb") as f: + f.seek(m.DATA_OFFSET) + # Read enough data to get a mutable header to unpack. + data = f.read(2000) # make sure this slot contains an SMDF share assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" f.close() From 08cf881e28f84e74865284697234e9760776d9f5 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 22:16:14 -0600 Subject: [PATCH 126/916] test with real-size keys --- src/allmydata/test/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 38282297a..fc268311e 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -133,6 +133,7 @@ from subprocess import ( ) TEST_RSA_KEY_SIZE = 522 +TEST_RSA_KEY_SIZE = 2048 EMPTY_CLIENT_CONFIG = config_from_string( "/dev/null", From b3d1acd14a1f602df5bba424214070a4643a8bab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 09:55:16 -0400 Subject: [PATCH 127/916] try skipping Tor integration tests on Python 2 --- integration/test_tor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/test_tor.py b/integration/test_tor.py index 15d888e36..b0419f0d2 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -35,6 +35,9 @@ from allmydata.test.common import ( if sys.platform.startswith('win'): pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) +if PY2: + pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True) + @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) From 4606c3c9dde91de11d769bf9d8c6fd6f2fd1f877 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 09:59:19 -0400 Subject: [PATCH 128/916] news fragment --- newsfragments/3837.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3837.other diff --git a/newsfragments/3837.other b/newsfragments/3837.other new file mode 100644 index 000000000..a9e4e6986 --- /dev/null +++ b/newsfragments/3837.other @@ -0,0 +1 @@ +Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies. From 8e150cce6a27b6616db54cfd4c2ac08fbdd13794 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 13:14:55 -0400 Subject: [PATCH 129/916] add explicit direct tests for the new methods --- src/allmydata/test/test_storage.py | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index aac40362c..460653bd0 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3066,3 +3066,64 @@ class ShareFileTests(unittest.TestCase): sf.add_lease(lease) (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_cancel_secret(cancel_secret)) + + +class LeaseInfoTests(unittest.TestCase): + """ + Tests for ``allmydata.storage.lease.LeaseInfo``. + """ + def test_is_renew_secret(self): + """ + ``LeaseInfo.is_renew_secret`` returns ``True`` if the value given is the + renew secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertTrue(lease.is_renew_secret(renew_secret)) + + def test_is_not_renew_secret(self): + """ + ``LeaseInfo.is_renew_secret`` returns ``False`` if the value given is not + the renew secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertFalse(lease.is_renew_secret(cancel_secret)) + + def test_is_cancel_secret(self): + """ + ``LeaseInfo.is_cancel_secret`` returns ``True`` if the value given is the + cancel secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertTrue(lease.is_cancel_secret(cancel_secret)) + + def test_is_not_cancel_secret(self): + """ + ``LeaseInfo.is_cancel_secret`` returns ``False`` if the value given is not + the cancel secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertFalse(lease.is_cancel_secret(renew_secret)) From 7335b2a5977752c0805a7fd9c7759cafa8ac31b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 13:16:15 -0400 Subject: [PATCH 130/916] remove unused import --- src/allmydata/storage/immutable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 4f6a1c9c7..8a7a5a966 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -24,7 +24,6 @@ from allmydata.interfaces import ( ) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition -from allmydata.util.hashutil import timing_safe_compare from allmydata.storage.lease import LeaseInfo from allmydata.storage.common import UnknownImmutableContainerVersionError From 86ca463c3198746e31b61569f79e860f4a6e7d6d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 13:24:04 -0400 Subject: [PATCH 131/916] news fragment --- newsfragments/3834.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3834.minor diff --git a/newsfragments/3834.minor b/newsfragments/3834.minor new file mode 100644 index 000000000..e69de29bb From 797e0994596dd916a978b5fc8a757d15322b3100 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:05:28 -0400 Subject: [PATCH 132/916] make create_introducer_webish assign a main tub port --- src/allmydata/test/web/test_introducer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 4b5850cbc..69309d35b 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -83,12 +83,18 @@ def create_introducer_webish(reactor, port_assigner, basedir): with the node and its webish service. """ node.create_node_dir(basedir, "testing") - _, port_endpoint = port_assigner.assign(reactor) + main_tub_location, main_tub_endpoint = port_assigner.assign(reactor) + _, web_port_endpoint = port_assigner.assign(reactor) with open(join(basedir, "tahoe.cfg"), "w") as f: f.write( "[node]\n" - "tub.location = 127.0.0.1:1\n" + - "web.port = {}\n".format(port_endpoint) + "tub.port = {main_tub_endpoint}\n" + "tub.location = {main_tub_location}\n" + "web.port = {web_port_endpoint}\n".format( + main_tub_endpoint=main_tub_endpoint, + main_tub_location=main_tub_location, + web_port_endpoint=web_port_endpoint, + ) ) intro_node = yield create_introducer(basedir) From 31649890ef47c2169a0aedee2d7488b8f6da6959 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:08:08 -0400 Subject: [PATCH 133/916] Teach UseNode to use a port assigner for tub.port Then use it to assign ports for tub.port unless the caller supplied their own value. --- src/allmydata/test/common.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 97368ee92..e0472edce 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -267,8 +267,12 @@ class UseNode(object): node_config = attr.ib(default=attr.Factory(dict)) config = attr.ib(default=None) + reactor = attr.ib(default=None) def setUp(self): + self.assigner = SameProcessStreamEndpointAssigner() + self.assigner.setUp() + def format_config_items(config): return "\n".join( " = ".join((key, value)) @@ -292,6 +296,23 @@ class UseNode(object): "default", self.introducer_furl, ) + + node_config = self.node_config.copy() + if "tub.port" not in node_config: + if "tub.location" in node_config: + raise ValueError( + "UseNode fixture does not support specifying tub.location " + "without tub.port" + ) + + # Don't use the normal port auto-assignment logic. It produces + # collisions and makes tests fail spuriously. + tub_location, tub_endpoint = self.assigner.assign(self.reactor) + node_config.update({ + "tub.port": tub_endpoint, + "tub.location": tub_location, + }) + self.config = config_from_string( self.basedir.asTextMode().path, "tub.port", @@ -304,7 +325,7 @@ storage.plugins = {storage_plugin} {plugin_config_section} """.format( storage_plugin=self.storage_plugin, - node_config=format_config_items(self.node_config), + node_config=format_config_items(node_config), plugin_config_section=plugin_config_section, ) ) @@ -316,7 +337,7 @@ storage.plugins = {storage_plugin} ) def cleanUp(self): - pass + self.assigner.tearDown() def getDetails(self): From 5a71774bf875a71c8ddbfb8b4fcfcb2dda7a4f9d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:10:32 -0400 Subject: [PATCH 134/916] use port assigner and UseNode more in test_node.py --- src/allmydata/test/test_node.py | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index cf5fa27f3..c6cff1bab 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -69,6 +69,8 @@ import allmydata.test.common_util as testutil from .common import ( ConstantAddresses, + SameProcessStreamEndpointAssigner, + UseNode, ) def port_numbers(): @@ -80,11 +82,10 @@ class LoggingMultiService(service.MultiService): # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2946 -def testing_tub(config_data=''): +def testing_tub(reactor, config_data=''): """ Creates a 'main' Tub for testing purposes, from config data """ - from twisted.internet import reactor basedir = 'dummy_basedir' config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) fileutil.make_dirs(os.path.join(basedir, 'private')) @@ -112,6 +113,9 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): # try to bind the port. We'll use a low-numbered one that's likely to # conflict with another service to prove it. self._available_port = 22 + self.port_assigner = SameProcessStreamEndpointAssigner() + self.port_assigner.setUp() + self.addCleanup(self.port_assigner.tearDown) def _test_location( self, @@ -137,11 +141,23 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): :param local_addresses: If not ``None`` then a list of addresses to supply to the system under test as local addresses. """ + from twisted.internet import reactor + basedir = self.mktemp() create_node_dir(basedir, "testing") + if tub_port is None: + # Always configure a usable tub.port address instead of relying on + # the automatic port assignment. The automatic port assignment is + # prone to collisions and spurious test failures. + _, tub_port = self.port_assigner.assign(reactor) + config_data = "[node]\n" - if tub_port: - config_data += "tub.port = {}\n".format(tub_port) + config_data += "tub.port = {}\n".format(tub_port) + + # If they wanted a certain location, go for it. This probably won't + # agree with the tub.port value we set but that only matters if + # anything tries to use this to establish a connection ... which + # nothing in this test suite will. if tub_location is not None: config_data += "tub.location = {}\n".format(tub_location) @@ -149,7 +165,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.patch(iputil, 'get_local_addresses_sync', lambda: local_addresses) - tub = testing_tub(config_data) + tub = testing_tub(reactor, config_data) class Foo(object): pass @@ -431,7 +447,12 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): @defer.inlineCallbacks def test_logdir_is_str(self): - basedir = "test_node/test_logdir_is_str" + from twisted.internet import reactor + + basedir = FilePath(self.mktemp()) + fixture = UseNode(None, None, basedir, "pb://introducer/furl", {}, reactor=reactor) + fixture.setUp() + self.addCleanup(fixture.cleanUp) ns = Namespace() ns.called = False @@ -440,8 +461,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.failUnless(isinstance(logdir, str), logdir) self.patch(foolscap.logging.log, 'setLogDir', call_setLogDir) - create_node_dir(basedir, "nothing to see here") - yield client.create_client(basedir) + yield fixture.create_node() self.failUnless(ns.called) def test_set_config_unescaped_furl_hash(self): From 5caa80fe383630aab8afa8a9a1667fb3d4cd8f60 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:11:08 -0400 Subject: [PATCH 135/916] use UseNode more in test_client.py Also make write_introducer more lenient about filesystem state --- src/allmydata/scripts/common.py | 4 +++- src/allmydata/test/test_client.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 0a9ab8714..c9fc8e031 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -141,7 +141,9 @@ def write_introducer(basedir, petname, furl): """ if isinstance(furl, bytes): furl = furl.decode("utf-8") - basedir.child(b"private").child(b"introducers.yaml").setContent( + private = basedir.child(b"private") + private.makedirs(ignoreExistingDirectory=True) + private.child(b"introducers.yaml").setContent( safe_dump({ "introducers": { petname: { diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index fd2837f1d..a2572e735 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -89,6 +89,7 @@ from .common import ( UseTestPlugins, MemoryIntroducerClient, get_published_announcements, + UseNode, ) from .matchers import ( MatchesSameElements, @@ -953,13 +954,14 @@ class Run(unittest.TestCase, testutil.StallMixin): @defer.inlineCallbacks def test_reloadable(self): - basedir = FilePath("test_client.Run.test_reloadable") - private = basedir.child("private") - private.makedirs() + from twisted.internet import reactor + dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" - write_introducer(basedir, "someintroducer", dummy) - basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii")) - c1 = yield client.create_client(basedir.path) + fixture = UseNode(None, None, FilePath(self.mktemp()), dummy, reactor=reactor) + fixture.setUp() + self.addCleanup(fixture.cleanUp) + + c1 = yield fixture.create_node() c1.setServiceParent(self.sparent) # delay to let the service start up completely. I'm not entirely sure @@ -981,7 +983,7 @@ class Run(unittest.TestCase, testutil.StallMixin): # also change _check_exit_trigger to use it instead of a raw # reactor.stop, also instrument the shutdown event in an # attribute that we can check.) - c2 = yield client.create_client(basedir.path) + c2 = yield fixture.create_node() c2.setServiceParent(self.sparent) yield c2.disownServiceParent() From 780be2691b9ca4a7b1b2d08c6d0bb44b11d8b9a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:11:28 -0400 Subject: [PATCH 136/916] assign a tub.port to all system test nodes --- src/allmydata/test/common_system.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 874c7f6ba..0c424136a 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -767,13 +767,15 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def _generate_config(self, which, basedir): config = {} - except1 = set(range(self.numclients)) - {1} + allclients = set(range(self.numclients)) + except1 = allclients - {1} feature_matrix = { ("client", "nickname"): except1, - # client 1 has to auto-assign an address. - ("node", "tub.port"): except1, - ("node", "tub.location"): except1, + # Auto-assigning addresses is extremely failure prone and not + # amenable to automated testing in _this_ manner. + ("node", "tub.port"): allclients, + ("node", "tub.location"): allclients, # client 0 runs a webserver and a helper # client 3 runs a webserver but no helper @@ -855,7 +857,13 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # connection-lost code basedir = FilePath(self.getdir("client%d" % client_num)) basedir.makedirs() - config = "[client]\n" + config = ( + "[node]\n" + "tub.location = {}\n" + "tub.port = {}\n" + "[client]\n" + ).format(*self.port_assigner.assign(reactor)) + if helper_furl: config += "helper.furl = %s\n" % helper_furl basedir.child("tahoe.cfg").setContent(config.encode("utf-8")) From b4bc95cb5a36b7507ce3745cfc3a273b5eedecb6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:15:38 -0400 Subject: [PATCH 137/916] news fragment --- newsfragments/3838.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3838.minor diff --git a/newsfragments/3838.minor b/newsfragments/3838.minor new file mode 100644 index 000000000..e69de29bb From 0459b712b02b8ba686687d696325bcdb650f770c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 08:54:55 -0400 Subject: [PATCH 138/916] news fragment --- newsfragments/3839.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3839.security diff --git a/newsfragments/3839.security b/newsfragments/3839.security new file mode 100644 index 000000000..1ae054542 --- /dev/null +++ b/newsfragments/3839.security @@ -0,0 +1 @@ +The storage server now keeps hashes of lease renew and cancel secrets for immutable share files instead of keeping the original secrets. From 274dc6e837dd7181fb1f6ba9116570dc4b255d66 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 08:55:37 -0400 Subject: [PATCH 139/916] Introduce `UnknownContainerVersionError` base w/ structured args --- src/allmydata/storage/common.py | 11 ++++++++--- src/allmydata/storage/immutable.py | 4 +--- src/allmydata/storage/mutable.py | 4 +--- src/allmydata/test/test_storage.py | 7 ++++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index e5563647f..48fc77840 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -16,11 +16,16 @@ from allmydata.util import base32 # Backwards compatibility. from allmydata.interfaces import DataTooLargeError # noqa: F401 -class UnknownMutableContainerVersionError(Exception): - pass -class UnknownImmutableContainerVersionError(Exception): +class UnknownContainerVersionError(Exception): + def __init__(self, filename, version): + self.filename = filename + self.version = version + +class UnknownMutableContainerVersionError(UnknownContainerVersionError): pass +class UnknownImmutableContainerVersionError(UnknownContainerVersionError): + pass def si_b2a(storageindex): return base32.b2a(storageindex) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index a43860138..fcc60509c 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -174,9 +174,7 @@ class ShareFile(object): filesize = os.path.getsize(self.home) (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) if version != 1: - msg = "sharefile %s had version %d but we wanted 1" % \ - (filename, version) - raise UnknownImmutableContainerVersionError(msg) + raise UnknownImmutableContainerVersionError(filename, version) self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) self._data_offset = 0xc diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 4abf22064..ce9cc5ff4 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -95,9 +95,7 @@ class MutableShareFile(object): data_length, extra_least_offset) = \ struct.unpack(">32s20s32sQQ", data) if not self.is_valid_header(data): - msg = "sharefile %s had magic '%r' but we wanted '%r'" % \ - (filename, magic, self.MAGIC) - raise UnknownMutableContainerVersionError(msg) + raise UnknownMutableContainerVersionError(filename, magic) self.parent = parent # for logging def log(self, *args, **kwargs): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 2c8d84b9e..bf9eff37a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -646,7 +646,8 @@ class Server(unittest.TestCase): e = self.failUnlessRaises(UnknownImmutableContainerVersionError, ss.remote_get_buckets, b"si1") - self.failUnlessIn(" had version 0 but we wanted 1", str(e)) + self.assertEqual(e.filename, fn) + self.assertEqual(e.version, 0) def test_disconnect(self): # simulate a disconnection @@ -1127,8 +1128,8 @@ class MutableServer(unittest.TestCase): read = ss.remote_slot_readv e = self.failUnlessRaises(UnknownMutableContainerVersionError, read, b"si1", [0], [(0,10)]) - self.failUnlessIn(" had magic ", str(e)) - self.failUnlessIn(" but we wanted ", str(e)) + self.assertEqual(e.filename, fn) + self.assertTrue(e.version.startswith(b"BAD MAGIC")) def test_container_size(self): ss = self.create("test_container_size") From 10724a91f9ca2fe929f4e29adb03b876b21f9fe5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 10:17:36 -0400 Subject: [PATCH 140/916] introduce an explicit representation of the v1 immutable container schema This is only a partial representation, sufficient to express the changes that are coming in v2. --- src/allmydata/storage/immutable.py | 37 ++++++----- src/allmydata/storage/immutable_schema.py | 81 +++++++++++++++++++++++ 2 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 src/allmydata/storage/immutable_schema.py diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index fcc60509c..ae5a710af 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -25,9 +25,14 @@ from allmydata.interfaces import ( ) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition -from allmydata.storage.lease import LeaseInfo from allmydata.storage.common import UnknownImmutableContainerVersionError +from .immutable_schema import ( + NEWEST_SCHEMA_VERSION, + schema_from_version, +) + + # each share file (in storage/shares/$SI/$SHNUM) contains lease information # and share data. The share data is accessed by RIBucketWriter.write and # RIBucketReader.read . The lease information is not accessible through these @@ -118,9 +123,16 @@ class ShareFile(object): ``False`` otherwise. """ (version,) = struct.unpack(">L", header[:4]) - return version == 1 + return schema_from_version(version) is not None - def __init__(self, filename, max_size=None, create=False, lease_count_format="L"): + def __init__( + self, + filename, + max_size=None, + create=False, + lease_count_format="L", + schema=NEWEST_SCHEMA_VERSION, + ): """ Initialize a ``ShareFile``. @@ -156,24 +168,17 @@ class ShareFile(object): # it. Also construct the metadata. assert not os.path.exists(self.home) fileutil.make_dirs(os.path.dirname(self.home)) - # The second field -- the four-byte share data length -- is no - # longer used as of Tahoe v1.3.0, but we continue to write it in - # there in case someone downgrades a storage server from >= - # Tahoe-1.3.0 to < Tahoe-1.3.0, or moves a share file from one - # server to another, etc. We do saturation -- a share data length - # larger than 2**32-1 (what can fit into the field) is marked as - # the largest length that can fit into the field. That way, even - # if this does happen, the old < v1.3.0 server will still allow - # clients to read the first part of the share. + self._schema = schema with open(self.home, 'wb') as f: - f.write(struct.pack(">LLL", 1, min(2**32-1, max_size), 0)) + f.write(self._schema.header(max_size)) self._lease_offset = max_size + 0x0c self._num_leases = 0 else: with open(self.home, 'rb') as f: filesize = os.path.getsize(self.home) (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) - if version != 1: + self._schema = schema_from_version(version) + if self._schema is None: raise UnknownImmutableContainerVersionError(filename, version) self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) @@ -209,7 +214,7 @@ class ShareFile(object): offset = self._lease_offset + lease_number * self.LEASE_SIZE f.seek(offset) assert f.tell() == offset - f.write(lease_info.to_immutable_data()) + f.write(self._schema.serialize_lease(lease_info)) def _read_num_leases(self, f): f.seek(0x08) @@ -240,7 +245,7 @@ class ShareFile(object): for i in range(num_leases): data = f.read(self.LEASE_SIZE) if data: - yield LeaseInfo.from_immutable_data(data) + yield self._schema.unserialize_lease(data) def add_lease(self, lease_info): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py new file mode 100644 index 000000000..759752bed --- /dev/null +++ b/src/allmydata/storage/immutable_schema.py @@ -0,0 +1,81 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import struct + +from .lease import ( + LeaseInfo, +) + +def _header(version, max_size): + # (int, int) -> bytes + """ + Construct the header for an immutable container. + + :param version: The container version to include the in header. + :param max_size: The maximum data size the container will hold. + + :return: Some bytes to write at the beginning of the container. + """ + # The second field -- the four-byte share data length -- is no longer + # used as of Tahoe v1.3.0, but we continue to write it in there in + # case someone downgrades a storage server from >= Tahoe-1.3.0 to < + # Tahoe-1.3.0, or moves a share file from one server to another, + # etc. We do saturation -- a share data length larger than 2**32-1 + # (what can fit into the field) is marked as the largest length that + # can fit into the field. That way, even if this does happen, the old + # < v1.3.0 server will still allow clients to read the first part of + # the share. + return struct.pack(">LLL", version, min(2**32 - 1, max_size), 0) + +class _V1(object): + """ + Implement encoding and decoding for v1 of the immutable container. + """ + version = 1 + + @classmethod + def header(cls, max_size): + return _header(cls.version, max_size) + + @classmethod + def serialize_lease(cls, lease): + if isinstance(lease, LeaseInfo): + return lease.to_immutable_data() + raise ValueError( + "ShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # In v1 of the immutable schema lease secrets are stored plaintext. + # So load the data into a plain LeaseInfo which works on plaintext + # secrets. + return LeaseInfo.from_immutable_data(data) + + +ALL_SCHEMAS = {_V1} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) + +def schema_from_version(version): + # (int) -> Optional[type] + """ + Find the schema object that corresponds to a certain version number. + """ + for schema in ALL_SCHEMAS: + if schema.version == version: + return schema + return None From 3b4141952387452894ed5c0ed58113e272ad3e4f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 10:32:59 -0400 Subject: [PATCH 141/916] apply the ShareFile tests to all schema versions using hypothesis --- src/allmydata/test/test_storage.py | 60 +++++++++++++++++++----------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bf9eff37a..655395042 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -43,6 +43,9 @@ from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile +from allmydata.storage.immutable_schema import ( + ALL_SCHEMAS, +) from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ si_b2a, si_a2b @@ -844,6 +847,9 @@ class Server(unittest.TestCase): # Create a bucket: rs0, cs0 = self.create_bucket_5_shares(ss, b"si0") + + # Upload of an immutable implies creation of a single lease with the + # supplied secrets. (lease,) = ss.get_leases(b"si0") self.assertTrue(lease.is_renew_secret(rs0)) @@ -3125,6 +3131,7 @@ class Stats(unittest.TestCase): self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) +immutable_schemas = strategies.sampled_from(list(ALL_SCHEMAS)) class ShareFileTests(unittest.TestCase): """Tests for allmydata.storage.immutable.ShareFile.""" @@ -3136,47 +3143,54 @@ class ShareFileTests(unittest.TestCase): # Should be b'abDEF' now. return sf - def test_read_write(self): + @given(immutable_schemas) + def test_read_write(self, schema): """Basic writes can be read.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) self.assertEqual(sf.read_share_data(0, 3), b"abD") self.assertEqual(sf.read_share_data(1, 4), b"bDEF") - def test_reads_beyond_file_end(self): + @given(immutable_schemas) + def test_reads_beyond_file_end(self, schema): """Reads beyond the file size are truncated.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) self.assertEqual(sf.read_share_data(0, 10), b"abDEF") self.assertEqual(sf.read_share_data(5, 10), b"") - def test_too_large_write(self): + @given(immutable_schemas) + def test_too_large_write(self, schema): """Can't do write larger than file size.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) with self.assertRaises(DataTooLargeError): sf.write_share_data(0, b"x" * 3000) - def test_no_leases_cancelled(self): + @given(immutable_schemas) + def test_no_leases_cancelled(self, schema): """If no leases were cancelled, IndexError is raised.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) with self.assertRaises(IndexError): sf.cancel_lease(b"garbage") - def test_long_lease_count_format(self): + @given(immutable_schemas) + def test_long_lease_count_format(self, schema): """ ``ShareFile.__init__`` raises ``ValueError`` if the lease count format given is longer than one character. """ with self.assertRaises(ValueError): - self.get_sharefile(lease_count_format="BB") + self.get_sharefile(schema=schema, lease_count_format="BB") - def test_large_lease_count_format(self): + @given(immutable_schemas) + def test_large_lease_count_format(self, schema): """ ``ShareFile.__init__`` raises ``ValueError`` if the lease count format encodes to a size larger than 8 bytes. """ with self.assertRaises(ValueError): - self.get_sharefile(lease_count_format="Q") + self.get_sharefile(schema=schema, lease_count_format="Q") - def test_avoid_lease_overflow(self): + @given(immutable_schemas) + def test_avoid_lease_overflow(self, schema): """ If the share file already has the maximum number of leases supported then ``ShareFile.add_lease`` raises ``struct.error`` and makes no changes @@ -3190,7 +3204,7 @@ class ShareFileTests(unittest.TestCase): ) # Make it a little easier to reach the condition by limiting the # number of leases to only 255. - sf = self.get_sharefile(lease_count_format="B") + sf = self.get_sharefile(schema=schema, lease_count_format="B") # Add the leases. for i in range(2 ** 8 - 1): @@ -3214,16 +3228,17 @@ class ShareFileTests(unittest.TestCase): self.assertEqual(before_data, after_data) - def test_renew_secret(self): + @given(immutable_schemas) + def test_renew_secret(self, schema): """ - A lease loaded from an immutable share file can have its renew secret - verified. + A lease loaded from an immutable share file at any schema version can have + its renew secret verified. """ renew_secret = b"r" * 32 cancel_secret = b"c" * 32 expiration_time = 2 ** 31 - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) lease = LeaseInfo( owner_num=0, renew_secret=renew_secret, @@ -3234,16 +3249,17 @@ class ShareFileTests(unittest.TestCase): (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_renew_secret(renew_secret)) - def test_cancel_secret(self): + @given(immutable_schemas) + def test_cancel_secret(self, schema): """ - A lease loaded from an immutable share file can have its cancel secret - verified. + A lease loaded from an immutable share file at any schema version can have + its cancel secret verified. """ renew_secret = b"r" * 32 cancel_secret = b"c" * 32 expiration_time = 2 ** 31 - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) lease = LeaseInfo( owner_num=0, renew_secret=renew_secret, From 234b8dcde2febc2b3eee96e1ede4d123f634dcb1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 11:56:49 -0400 Subject: [PATCH 142/916] Formalize LeaseInfo interface in preparation for another implementation --- src/allmydata/storage/lease.py | 83 ++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 6d21bb2b2..23071707a 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -15,6 +15,11 @@ import struct, time import attr +from zope.interface import ( + Interface, + implementer, +) + from allmydata.util.hashutil import timing_safe_compare # struct format for representation of a lease in an immutable share @@ -23,6 +28,84 @@ IMMUTABLE_FORMAT = ">L32s32sL" # struct format for representation of a lease in a mutable share MUTABLE_FORMAT = ">LL32s32s20s" + +class ILeaseInfo(Interface): + """ + Represent a marker attached to a share that indicates that share should be + retained for some amount of time. + + Typically clients will create and renew leases on their shares as a way to + inform storage servers that there is still interest in those shares. A + share may have more than one lease. If all leases on a share have + expiration times in the past then the storage server may take this as a + strong hint that no one is interested in the share anymore and therefore + the share may be deleted to reclaim the space. + """ + def renew(new_expire_time): + """ + Create a new ``ILeaseInfo`` with the given expiration time. + + :param Union[int, float] new_expire_time: The expiration time the new + ``ILeaseInfo`` will have. + + :return: The new ``ILeaseInfo`` provider with the new expiration time. + """ + + def get_expiration_time(): + """ + :return Union[int, float]: this lease's expiration time + """ + + def get_grant_renew_time_time(): + """ + :return Union[int, float]: a guess about the last time this lease was + renewed + """ + + def get_age(): + """ + :return Union[int, float]: a guess about how long it has been since this + lease was renewed + """ + + def to_immutable_data(): + """ + :return bytes: a serialized representation of this lease suitable for + inclusion in an immutable container + """ + + def to_mutable_data(): + """ + :return bytes: a serialized representation of this lease suitable for + inclusion in a mutable container + """ + + def immutable_size(): + """ + :return int: the size of the serialized representation of this lease in an + immutable container + """ + + def mutable_size(): + """ + :return int: the size of the serialized representation of this lease in a + mutable container + """ + + def is_renew_secret(candidate_secret): + """ + :return bool: ``True`` if the given byte string is this lease's renew + secret, ``False`` otherwise + """ + + def is_cancel_secret(candidate_secret): + """ + :return bool: ``True`` if the given byte string is this lease's cancel + secret, ``False`` otherwise + """ + + +@implementer(ILeaseInfo) @attr.s(frozen=True) class LeaseInfo(object): """ From b69e8d013bfc32f8b7ca5948ad36a5c60b3db73a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:07:49 -0400 Subject: [PATCH 143/916] introduce immutable container schema version 2 This version used on-disk hashed secrets to reduce the chance of secrets leaking to unintended parties. --- src/allmydata/storage/immutable.py | 23 ++++- src/allmydata/storage/immutable_schema.py | 105 +++++++++++++++++++++- src/allmydata/storage/lease.py | 82 +++++++++++++++++ src/allmydata/test/test_download.py | 14 ++- 4 files changed, 214 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index ae5a710af..216262a81 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -39,14 +39,14 @@ from .immutable_schema import ( # interfaces. # The share file has the following layout: -# 0x00: share file version number, four bytes, current version is 1 +# 0x00: share file version number, four bytes, current version is 2 # 0x04: share data length, four bytes big-endian = A # See Footnote 1 below. # 0x08: number of leases, four bytes big-endian # 0x0c: beginning of share data (see immutable.layout.WriteBucketProxy) # A+0x0c = B: first lease. Lease format is: # B+0x00: owner number, 4 bytes big-endian, 0 is reserved for no-owner -# B+0x04: renew secret, 32 bytes (SHA256) -# B+0x24: cancel secret, 32 bytes (SHA256) +# B+0x04: renew secret, 32 bytes (SHA256 + blake2b) # See Footnote 2 below. +# B+0x24: cancel secret, 32 bytes (SHA256 + blake2b) # B+0x44: expiration time, 4 bytes big-endian seconds-since-epoch # B+0x48: next lease, or end of record @@ -58,6 +58,23 @@ from .immutable_schema import ( # then the value stored in this field will be the actual share data length # modulo 2**32. +# Footnote 2: The change between share file version number 1 and 2 is that +# storage of lease secrets is changed from plaintext to hashed. This change +# protects the secrets from compromises of local storage on the server: if a +# plaintext cancel secret is somehow exfiltrated from the storage server, an +# attacker could use it to cancel that lease and potentially cause user data +# to be discarded before intended by the real owner. As of this comment, +# lease cancellation is disabled because there have been at least two bugs +# which leak the persisted value of the cancellation secret. If lease secrets +# were stored hashed instead of plaintext then neither of these bugs would +# have allowed an attacker to learn a usable cancel secret. +# +# Clients are free to construct these secrets however they like. The +# Tahoe-LAFS client uses a SHA256-based construction. The server then uses +# blake2b to hash these values for storage so that it retains no persistent +# copy of the original secret. +# + def _fix_lease_count_format(lease_count_format): """ Turn a single character struct format string into a format string suitable diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 759752bed..fc823507a 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -13,8 +13,14 @@ if PY2: import struct +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + from .lease import ( LeaseInfo, + HashedLeaseInfo, ) def _header(version, max_size): @@ -22,10 +28,10 @@ def _header(version, max_size): """ Construct the header for an immutable container. - :param version: The container version to include the in header. - :param max_size: The maximum data size the container will hold. + :param version: the container version to include the in header + :param max_size: the maximum data size the container will hold - :return: Some bytes to write at the beginning of the container. + :return: some bytes to write at the beginning of the container """ # The second field -- the four-byte share data length -- is no longer # used as of Tahoe v1.3.0, but we continue to write it in there in @@ -38,6 +44,97 @@ def _header(version, max_size): # the share. return struct.pack(">LLL", version, min(2**32 - 1, max_size), 0) + +class _V2(object): + """ + Implement encoding and decoding for v2 of the immutable container. + """ + version = 2 + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + @classmethod + def header(cls, max_size): + # type: (int) -> bytes + """ + Construct a container header. + + :param max_size: the maximum size the container can hold + + :return: the header bytes + """ + return _header(cls.version, max_size) + + @classmethod + def serialize_lease(cls, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + """ + Serialize a lease to be written to a v2 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + if isinstance(lease, LeaseInfo): + # v2 of the immutable schema stores lease secrets hashed. If + # we're given a LeaseInfo then it holds plaintext secrets. Hash + # them before trying to serialize. + lease = cls._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return lease.to_immutable_data() + raise ValueError( + "ShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> HashedLeaseInfo + """ + Unserialize some bytes from a v2 container. + + :param data: the bytes from the container + + :return: the ``HashedLeaseInfo`` the bytes represent + """ + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + return HashedLeaseInfo(LeaseInfo.from_immutable_data(data), cls._hash_secret) + + class _V1(object): """ Implement encoding and decoding for v1 of the immutable container. @@ -66,7 +163,7 @@ class _V1(object): return LeaseInfo.from_immutable_data(data) -ALL_SCHEMAS = {_V1} +ALL_SCHEMAS = {_V2, _V1} ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 23071707a..895a0970c 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -20,6 +20,10 @@ from zope.interface import ( implementer, ) +from twisted.python.components import ( + proxyForInterface, +) + from allmydata.util.hashutil import timing_safe_compare # struct format for representation of a lease in an immutable share @@ -245,3 +249,81 @@ class LeaseInfo(object): ] values = struct.unpack(">LL32s32s20s", data) return cls(**dict(zip(names, values))) + + +@attr.s(frozen=True) +class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): + """ + A ``HashedLeaseInfo`` wraps lease information in which the secrets have + been hashed. + """ + _lease_info = attr.ib() + _hash = attr.ib() + + def is_renew_secret(self, candidate_secret): + """ + Hash the candidate secret and compare the result to the stored hashed + secret. + """ + return super(HashedLeaseInfo, self).is_renew_secret(self._hash(candidate_secret)) + + def is_cancel_secret(self, candidate_secret): + """ + Hash the candidate secret and compare the result to the stored hashed + secret. + """ + if isinstance(candidate_secret, _HashedCancelSecret): + # Someone read it off of this object in this project - probably + # the lease crawler - and is just trying to use it to identify + # which lease it wants to operate on. Avoid re-hashing the value. + # + # It is important that this codepath is only availably internally + # for this process to talk to itself. If it were to be exposed to + # clients over the network, they could just provide the hashed + # value to avoid having to ever learn the original value. + hashed_candidate = candidate_secret.hashed_value + else: + # It is not yet hashed so hash it. + hashed_candidate = self._hash(candidate_secret) + + return super(HashedLeaseInfo, self).is_cancel_secret(hashed_candidate) + + @property + def owner_num(self): + return self._lease_info.owner_num + + @property + def cancel_secret(self): + """ + Give back an opaque wrapper around the hashed cancel secret which can + later be presented for a succesful equality comparison. + """ + # We don't *have* the cancel secret. We hashed it and threw away the + # original. That's good. It does mean that some code that runs + # in-process with the storage service (LeaseCheckingCrawler) runs into + # some difficulty. That code wants to cancel leases and does so using + # the same interface that faces storage clients (or would face them, + # if lease cancellation were exposed). + # + # Since it can't use the hashed secret to cancel a lease (that's the + # point of the hashing) and we don't have the unhashed secret to give + # it, instead we give it a marker that `cancel_lease` will recognize. + # On recognizing it, if the hashed value given matches the hashed + # value stored it is considered a match and the lease can be + # cancelled. + # + # This isn't great. Maybe the internal and external consumers of + # cancellation should use different interfaces. + return _HashedCancelSecret(self._lease_info.cancel_secret) + + +@attr.s(frozen=True) +class _HashedCancelSecret(object): + """ + ``_HashedCancelSecret`` is a marker type for an already-hashed lease + cancel secret that lets internal lease cancellers bypass the hash-based + protection that's imposed on external lease cancellers. + + :ivar bytes hashed_value: The already-hashed secret. + """ + hashed_value = attr.ib() diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index ca5b5650b..85d89cde6 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -1113,9 +1113,17 @@ class Corruption(_Base, unittest.TestCase): d.addCallback(_download, imm_uri, i, expected) d.addCallback(lambda ign: self.restore_all_shares(self.shares)) d.addCallback(fireEventually) - corrupt_values = [(3, 2, "no-sh2"), - (15, 2, "need-4th"), # share looks v2 - ] + corrupt_values = [ + # Make the container version for share number 2 look + # unsupported. If you add support for immutable share file + # version number much past 16 million then you will have to + # update this test. Also maybe you have other problems. + (1, 255, "no-sh2"), + # Make the immutable share number 2 (not the container, the + # thing inside the container) look unsupported. Ditto the + # above about version numbers in the ballpark of 16 million. + (13, 255, "need-4th"), + ] for i,newvalue,expected in corrupt_values: d.addCallback(self._corrupt_set, imm_uri, i, newvalue) d.addCallback(_download, imm_uri, i, expected) From 7a59aa83bb9e429d0b44f47fff6365dbfa24f42f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:12:54 -0400 Subject: [PATCH 144/916] add missing import --- src/allmydata/storage/immutable_schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index fc823507a..6ac49f6f1 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -13,6 +13,11 @@ if PY2: import struct +try: + from typing import Union +except ImportError: + pass + import attr from nacl.hash import blake2b From 6889ab2a76a1665dc7adb11dfa2205760641f303 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:16:55 -0400 Subject: [PATCH 145/916] fix syntax of type hint --- src/allmydata/storage/immutable_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 6ac49f6f1..7ffec418a 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -29,7 +29,7 @@ from .lease import ( ) def _header(version, max_size): - # (int, int) -> bytes + # type: (int, int) -> bytes """ Construct the header for an immutable container. From 2186bfcc372d01ab79f6899e8d0a54157ee83444 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:40:43 -0400 Subject: [PATCH 146/916] silence some mypy errors :/ I don't know the "right" way to make mypy happy with these things --- src/allmydata/storage/immutable_schema.py | 4 ++-- src/allmydata/storage/lease.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 7ffec418a..440755b01 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -169,8 +169,8 @@ class _V1(object): ALL_SCHEMAS = {_V2, _V1} -ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} -NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore def schema_from_version(version): # (int) -> Optional[type] diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 895a0970c..63dba15e8 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -252,7 +252,7 @@ class LeaseInfo(object): @attr.s(frozen=True) -class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): +class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ignore # unsupported dynamic base class """ A ``HashedLeaseInfo`` wraps lease information in which the secrets have been hashed. From 931ddf85a532178ab83584820eda8605a495d5ab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 15:26:58 -0400 Subject: [PATCH 147/916] introduce an explicit representation of the v1 mutable container schema This is only a partial representation, sufficient to express the changes that are coming in v2. --- src/allmydata/storage/mutable.py | 46 +++------ src/allmydata/storage/mutable_schema.py | 119 ++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 src/allmydata/storage/mutable_schema.py diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index ce9cc5ff4..346edd53a 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -24,7 +24,10 @@ from allmydata.storage.lease import LeaseInfo from allmydata.storage.common import UnknownMutableContainerVersionError, \ DataTooLargeError from allmydata.mutable.layout import MAX_MUTABLE_SHARE_SIZE - +from .mutable_schema import ( + NEWEST_SCHEMA_VERSION, + schema_from_header, +) # the MutableShareFile is like the ShareFile, but used for mutable data. It # has a different layout. See docs/mutable.txt for more details. @@ -64,9 +67,6 @@ class MutableShareFile(object): # our sharefiles share with a recognizable string, plus some random # binary data to reduce the chance that a regular text file will look # like a sharefile. - MAGIC = b"Tahoe mutable container v1\n" + b"\x75\x09\x44\x03\x8e" - assert len(MAGIC) == 32 - assert isinstance(MAGIC, bytes) MAX_SIZE = MAX_MUTABLE_SHARE_SIZE # TODO: decide upon a policy for max share size @@ -82,20 +82,19 @@ class MutableShareFile(object): :return: ``True`` if the bytes could belong to this container, ``False`` otherwise. """ - return header.startswith(cls.MAGIC) + return schema_from_header(header) is not None - def __init__(self, filename, parent=None): + def __init__(self, filename, parent=None, schema=NEWEST_SCHEMA_VERSION): self.home = filename if os.path.exists(self.home): # we don't cache anything, just check the magic with open(self.home, 'rb') as f: - data = f.read(self.HEADER_SIZE) - (magic, - write_enabler_nodeid, write_enabler, - data_length, extra_least_offset) = \ - struct.unpack(">32s20s32sQQ", data) - if not self.is_valid_header(data): - raise UnknownMutableContainerVersionError(filename, magic) + header = f.read(self.HEADER_SIZE) + self._schema = schema_from_header(header) + if self._schema is None: + raise UnknownMutableContainerVersionError(filename, header) + else: + self._schema = schema self.parent = parent # for logging def log(self, *args, **kwargs): @@ -103,23 +102,8 @@ class MutableShareFile(object): def create(self, my_nodeid, write_enabler): assert not os.path.exists(self.home) - data_length = 0 - extra_lease_offset = (self.HEADER_SIZE - + 4 * self.LEASE_SIZE - + data_length) - assert extra_lease_offset == self.DATA_OFFSET # true at creation - num_extra_leases = 0 with open(self.home, 'wb') as f: - header = struct.pack( - ">32s20s32sQQ", - self.MAGIC, my_nodeid, write_enabler, - data_length, extra_lease_offset, - ) - leases = (b"\x00" * self.LEASE_SIZE) * 4 - f.write(header + leases) - # data goes here, empty after creation - f.write(struct.pack(">L", num_extra_leases)) - # extra leases go here, none at creation + f.write(self._schema.header(my_nodeid, write_enabler)) def unlink(self): os.unlink(self.home) @@ -252,7 +236,7 @@ class MutableShareFile(object): + (lease_number-4)*self.LEASE_SIZE) f.seek(offset) assert f.tell() == offset - f.write(lease_info.to_mutable_data()) + f.write(self._schema.serialize_lease(lease_info)) def _read_lease_record(self, f, lease_number): # returns a LeaseInfo instance, or None @@ -269,7 +253,7 @@ class MutableShareFile(object): f.seek(offset) assert f.tell() == offset data = f.read(self.LEASE_SIZE) - lease_info = LeaseInfo.from_mutable_data(data) + lease_info = self._schema.unserialize_lease(data) if lease_info.owner_num == 0: return None return lease_info diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py new file mode 100644 index 000000000..25f24ea1f --- /dev/null +++ b/src/allmydata/storage/mutable_schema.py @@ -0,0 +1,119 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import struct + +from .lease import ( + LeaseInfo, +) + +class _V1(object): + """ + Implement encoding and decoding for v1 of the mutable container. + """ + version = 1 + + _MAGIC = ( + # Make it easy for people to recognize + b"Tahoe mutable container v1\n" + # But also keep the chance of accidental collision low + b"\x75\x09\x44\x03\x8e" + ) + assert len(_MAGIC) == 32 + + _HEADER_FORMAT = ">32s20s32sQQ" + + # This size excludes leases + _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + + _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + @classmethod + def magic_matches(cls, candidate_magic): + # type: (bytes) -> bool + """ + Return ``True`` if a candidate string matches the expected magic string + from a mutable container header, ``False`` otherwise. + """ + return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + + @classmethod + def header(cls, nodeid, write_enabler): + # type: (bytes, bytes) -> bytes + """ + Construct a container header. + + :param nodeid: A unique identifier for the node holding this + container. + + :param write_enabler: A secret shared with the client used to + authorize changes to the contents of this container. + """ + fixed_header = struct.pack( + ">32s20s32sQQ", + cls._MAGIC, + nodeid, + write_enabler, + # data length, initially the container is empty + 0, + cls._EXTRA_LEASE_OFFSET, + ) + blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 + extra_lease_count = struct.pack(">L", 0) + + return b"".join([ + fixed_header, + # share data will go in between the next two items eventually but + # for now there is none. + blank_leases, + extra_lease_count, + ]) + + @classmethod + def serialize_lease(cls, lease_info): + # type: (LeaseInfo) -> bytes + """ + Serialize a lease to be written to a v1 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + return lease_info.to_mutable_data() + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> LeaseInfo + """ + Unserialize some bytes from a v1 container. + + :param data: the bytes from the container + + :return: the ``LeaseInfo`` the bytes represent + """ + return LeaseInfo.from_mutable_data(data) + + +ALL_SCHEMAS = {_V1} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore + +def schema_from_header(header): + # (int) -> Optional[type] + """ + Find the schema object that corresponds to a certain version number. + """ + for schema in ALL_SCHEMAS: + if schema.magic_matches(header): + return schema + return None From 728638fe230dfdf0149c5835b0a8077230dbf021 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 15:37:29 -0400 Subject: [PATCH 148/916] apply the MutableShareFile tests to all known schemas --- src/allmydata/test/test_storage.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 655395042..fbd005050 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -42,9 +42,12 @@ from allmydata.util import fileutil, hashutil, base32 from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.mutable_schema import ( + ALL_SCHEMAS as ALL_MUTABLE_SCHEMAS, +) from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile from allmydata.storage.immutable_schema import ( - ALL_SCHEMAS, + ALL_SCHEMAS as ALL_IMMUTABLE_SCHEMAS, ) from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ @@ -3131,7 +3134,7 @@ class Stats(unittest.TestCase): self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) -immutable_schemas = strategies.sampled_from(list(ALL_SCHEMAS)) +immutable_schemas = strategies.sampled_from(list(ALL_IMMUTABLE_SCHEMAS)) class ShareFileTests(unittest.TestCase): """Tests for allmydata.storage.immutable.ShareFile.""" @@ -3270,15 +3273,17 @@ class ShareFileTests(unittest.TestCase): (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_cancel_secret(cancel_secret)) +mutable_schemas = strategies.sampled_from(list(ALL_MUTABLE_SCHEMAS)) class MutableShareFileTests(unittest.TestCase): """ Tests for allmydata.storage.mutable.MutableShareFile. """ - def get_sharefile(self): - return MutableShareFile(self.mktemp()) + def get_sharefile(self, **kwargs): + return MutableShareFile(self.mktemp(), **kwargs) @given( + schema=mutable_schemas, nodeid=strategies.just(b"x" * 20), write_enabler=strategies.just(b"y" * 32), datav=strategies.lists( @@ -3289,12 +3294,12 @@ class MutableShareFileTests(unittest.TestCase): ), new_length=offsets(), ) - def test_readv_reads_share_data(self, nodeid, write_enabler, datav, new_length): + def test_readv_reads_share_data(self, schema, nodeid, write_enabler, datav, new_length): """ ``MutableShareFile.readv`` returns bytes from the share data portion of the share file. """ - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) sf.create(my_nodeid=nodeid, write_enabler=write_enabler) sf.writev(datav=datav, new_length=new_length) @@ -3329,12 +3334,13 @@ class MutableShareFileTests(unittest.TestCase): self.assertEqual(expected_data, read_data) @given( + schema=mutable_schemas, nodeid=strategies.just(b"x" * 20), write_enabler=strategies.just(b"y" * 32), readv=strategies.lists(strategies.tuples(offsets(), lengths()), min_size=1), random=strategies.randoms(), ) - def test_readv_rejects_negative_length(self, nodeid, write_enabler, readv, random): + def test_readv_rejects_negative_length(self, schema, nodeid, write_enabler, readv, random): """ If a negative length is given to ``MutableShareFile.readv`` in a read vector then ``AssertionError`` is raised. @@ -3373,7 +3379,7 @@ class MutableShareFileTests(unittest.TestCase): *broken_readv[readv_index] ) - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) sf.create(my_nodeid=nodeid, write_enabler=write_enabler) # A read with a broken read vector is an error. From 8adff050a7f179c6c4796f4d3b04fab60924cbad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 13:51:46 -0400 Subject: [PATCH 149/916] compare without breaking out all of the fields HashedLeaseInfo doesn't have all of these attributes --- src/allmydata/test/test_storage.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index fbd005050..92176ce52 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1361,14 +1361,21 @@ class MutableServer(unittest.TestCase): 2: [b"2"*10]}) def compare_leases_without_timestamps(self, leases_a, leases_b): - self.failUnlessEqual(len(leases_a), len(leases_b)) - for i in range(len(leases_a)): - a = leases_a[i] - b = leases_b[i] - self.failUnlessEqual(a.owner_num, b.owner_num) - self.failUnlessEqual(a.renew_secret, b.renew_secret) - self.failUnlessEqual(a.cancel_secret, b.cancel_secret) - self.failUnlessEqual(a.nodeid, b.nodeid) + for a, b in zip(leases_a, leases_b): + # The leases aren't always of the same type (though of course + # corresponding elements in the two lists should be of the same + # type as each other) so it's inconvenient to just reach in and + # normalize the expiration timestamp. We don't want to call + # `renew` on both objects to normalize the expiration timestamp in + # case `renew` is broken and gives us back equal outputs from + # non-equal inputs (expiration timestamp aside). It seems + # reasonably safe to use `renew` to make _one_ of the timestamps + # equal to the other though. + self.assertEqual( + a.renew(b.get_expiration_time()), + b, + ) + self.assertEqual(len(leases_a), len(leases_b)) def test_leases(self): ss = self.create("test_leases") From 0cd96ed713ba6429b76e2520752acb7e8e166e40 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:09:46 -0400 Subject: [PATCH 150/916] fix the debug tool for the hashed lease secret case --- src/allmydata/scripts/debug.py | 4 ++-- src/allmydata/storage/lease.py | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 260cca55b..6201ce28f 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -230,8 +230,8 @@ def dump_mutable_share(options): print(" ownerid: %d" % lease.owner_num, file=out) when = format_expiration_time(lease.get_expiration_time()) print(" expires in %s" % when, file=out) - print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) - print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) + print(" renew_secret: %s" % lease.present_renew_secret(), file=out) + print(" cancel_secret: %s" % lease.present_cancel_secret(), file=out) print(" secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out) else: print("No leases.", file=out) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 63dba15e8..3ec760dbe 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -25,6 +25,7 @@ from twisted.python.components import ( ) from allmydata.util.hashutil import timing_safe_compare +from allmydata.util import base32 # struct format for representation of a lease in an immutable share IMMUTABLE_FORMAT = ">L32s32sL" @@ -102,12 +103,24 @@ class ILeaseInfo(Interface): secret, ``False`` otherwise """ + def present_renew_secret(): + """ + :return str: Text which could reasonably be shown to a person representing + this lease's renew secret. + """ + def is_cancel_secret(candidate_secret): """ :return bool: ``True`` if the given byte string is this lease's cancel secret, ``False`` otherwise """ + def present_cancel_secret(): + """ + :return str: Text which could reasonably be shown to a person representing + this lease's cancel secret. + """ + @implementer(ILeaseInfo) @attr.s(frozen=True) @@ -173,6 +186,13 @@ class LeaseInfo(object): """ return timing_safe_compare(self.renew_secret, candidate_secret) + def present_renew_secret(self): + # type: () -> bytes + """ + Return the renew secret, base32-encoded. + """ + return str(base32.b2a(self.renew_secret), "utf-8") + def is_cancel_secret(self, candidate_secret): # type: (bytes) -> bool """ @@ -183,6 +203,13 @@ class LeaseInfo(object): """ return timing_safe_compare(self.cancel_secret, candidate_secret) + def present_cancel_secret(self): + # type: () -> bytes + """ + Return the cancel secret, base32-encoded. + """ + return str(base32.b2a(self.cancel_secret), "utf-8") + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self._expiration_time - 31*24*60*60 @@ -267,6 +294,12 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign """ return super(HashedLeaseInfo, self).is_renew_secret(self._hash(candidate_secret)) + def present_renew_secret(self): + """ + Present the hash of the secret with a marker indicating it is a hash. + """ + return u"hash:" + super(HashedLeaseInfo, self).present_renew_secret() + def is_cancel_secret(self, candidate_secret): """ Hash the candidate secret and compare the result to the stored hashed @@ -288,10 +321,20 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign return super(HashedLeaseInfo, self).is_cancel_secret(hashed_candidate) + def present_cancel_secret(self): + """ + Present the hash of the secret with a marker indicating it is a hash. + """ + return u"hash:" + super(HashedLeaseInfo, self).present_cancel_secret() + @property def owner_num(self): return self._lease_info.owner_num + @property + def nodeid(self): + return self._lease_info.nodeid + @property def cancel_secret(self): """ From 5d703d989339587cfd5706fea1728ecb59e17808 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:10:27 -0400 Subject: [PATCH 151/916] some type annotations --- src/allmydata/storage/lease.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 3ec760dbe..9ddbc9c68 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -187,7 +187,7 @@ class LeaseInfo(object): return timing_safe_compare(self.renew_secret, candidate_secret) def present_renew_secret(self): - # type: () -> bytes + # type: () -> str """ Return the renew secret, base32-encoded. """ @@ -204,7 +204,7 @@ class LeaseInfo(object): return timing_safe_compare(self.cancel_secret, candidate_secret) def present_cancel_secret(self): - # type: () -> bytes + # type: () -> str """ Return the cancel secret, base32-encoded. """ @@ -288,6 +288,7 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign _hash = attr.ib() def is_renew_secret(self, candidate_secret): + # type: (bytes) -> bool """ Hash the candidate secret and compare the result to the stored hashed secret. @@ -295,12 +296,14 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign return super(HashedLeaseInfo, self).is_renew_secret(self._hash(candidate_secret)) def present_renew_secret(self): + # type: () -> str """ Present the hash of the secret with a marker indicating it is a hash. """ return u"hash:" + super(HashedLeaseInfo, self).present_renew_secret() def is_cancel_secret(self, candidate_secret): + # type: (bytes) -> bool """ Hash the candidate secret and compare the result to the stored hashed secret. @@ -322,6 +325,7 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign return super(HashedLeaseInfo, self).is_cancel_secret(hashed_candidate) def present_cancel_secret(self): + # type: () -> str """ Present the hash of the secret with a marker indicating it is a hash. """ From 3de9c73b0b066e5e15978a15c2903d10e398ed0a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:11:05 -0400 Subject: [PATCH 152/916] preserve the type when renewing HashedLeaseInfo does this mean immutable lease renewal is untested? maybe --- src/allmydata/storage/lease.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 9ddbc9c68..1a5416d6a 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -287,6 +287,13 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign _lease_info = attr.ib() _hash = attr.ib() + def renew(self, new_expire_time): + # Preserve the HashedLeaseInfo wrapper around the renewed LeaseInfo. + return attr.assoc( + self, + _lease_info=super(HashedLeaseInfo, self).renew(new_expire_time), + ) + def is_renew_secret(self, candidate_secret): # type: (bytes) -> bool """ From 456df65a07a0c48f3a056519282cc96b5e4e2f25 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:16:43 -0400 Subject: [PATCH 153/916] Add v2 of the mutable container schema It uses hashed lease secrets, like v2 of the immutable container schema. --- src/allmydata/storage/mutable_schema.py | 225 ++++++++++++++++++++---- 1 file changed, 187 insertions(+), 38 deletions(-) diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index 25f24ea1f..9496fe571 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -13,23 +13,193 @@ if PY2: import struct +try: + from typing import Union +except ImportError: + pass + +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + +from ..util.hashutil import ( + tagged_hash, +) from .lease import ( LeaseInfo, + HashedLeaseInfo, ) +def _magic(version): + # type: (int) -> bytes + """ + Compute a "magic" header string for a container of the given version. + + :param version: The version number of the container. + """ + # Make it easy for people to recognize + human_readable = u"Tahoe mutable container v{:d}\n".format(version).encode("ascii") + # But also keep the chance of accidental collision low + if version == 1: + # It's unclear where this byte sequence came from. It may have just + # been random. In any case, preserve it since it is the magic marker + # in all v1 share files. + random_bytes = b"\x75\x09\x44\x03\x8e" + else: + # For future versions, use a reproducable scheme. + random_bytes = tagged_hash( + b"allmydata_mutable_container_header", + human_readable, + truncate_to=5, + ) + magic = human_readable + random_bytes + assert len(magic) == 32 + if version > 1: + # The chance of collision is pretty low but let's just be sure about + # it. + assert magic != _magic(version - 1) + + return magic + +def _header(magic, extra_lease_offset, nodeid, write_enabler): + # type: (bytes, int, bytes, bytes) -> bytes + """ + Construct a container header. + + :param nodeid: A unique identifier for the node holding this + container. + + :param write_enabler: A secret shared with the client used to + authorize changes to the contents of this container. + """ + fixed_header = struct.pack( + ">32s20s32sQQ", + magic, + nodeid, + write_enabler, + # data length, initially the container is empty + 0, + extra_lease_offset, + ) + blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 + extra_lease_count = struct.pack(">L", 0) + + return b"".join([ + fixed_header, + # share data will go in between the next two items eventually but + # for now there is none. + blank_leases, + extra_lease_count, + ]) + + +class _V2(object): + """ + Implement encoding and decoding for v2 of the mutable container. + """ + version = 2 + _MAGIC = _magic(version) + + _HEADER_FORMAT = ">32s20s32sQQ" + + # This size excludes leases + _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + + _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + @classmethod + def magic_matches(cls, candidate_magic): + # type: (bytes) -> bool + """ + Return ``True`` if a candidate string matches the expected magic string + from a mutable container header, ``False`` otherwise. + """ + return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + + @classmethod + def header(cls, nodeid, write_enabler): + return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) + + @classmethod + def serialize_lease(cls, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + """ + Serialize a lease to be written to a v2 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + if isinstance(lease, LeaseInfo): + # v2 of the mutable schema stores lease secrets hashed. If we're + # given a LeaseInfo then it holds plaintext secrets. Hash them + # before trying to serialize. + lease = cls._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return lease.to_mutable_data() + raise ValueError( + "MutableShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> HashedLeaseInfo + """ + Unserialize some bytes from a v2 container. + + :param data: the bytes from the container + + :return: the ``HashedLeaseInfo`` the bytes represent + """ + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + lease = LeaseInfo.from_mutable_data(data) + return HashedLeaseInfo(lease, cls._hash_secret) + + class _V1(object): """ Implement encoding and decoding for v1 of the mutable container. """ version = 1 - - _MAGIC = ( - # Make it easy for people to recognize - b"Tahoe mutable container v1\n" - # But also keep the chance of accidental collision low - b"\x75\x09\x44\x03\x8e" - ) - assert len(_MAGIC) == 32 + _MAGIC = _magic(version) _HEADER_FORMAT = ">32s20s32sQQ" @@ -49,35 +219,8 @@ class _V1(object): @classmethod def header(cls, nodeid, write_enabler): - # type: (bytes, bytes) -> bytes - """ - Construct a container header. + return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) - :param nodeid: A unique identifier for the node holding this - container. - - :param write_enabler: A secret shared with the client used to - authorize changes to the contents of this container. - """ - fixed_header = struct.pack( - ">32s20s32sQQ", - cls._MAGIC, - nodeid, - write_enabler, - # data length, initially the container is empty - 0, - cls._EXTRA_LEASE_OFFSET, - ) - blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 - extra_lease_count = struct.pack(">L", 0) - - return b"".join([ - fixed_header, - # share data will go in between the next two items eventually but - # for now there is none. - blank_leases, - extra_lease_count, - ]) @classmethod def serialize_lease(cls, lease_info): @@ -89,7 +232,13 @@ class _V1(object): :return: the serialized bytes """ - return lease_info.to_mutable_data() + if isinstance(lease, LeaseInfo): + return lease_info.to_mutable_data() + raise ValueError( + "MutableShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) @classmethod def unserialize_lease(cls, data): @@ -104,7 +253,7 @@ class _V1(object): return LeaseInfo.from_mutable_data(data) -ALL_SCHEMAS = {_V1} +ALL_SCHEMAS = {_V2, _V1} ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore From 617a1eac9d848661b1fce2fe18976796ce02ac2a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 15:30:49 -0400 Subject: [PATCH 154/916] refactor lease hashing logic to avoid mutable/immutable duplication --- src/allmydata/storage/immutable.py | 4 +- src/allmydata/storage/immutable_schema.py | 169 ++++--------------- src/allmydata/storage/lease.py | 4 +- src/allmydata/storage/lease_schema.py | 129 ++++++++++++++ src/allmydata/storage/mutable.py | 4 +- src/allmydata/storage/mutable_schema.py | 194 ++++------------------ 6 files changed, 199 insertions(+), 305 deletions(-) create mode 100644 src/allmydata/storage/lease_schema.py diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 216262a81..e9992d96e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -231,7 +231,7 @@ class ShareFile(object): offset = self._lease_offset + lease_number * self.LEASE_SIZE f.seek(offset) assert f.tell() == offset - f.write(self._schema.serialize_lease(lease_info)) + f.write(self._schema.lease_serializer.serialize(lease_info)) def _read_num_leases(self, f): f.seek(0x08) @@ -262,7 +262,7 @@ class ShareFile(object): for i in range(num_leases): data = f.read(self.LEASE_SIZE) if data: - yield self._schema.unserialize_lease(data) + yield self._schema.lease_serializer.unserialize(data) def add_lease(self, lease_info): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 440755b01..40663b935 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -13,84 +13,28 @@ if PY2: import struct -try: - from typing import Union -except ImportError: - pass - import attr -from nacl.hash import blake2b -from nacl.encoding import RawEncoder - -from .lease import ( - LeaseInfo, - HashedLeaseInfo, +from .lease_schema import ( + v1_immutable, + v2_immutable, ) -def _header(version, max_size): - # type: (int, int) -> bytes +@attr.s(frozen=True) +class _Schema(object): """ - Construct the header for an immutable container. + Implement encoding and decoding for multiple versions of the immutable + container schema. - :param version: the container version to include the in header - :param max_size: the maximum data size the container will hold + :ivar int version: the version number of the schema this object supports - :return: some bytes to write at the beginning of the container + :ivar lease_serializer: an object that is responsible for lease + serialization and unserialization """ - # The second field -- the four-byte share data length -- is no longer - # used as of Tahoe v1.3.0, but we continue to write it in there in - # case someone downgrades a storage server from >= Tahoe-1.3.0 to < - # Tahoe-1.3.0, or moves a share file from one server to another, - # etc. We do saturation -- a share data length larger than 2**32-1 - # (what can fit into the field) is marked as the largest length that - # can fit into the field. That way, even if this does happen, the old - # < v1.3.0 server will still allow clients to read the first part of - # the share. - return struct.pack(">LLL", version, min(2**32 - 1, max_size), 0) + version = attr.ib() + lease_serializer = attr.ib() - -class _V2(object): - """ - Implement encoding and decoding for v2 of the immutable container. - """ - version = 2 - - @classmethod - def _hash_secret(cls, secret): - # type: (bytes) -> bytes - """ - Hash a lease secret for storage. - """ - return blake2b(secret, digest_size=32, encoder=RawEncoder()) - - @classmethod - def _hash_lease_info(cls, lease_info): - # type: (LeaseInfo) -> HashedLeaseInfo - """ - Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. - """ - if not isinstance(lease_info, LeaseInfo): - # Provide a little safety against misuse, especially an attempt to - # re-hash an already-hashed lease info which is represented as a - # different type. - raise TypeError( - "Can only hash LeaseInfo, not {!r}".format(lease_info), - ) - - # Hash the cleartext secrets in the lease info and wrap the result in - # a new type. - return HashedLeaseInfo( - attr.assoc( - lease_info, - renew_secret=cls._hash_secret(lease_info.renew_secret), - cancel_secret=cls._hash_secret(lease_info.cancel_secret), - ), - cls._hash_secret, - ) - - @classmethod - def header(cls, max_size): + def header(self, max_size): # type: (int) -> bytes """ Construct a container header. @@ -99,78 +43,23 @@ class _V2(object): :return: the header bytes """ - return _header(cls.version, max_size) + # The second field -- the four-byte share data length -- is no longer + # used as of Tahoe v1.3.0, but we continue to write it in there in + # case someone downgrades a storage server from >= Tahoe-1.3.0 to < + # Tahoe-1.3.0, or moves a share file from one server to another, + # etc. We do saturation -- a share data length larger than 2**32-1 + # (what can fit into the field) is marked as the largest length that + # can fit into the field. That way, even if this does happen, the old + # < v1.3.0 server will still allow clients to read the first part of + # the share. + return struct.pack(">LLL", self.version, min(2**32 - 1, max_size), 0) - @classmethod - def serialize_lease(cls, lease): - # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes - """ - Serialize a lease to be written to a v2 container. - - :param lease: the lease to serialize - - :return: the serialized bytes - """ - if isinstance(lease, LeaseInfo): - # v2 of the immutable schema stores lease secrets hashed. If - # we're given a LeaseInfo then it holds plaintext secrets. Hash - # them before trying to serialize. - lease = cls._hash_lease_info(lease) - if isinstance(lease, HashedLeaseInfo): - return lease.to_immutable_data() - raise ValueError( - "ShareFile v2 schema cannot represent lease {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # type: (bytes) -> HashedLeaseInfo - """ - Unserialize some bytes from a v2 container. - - :param data: the bytes from the container - - :return: the ``HashedLeaseInfo`` the bytes represent - """ - # In v2 of the immutable schema lease secrets are stored hashed. Wrap - # a LeaseInfo in a HashedLeaseInfo so it can supply the correct - # interpretation for those values. - return HashedLeaseInfo(LeaseInfo.from_immutable_data(data), cls._hash_secret) - - -class _V1(object): - """ - Implement encoding and decoding for v1 of the immutable container. - """ - version = 1 - - @classmethod - def header(cls, max_size): - return _header(cls.version, max_size) - - @classmethod - def serialize_lease(cls, lease): - if isinstance(lease, LeaseInfo): - return lease.to_immutable_data() - raise ValueError( - "ShareFile v1 schema only supports LeaseInfo, not {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # In v1 of the immutable schema lease secrets are stored plaintext. - # So load the data into a plain LeaseInfo which works on plaintext - # secrets. - return LeaseInfo.from_immutable_data(data) - - -ALL_SCHEMAS = {_V2, _V1} -ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore -NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore +ALL_SCHEMAS = { + _Schema(version=2, lease_serializer=v2_immutable), + _Schema(version=1, lease_serializer=v1_immutable), +} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) def schema_from_version(version): # (int) -> Optional[type] diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 1a5416d6a..8be44bafd 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -230,7 +230,7 @@ class LeaseInfo(object): "cancel_secret", "expiration_time", ] - values = struct.unpack(">L32s32sL", data) + values = struct.unpack(IMMUTABLE_FORMAT, data) return cls(nodeid=None, **dict(zip(names, values))) def immutable_size(self): @@ -274,7 +274,7 @@ class LeaseInfo(object): "cancel_secret", "nodeid", ] - values = struct.unpack(">LL32s32s20s", data) + values = struct.unpack(MUTABLE_FORMAT, data) return cls(**dict(zip(names, values))) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py new file mode 100644 index 000000000..697ac9e34 --- /dev/null +++ b/src/allmydata/storage/lease_schema.py @@ -0,0 +1,129 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +try: + from typing import Union +except ImportError: + pass + +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + +from .lease import ( + LeaseInfo, + HashedLeaseInfo, +) + +@attr.s(frozen=True) +class CleartextLeaseSerializer(object): + _to_data = attr.ib() + _from_data = attr.ib() + + def serialize(self, lease): + # type: (LeaseInfo) -> bytes + if isinstance(lease, LeaseInfo): + return self._to_data(lease) + raise ValueError( + "ShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) + + def unserialize(self, data): + # type: (bytes) -> LeaseInfo + # In v1 of the immutable schema lease secrets are stored plaintext. + # So load the data into a plain LeaseInfo which works on plaintext + # secrets. + return self._from_data(data) + +@attr.s(frozen=True) +class HashedLeaseSerializer(object): + _to_data = attr.ib() + _from_data = attr.ib() + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + def serialize(self, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + if isinstance(lease, LeaseInfo): + # v2 of the immutable schema stores lease secrets hashed. If + # we're given a LeaseInfo then it holds plaintext secrets. Hash + # them before trying to serialize. + lease = self._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return self._to_data(lease) + raise ValueError( + "ShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + def unserialize(self, data): + # type: (bytes) -> HashedLeaseInfo + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + return HashedLeaseInfo(self._from_data(data), self._hash_secret) + +v1_immutable = CleartextLeaseSerializer( + LeaseInfo.to_immutable_data, + LeaseInfo.from_immutable_data, +) + +v2_immutable = HashedLeaseSerializer( + HashedLeaseInfo.to_immutable_data, + LeaseInfo.from_immutable_data, +) + +v1_mutable = CleartextLeaseSerializer( + LeaseInfo.to_mutable_data, + LeaseInfo.from_mutable_data, +) + +v2_mutable = HashedLeaseSerializer( + HashedLeaseInfo.to_mutable_data, + LeaseInfo.from_mutable_data, +) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 346edd53a..bd59d96b8 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -236,7 +236,7 @@ class MutableShareFile(object): + (lease_number-4)*self.LEASE_SIZE) f.seek(offset) assert f.tell() == offset - f.write(self._schema.serialize_lease(lease_info)) + f.write(self._schema.lease_serializer.serialize(lease_info)) def _read_lease_record(self, f, lease_number): # returns a LeaseInfo instance, or None @@ -253,7 +253,7 @@ class MutableShareFile(object): f.seek(offset) assert f.tell() == offset data = f.read(self.LEASE_SIZE) - lease_info = self._schema.unserialize_lease(data) + lease_info = self._schema.lease_serializer.unserialize(data) if lease_info.owner_num == 0: return None return lease_info diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index 9496fe571..4be0d2137 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -13,22 +13,17 @@ if PY2: import struct -try: - from typing import Union -except ImportError: - pass - import attr -from nacl.hash import blake2b -from nacl.encoding import RawEncoder - from ..util.hashutil import ( tagged_hash, ) from .lease import ( LeaseInfo, - HashedLeaseInfo, +) +from .lease_schema import ( + v1_mutable, + v2_mutable, ) def _magic(version): @@ -94,168 +89,49 @@ def _header(magic, extra_lease_offset, nodeid, write_enabler): ]) -class _V2(object): +_HEADER_FORMAT = ">32s20s32sQQ" + +# This size excludes leases +_HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + +_EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + +@attr.s(frozen=True) +class _Schema(object): """ - Implement encoding and decoding for v2 of the mutable container. + Implement encoding and decoding for the mutable container. + + :ivar int version: the version number of the schema this object supports + + :ivar lease_serializer: an object that is responsible for lease + serialization and unserialization """ - version = 2 - _MAGIC = _magic(version) - - _HEADER_FORMAT = ">32s20s32sQQ" - - # This size excludes leases - _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) - - _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + version = attr.ib() + lease_serializer = attr.ib() + _magic = attr.ib() @classmethod - def _hash_secret(cls, secret): - # type: (bytes) -> bytes - """ - Hash a lease secret for storage. - """ - return blake2b(secret, digest_size=32, encoder=RawEncoder()) + def for_version(cls, version, lease_serializer): + return cls(version, lease_serializer, magic=_magic(version)) - @classmethod - def _hash_lease_info(cls, lease_info): - # type: (LeaseInfo) -> HashedLeaseInfo - """ - Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. - """ - if not isinstance(lease_info, LeaseInfo): - # Provide a little safety against misuse, especially an attempt to - # re-hash an already-hashed lease info which is represented as a - # different type. - raise TypeError( - "Can only hash LeaseInfo, not {!r}".format(lease_info), - ) - - # Hash the cleartext secrets in the lease info and wrap the result in - # a new type. - return HashedLeaseInfo( - attr.assoc( - lease_info, - renew_secret=cls._hash_secret(lease_info.renew_secret), - cancel_secret=cls._hash_secret(lease_info.cancel_secret), - ), - cls._hash_secret, - ) - - @classmethod - def magic_matches(cls, candidate_magic): + def magic_matches(self, candidate_magic): # type: (bytes) -> bool """ Return ``True`` if a candidate string matches the expected magic string from a mutable container header, ``False`` otherwise. """ - return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + return candidate_magic[:len(self._magic)] == self._magic - @classmethod - def header(cls, nodeid, write_enabler): - return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) + def header(self, nodeid, write_enabler): + return _header(self._magic, _EXTRA_LEASE_OFFSET, nodeid, write_enabler) - @classmethod - def serialize_lease(cls, lease): - # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes - """ - Serialize a lease to be written to a v2 container. - - :param lease: the lease to serialize - - :return: the serialized bytes - """ - if isinstance(lease, LeaseInfo): - # v2 of the mutable schema stores lease secrets hashed. If we're - # given a LeaseInfo then it holds plaintext secrets. Hash them - # before trying to serialize. - lease = cls._hash_lease_info(lease) - if isinstance(lease, HashedLeaseInfo): - return lease.to_mutable_data() - raise ValueError( - "MutableShareFile v2 schema cannot represent lease {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # type: (bytes) -> HashedLeaseInfo - """ - Unserialize some bytes from a v2 container. - - :param data: the bytes from the container - - :return: the ``HashedLeaseInfo`` the bytes represent - """ - # In v2 of the immutable schema lease secrets are stored hashed. Wrap - # a LeaseInfo in a HashedLeaseInfo so it can supply the correct - # interpretation for those values. - lease = LeaseInfo.from_mutable_data(data) - return HashedLeaseInfo(lease, cls._hash_secret) - - -class _V1(object): - """ - Implement encoding and decoding for v1 of the mutable container. - """ - version = 1 - _MAGIC = _magic(version) - - _HEADER_FORMAT = ">32s20s32sQQ" - - # This size excludes leases - _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) - - _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() - - @classmethod - def magic_matches(cls, candidate_magic): - # type: (bytes) -> bool - """ - Return ``True`` if a candidate string matches the expected magic string - from a mutable container header, ``False`` otherwise. - """ - return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC - - @classmethod - def header(cls, nodeid, write_enabler): - return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) - - - @classmethod - def serialize_lease(cls, lease_info): - # type: (LeaseInfo) -> bytes - """ - Serialize a lease to be written to a v1 container. - - :param lease: the lease to serialize - - :return: the serialized bytes - """ - if isinstance(lease, LeaseInfo): - return lease_info.to_mutable_data() - raise ValueError( - "MutableShareFile v1 schema only supports LeaseInfo, not {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # type: (bytes) -> LeaseInfo - """ - Unserialize some bytes from a v1 container. - - :param data: the bytes from the container - - :return: the ``LeaseInfo`` the bytes represent - """ - return LeaseInfo.from_mutable_data(data) - - -ALL_SCHEMAS = {_V2, _V1} -ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore -NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore +ALL_SCHEMAS = { + _Schema.for_version(version=2, lease_serializer=v2_mutable), + _Schema.for_version(version=1, lease_serializer=v1_mutable), +} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) def schema_from_header(header): # (int) -> Optional[type] From 66644791cbce41f31a08a7a9ba56449ccf02e33e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 15:36:26 -0400 Subject: [PATCH 155/916] news fragment --- newsfragments/3841.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3841.security diff --git a/newsfragments/3841.security b/newsfragments/3841.security new file mode 100644 index 000000000..867322e0a --- /dev/null +++ b/newsfragments/3841.security @@ -0,0 +1 @@ +The storage server now keeps hashes of lease renew and cancel secrets for mutable share files instead of keeping the original secrets. \ No newline at end of file From 8dd4aaebb6e579b4874951bc4b6c6218ed667b79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 10 Nov 2021 14:42:22 -0500 Subject: [PATCH 156/916] More consistent header system. --- docs/proposed/http-storage-node-protocol.rst | 106 +++++++++++-------- 1 file changed, 63 insertions(+), 43 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index fd1db5c4c..2a392fb20 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -363,11 +363,11 @@ one branch contains all of the share data; another branch contains all of the lease data; etc. -Authorization is required for all endpoints. +An ``Authorization`` header in requests is required for all endpoints. The standard HTTP authorization protocol is used. The authentication *type* used is ``Tahoe-LAFS``. The swissnum from the NURL used to locate the storage service is used as the *credentials*. -If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``UNAUTHORIZED`` response. +If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response. General ~~~~~~~ @@ -396,17 +396,26 @@ For example:: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Either renew or create a new lease on the bucket addressed by ``storage_index``. -The details of the lease are encoded in the request body. + +For a renewal, the renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. For example:: - {"renew-secret": "abcd", "cancel-secret": "efgh"} + X-Tahoe-Authorization: lease-renew-secret + X-Tahoe-Authorization: lease-cancel-secret -If the ``renew-secret`` value matches an existing lease -then the expiration time of that lease will be changed to 31 days after the time of this operation. -If it does not match an existing lease -then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. +For a new lease, ``X-Tahoe-Set-Authorization`` headers should be used instead. +For example:: -``renew-secret`` and ``cancel-secret`` values must be 32 bytes long. + X-Tahoe-Set-Authorization: lease-renew-secret + X-Tahoe-Set-Authorization: lease-cancel-secret + +For renewal, the expiration time of that lease will be changed to 31 days after the time of this operation. +If the renewal secret does not match, a new lease will be created, but clients should still not rely on this behavior if possible, and instead use the appropriate new lease headers. + +For the creation path, +then a new lease will be created with this ``lease-renew-secret`` which expires 31 days after the time of this operation. + +``lease-renew-secret`` and ``lease-cancel-secret`` values must be 32 bytes long. The server treats them as opaque values. :ref:`Share Leases` gives details about how the Tahoe-LAFS storage client constructs these values. @@ -423,7 +432,7 @@ In these cases the server takes no action and returns ``NOT FOUND``. Discussion `````````` -We considered an alternative where ``renew-secret`` and ``cancel-secret`` are placed in query arguments on the request path. +We considered an alternative where ``lease-renew-secret`` and ``lease-cancel-secret`` are placed in query arguments on the request path. We chose to put these values into the request body to make the URL simpler. Several behaviors here are blindly copied from the Foolscap-based storage server protocol. @@ -452,13 +461,13 @@ For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} -The request must include ``WWW-Authenticate`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +The request must include ``X-Tahoe-Set-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. For example:: - WWW-Authenticate: x-tahoe-renew-secret - WWW-Authenticate: x-tahoe-cancel-secret - WWW-Authenticate: x-tahoe-upload-secret + X-Tahoe-Set-Authorization: lease-renew-secret + X-Tahoe-Set-Authorization: lease-cancel-secret + X-Tahoe-Set-Authorization: upload-secret The response body includes encoded information about the created buckets. For example:: @@ -527,9 +536,9 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). -The request must include a ``Authorization`` header that includes the upload secret:: +The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - Authorization: x-tahoe-upload-secret + X-Tahoe-Authorization: upload-secret Responses: @@ -557,9 +566,9 @@ Responses: This cancels an *in-progress* upload. -The request body looks this:: +The request must include a ``Authorization`` header that includes the upload secret:: - { "upload-secret": "xyzf" } + X-Tahoe-Authorization: upload-secret The response code: @@ -658,16 +667,16 @@ The first write operation on a mutable storage index creates it (that is, there is no separate "create this storage index" operation as there is for the immutable storage index type). -The request body includes the secrets necessary to rewrite to the shares -along with test, read, and write vectors for the operation. +The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: + + X-Tahoe-Authorization: write-enabler + X-Tahoe-Authorization: lease-lease-cancel-secret + X-Tahoe-Authorization: lease-renew-secret + +The request body includes test, read, and write vectors for the operation. For example:: { - "secrets": { - "write-enabler": "abcd", - "lease-renew": "efgh", - "lease-cancel": "ijkl" - }, "test-write-vectors": { 0: { "test": [{ @@ -733,9 +742,10 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: POST /v1/immutable/AAAAAAAAAAAAAAAA - WWW-Authenticate: x-tahoe-renew-secret efgh - WWW-Authenticate: x-tahoe-cancel-secret jjkl - WWW-Authenticate: x-tahoe-upload-secret xyzf + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Set-Authorization: lease-renew-secret efgh + X-Tahoe-Set-Authorization: lease-cancel-secret jjkl + X-Tahoe-Set-Authorization: upload-secret xyzf {"share-numbers": [1, 7], "allocated-size": 48} @@ -745,22 +755,25 @@ Immutable Data #. Upload the content for immutable share ``7``:: PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 0-15/48 - Authorization: x-tahoe-upload-secret xyzf + X-Tahoe-Authorization: upload-secret xyzf 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 16-31/48 - Authorization: x-tahoe-upload-secret xyzf + X-Tahoe-Authorization: upload-secret xyzf 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 32-47/48 - Authorization: x-tahoe-upload-secret xyzf + X-Tahoe-Authorization: upload-secret xyzf 201 CREATED @@ -768,6 +781,7 @@ Immutable Data #. Download the content of the previously uploaded immutable share ``7``:: GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7 + Authorization: Tahoe-LAFS nurl-swissnum Range: bytes=0-47 200 OK @@ -776,7 +790,9 @@ Immutable Data #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: PUT /v1/lease/AAAAAAAAAAAAAAAA - {"renew-secret": "efgh", "cancel-secret": "ijkl"} + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: lease-cancel-secret jjkl + X-Tahoe-Authorization: upload-secret xyzf 204 NO CONTENT @@ -789,12 +805,12 @@ if there is no existing share, otherwise it will read a byte which won't match `b""`:: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: write-enabler abcd + X-Tahoe-Authorization: lease-cancel-secret efgh + X-Tahoe-Authorization: lease-renew-secret ijkl + { - "secrets": { - "write-enabler": "abcd", - "lease-renew": "efgh", - "lease-cancel": "ijkl" - }, "test-write-vectors": { 3: { "test": [{ @@ -821,12 +837,12 @@ otherwise it will read a byte which won't match `b""`:: #. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: write-enabler abcd + X-Tahoe-Authorization: lease-cancel-secret efgh + X-Tahoe-Authorization: lease-renew-secret ijkl + { - "secrets": { - "write-enabler": "abcd", - "lease-renew": "efgh", - "lease-cancel": "ijkl" - }, "test-write-vectors": { 3: { "test": [{ @@ -853,12 +869,16 @@ otherwise it will read a byte which won't match `b""`:: #. Download the contents of share number ``3``:: GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + Authorization: Tahoe-LAFS nurl-swissnum + #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: PUT /v1/lease/BBBBBBBBBBBBBBBB - {"renew-secret": "efgh", "cancel-secret": "ijkl"} + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: lease-cancel-secret efgh + X-Tahoe-Authorization: lease-renew-secret ijkl 204 NO CONTENT From 7faec6e5a0bb53f5d58a4253795047144a58d62d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 10 Nov 2021 15:48:58 -0500 Subject: [PATCH 157/916] news fragment --- newsfragments/3842.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3842.minor diff --git a/newsfragments/3842.minor b/newsfragments/3842.minor new file mode 100644 index 000000000..e69de29bb From 9af81d21c5af232c8e02e874b09ac33202cb5158 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 10 Nov 2021 16:08:40 -0500 Subject: [PATCH 158/916] add a way to turn off implicit bucket lease renewal too --- src/allmydata/storage/server.py | 50 ++++++++++++++++++++----- src/allmydata/test/test_storage.py | 59 +++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 3e2d3b5c6..d142646a8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -57,9 +57,23 @@ DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 @implementer(RIStorageServer, IStatsProducer) class StorageServer(service.MultiService, Referenceable): + """ + A filesystem-based implementation of ``RIStorageServer``. + + :ivar bool _implicit_bucket_lease_renewal: If and only if this is ``True`` + then ``allocate_buckets`` will renew leases on existing shares + associated with the storage index it operates on. + + :ivar bool _implicit_slot_lease_renewal: If and only if this is ``True`` + then ``slot_testv_and_readv_and_writev`` will renew leases on shares + associated with the slot it operates on. + """ name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler + _implicit_bucket_lease_renewal = True + _implicit_slot_lease_renewal = True + def __init__(self, storedir, nodeid, reserved_space=0, discard_storage=False, readonly_storage=False, stats_provider=None, @@ -135,6 +149,29 @@ class StorageServer(service.MultiService, Referenceable): def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) + def set_implicit_bucket_lease_renewal(self, enabled): + # type: (bool) -> None + """ + Control the behavior of implicit lease renewal by *allocate_buckets*. + + :param enabled: If and only if ``True`` then future *allocate_buckets* + calls will renew leases on shares that already exist in the bucket. + """ + self._implicit_bucket_lease_renewal = enabled + + def set_implicit_slot_lease_renewal(self, enabled): + # type: (bool) -> None + """ + Control the behavior of implicit lease renewal by + *slot_testv_and_readv_and_writev*. + + :param enabled: If and only if ``True`` then future + *slot_testv_and_readv_and_writev* calls will renew leases on + shares that still exist in the slot after the writev is applied + and which were touched by the writev. + """ + self._implicit_slot_lease_renewal = enabled + def have_shares(self): # quick test to decide if we need to commit to an implicit # permutation-seed or if we should use a new one @@ -319,8 +356,9 @@ class StorageServer(service.MultiService, Referenceable): # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot.add(shnum) - sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + if self._implicit_bucket_lease_renewal: + sf = ShareFile(fn) + sf.add_or_renew_lease(lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -625,15 +663,10 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, - renew_leases, ): """ Read data from shares and conditionally write some data to them. - :param bool renew_leases: If and only if this is ``True`` and the test - vectors pass then shares in this slot will also have an updated - lease applied to them. - See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. """ @@ -673,7 +706,7 @@ class StorageServer(service.MultiService, Referenceable): test_and_write_vectors, shares, ) - if renew_leases: + if self._implicit_slot_lease_renewal: lease_info = self._make_lease_info(renew_secret, cancel_secret) self._add_or_renew_leases(remaining_shares, lease_info) @@ -690,7 +723,6 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, - renew_leases=True, ) def _allocate_slot_share(self, bucketdir, secrets, sharenum, diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 460653bd0..efa889f8d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -608,6 +608,61 @@ class Server(unittest.TestCase): for i,wb in writers.items(): wb.remote_abort() + def test_allocate_without_lease_renewal(self): + """ + ``remote_allocate_buckets`` does not renew leases on existing shares if + ``set_implicit_bucket_lease_renewal(False)`` is called first. + """ + first_lease = 456 + second_lease = 543 + storage_index = b"allocate" + + clock = Clock() + clock.advance(first_lease) + ss = self.create( + "test_allocate_without_lease_renewal", + get_current_time=clock.seconds, + ) + ss.set_implicit_bucket_lease_renewal(False) + + # Put a share on there + already, writers = self.allocate(ss, storage_index, [0], 1) + (writer,) = writers.values() + writer.remote_write(0, b"x") + writer.remote_close() + + # It should have a lease granted at the current time. + shares = dict(ss._get_bucket_shares(storage_index)) + self.assertEqual( + [first_lease], + list( + lease.get_grant_renew_time_time() + for lease + in ShareFile(shares[0]).get_leases() + ), + ) + + # Let some time pass so we can tell if the lease on share 0 is + # renewed. + clock.advance(second_lease) + + # Put another share on there. + already, writers = self.allocate(ss, storage_index, [1], 1) + (writer,) = writers.values() + writer.remote_write(0, b"x") + writer.remote_close() + + # The first share's lease expiration time is unchanged. + shares = dict(ss._get_bucket_shares(storage_index)) + self.assertEqual( + [first_lease], + list( + lease.get_grant_renew_time_time() + for lease + in ShareFile(shares[0]).get_leases() + ), + ) + def test_bad_container_version(self): ss = self.create("test_bad_container_version") a,w = self.allocate(ss, b"si1", [0], 10) @@ -1408,9 +1463,10 @@ class MutableServer(unittest.TestCase): def test_writev_without_renew_lease(self): """ The helper method ``slot_testv_and_readv_and_writev`` does not renew - leases if ``False`` is passed for the ``renew_leases`` parameter. + leases if ``set_implicit_bucket_lease_renewal(False)`` is called first. """ ss = self.create("test_writev_without_renew_lease") + ss.set_implicit_slot_lease_renewal(False) storage_index = b"si2" secrets = ( @@ -1429,7 +1485,6 @@ class MutableServer(unittest.TestCase): sharenum: ([], datav, None), }, read_vector=[], - renew_leases=False, ) leases = list(ss.get_slot_leases(storage_index)) self.assertEqual([], leases) From 2742de6f7c1fa6cf77e35ecc5854bcf7db3e5963 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 10 Nov 2021 16:08:53 -0500 Subject: [PATCH 159/916] drop some ancient cruft allocated_size not used anywhere, so why have it --- src/allmydata/storage/server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index d142646a8..36cf06d0e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -617,10 +617,8 @@ class StorageServer(service.MultiService, Referenceable): else: if sharenum not in shares: # allocate a new share - allocated_size = 2000 # arbitrary, really share = self._allocate_slot_share(bucketdir, secrets, sharenum, - allocated_size, owner_num=0) shares[sharenum] = share shares[sharenum].writev(datav, new_length) @@ -726,7 +724,7 @@ class StorageServer(service.MultiService, Referenceable): ) def _allocate_slot_share(self, bucketdir, secrets, sharenum, - allocated_size, owner_num=0): + owner_num=0): (write_enabler, renew_secret, cancel_secret) = secrets my_nodeid = self.my_nodeid fileutil.make_dirs(bucketdir) From c270a346c6c7c247db08bf107bef93c4cccc7ced Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Nov 2021 11:02:51 -0500 Subject: [PATCH 160/916] Remove typo. --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 2a392fb20..19a64f5ca 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -670,7 +670,7 @@ there is no separate "create this storage index" operation as there is for the i The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: X-Tahoe-Authorization: write-enabler - X-Tahoe-Authorization: lease-lease-cancel-secret + X-Tahoe-Authorization: lease-cancel-secret X-Tahoe-Authorization: lease-renew-secret The request body includes test, read, and write vectors for the operation. From 24646c56d0aae56bd18d2d2ffa2acf1616cc2a62 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Nov 2021 11:29:05 -0500 Subject: [PATCH 161/916] Updates based on review. --- docs/proposed/http-storage-node-protocol.rst | 43 ++++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 19a64f5ca..44bda1205 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -397,22 +397,15 @@ For example:: Either renew or create a new lease on the bucket addressed by ``storage_index``. -For a renewal, the renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. +The renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. For example:: X-Tahoe-Authorization: lease-renew-secret X-Tahoe-Authorization: lease-cancel-secret -For a new lease, ``X-Tahoe-Set-Authorization`` headers should be used instead. -For example:: - - X-Tahoe-Set-Authorization: lease-renew-secret - X-Tahoe-Set-Authorization: lease-cancel-secret - -For renewal, the expiration time of that lease will be changed to 31 days after the time of this operation. -If the renewal secret does not match, a new lease will be created, but clients should still not rely on this behavior if possible, and instead use the appropriate new lease headers. - -For the creation path, +If the ``lease-renew-secret`` value matches an existing lease +then the expiration time of that lease will be changed to 31 days after the time of this operation. +If it does not match an existing lease then a new lease will be created with this ``lease-renew-secret`` which expires 31 days after the time of this operation. ``lease-renew-secret`` and ``lease-cancel-secret`` values must be 32 bytes long. @@ -433,7 +426,9 @@ Discussion `````````` We considered an alternative where ``lease-renew-secret`` and ``lease-cancel-secret`` are placed in query arguments on the request path. -We chose to put these values into the request body to make the URL simpler. +This increases chances of leaking secrets in logs. +Putting the secrets in the body reduces the chances of leaking secrets, +but eventually we chose headers as the least likely information to be logged. Several behaviors here are blindly copied from the Foolscap-based storage server protocol. @@ -461,13 +456,13 @@ For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} -The request must include ``X-Tahoe-Set-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. For example:: - X-Tahoe-Set-Authorization: lease-renew-secret - X-Tahoe-Set-Authorization: lease-cancel-secret - X-Tahoe-Set-Authorization: upload-secret + X-Tahoe-Authorization: lease-renew-secret + X-Tahoe-Authorization: lease-cancel-secret + X-Tahoe-Authorization: upload-secret The response body includes encoded information about the created buckets. For example:: @@ -475,12 +470,6 @@ For example:: {"already-have": [1, ...], "allocated": [7, ...]} The upload secret is an opaque _byte_ string. -It will be generated by hashing a combination of:b - -1. A tag. -2. The storage index, so it's unique across different source files. -3. The server ID, so it's unique across different servers. -4. The convergence secret, so that servers can't guess the upload secret for other servers. Discussion `````````` @@ -508,7 +497,7 @@ The response includes ``already-have`` and ``allocated`` for two reasons: Regarding upload secrets, the goal is for uploading and aborting (see next sections) to be authenticated by more than just the storage index. -In the future, we will want to generate them in a way that allows resuming/canceling when the client has issues. +In the future, we may want to generate them in a way that allows resuming/canceling when the client has issues. In the short term, they can just be a random byte string. The key security constraint is that each upload to each server has its own, unique upload key, tied to uploading that particular storage index to this particular server. @@ -566,7 +555,7 @@ Responses: This cancels an *in-progress* upload. -The request must include a ``Authorization`` header that includes the upload secret:: +The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: X-Tahoe-Authorization: upload-secret @@ -743,9 +732,9 @@ Immutable Data POST /v1/immutable/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum - X-Tahoe-Set-Authorization: lease-renew-secret efgh - X-Tahoe-Set-Authorization: lease-cancel-secret jjkl - X-Tahoe-Set-Authorization: upload-secret xyzf + X-Tahoe-Authorization: lease-renew-secret efgh + X-Tahoe-Authorization: lease-cancel-secret jjkl + X-Tahoe-Authorization: upload-secret xyzf {"share-numbers": [1, 7], "allocated-size": 48} From bea4cf18a0d7d91dece9fb4a45bb39c5b41b8e9d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 11:19:29 -0500 Subject: [PATCH 162/916] News file. --- newsfragments/3843.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3843.minor diff --git a/newsfragments/3843.minor b/newsfragments/3843.minor new file mode 100644 index 000000000..e69de29bb From e7a5d14c0e8c0077880e2a9ffbd1e3db3738dd93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 11:25:10 -0500 Subject: [PATCH 163/916] New requirements. --- nix/tahoe-lafs.nix | 2 +- setup.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index e42afc57f..f691677f6 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser +, html5lib, pyutil, distro, configparser, klein, treq }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.16.0). diff --git a/setup.py b/setup.py index 8c6396937..3d9f5a509 100644 --- a/setup.py +++ b/setup.py @@ -140,6 +140,10 @@ install_requires = [ # For the RangeMap datastructure. "collections-extended", + + # HTTP server and client + "klein", + "treq", ] setup_requires = [ @@ -397,7 +401,6 @@ setup(name="tahoe-lafs", # also set in __init__.py # Python 2.7. "decorator < 5", "hypothesis >= 3.6.1", - "treq", "towncrier", "testtools", "fixtures", From 777d630f481e3010c399d0cc2e872bacd572e700 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 12:00:07 -0500 Subject: [PATCH 164/916] Another dependency. --- nix/tahoe-lafs.nix | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f691677f6..a6a8a69ec 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, treq +, html5lib, pyutil, distro, configparser, klein, treq, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.16.0). diff --git a/setup.py b/setup.py index 3d9f5a509..7e7a955c6 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,7 @@ install_requires = [ # HTTP server and client "klein", "treq", + "cbor2" ] setup_requires = [ From a32c6be978f0c857ee0465cf123b56058178a21e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 12:02:58 -0500 Subject: [PATCH 165/916] A sketch of what the HTTP server will look like. --- src/allmydata/storage/http_server.py | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/allmydata/storage/http_server.py diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py new file mode 100644 index 000000000..87edda999 --- /dev/null +++ b/src/allmydata/storage/http_server.py @@ -0,0 +1,66 @@ +""" +HTTP server for storage. +""" + +from functools import wraps + +from klein import Klein +from twisted.web import http + +# Make sure to use pure Python versions: +from cbor2.encoder import dumps +from cbor2.decoder import loads + +from .server import StorageServer + + +def _authorization_decorator(f): + """ + Check the ``Authorization`` header, and (TODO: in later revision of code) + extract ``X-Tahoe-Authorization`` headers and pass them in. + """ + + @wraps(f) + def route(self, request, *args, **kwargs): + if request.headers["Authorization"] != self._swissnum: + request.setResponseCode(http.NOT_ALLOWED) + return b"" + # authorization = request.headers.getRawHeaders("X-Tahoe-Authorization", []) + # For now, just a placeholder: + authorization = None + return f(self, request, authorization, *args, **kwargs) + + +def _route(app, *route_args, **route_kwargs): + """ + Like Klein's @route, but with additional support for checking the + ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The + latter will (TODO: in later revision of code) get passed in as second + argument to wrapped functions. + """ + + def decorator(f): + @app.route(*route_args, **route_kwargs) + @_authorization_decorator + def handle_route(*args, **kwargs): + return f(*args, **kwargs) + + return handle_route + + return decorator + + +class HTTPServer(object): + """ + A HTTP interface to the storage server. + """ + + _app = Klein() + + def __init__(self, storage_server: StorageServer, swissnum): + self._storage_server = storage_server + self._swissnum = swissnum + + @_route(_app, "/v1/version", methods=["GET"]) + def version(self, request, authorization): + return dumps(self._storage_server.remote_get_version()) From ddd2780bd243436d3630fdcee8b0340480736e27 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 12:51:52 -0500 Subject: [PATCH 166/916] First sketch of HTTP client. --- src/allmydata/storage/http_client.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/allmydata/storage/http_client.py diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py new file mode 100644 index 000000000..ca80b704e --- /dev/null +++ b/src/allmydata/storage/http_client.py @@ -0,0 +1,36 @@ +""" +HTTP client that talks to the HTTP storage server. +""" + +# Make sure to import Python version: +from cbor2.encoder import loads +from cbor2.decoder import loads + +from twisted.internet.defer import inlineCallbacks, returnValue +from hyperlink import DecodedURL +import treq + + +def _decode_cbor(response): + """Given HTTP response, return decoded CBOR body.""" + return treq.content(response).addCallback(loads) + + +class StorageClient(object): + """ + HTTP client that talks to the HTTP storage server. + """ + + def __init__(self, url: DecodedURL, swissnum, treq=treq): + self._base_url = url + self._swissnum = swissnum + self._treq = treq + + @inlineCallbacks + def get_version(self): + """ + Return the version metadata for the server. + """ + url = self._base_url.child("v1", "version") + response = _decode_cbor((yield self._treq.get(url))) + returnValue(response) From 12cbf8a90109548aaba570d977863bacc2e8fdad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 13:03:53 -0500 Subject: [PATCH 167/916] First sketch of HTTP testing infrastructure. --- src/allmydata/test/test_storage_http.py | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/allmydata/test/test_storage_http.py diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py new file mode 100644 index 000000000..589cfdddf --- /dev/null +++ b/src/allmydata/test/test_storage_http.py @@ -0,0 +1,38 @@ +""" +Tests for HTTP storage client + server. +""" + +from twisted.trial.unittest import TestCase +from twisted.internet.defer import inlineCallbacks + +from treq.testing import StubTreq +from hyperlink import DecodedURL + +from ..storage.server import StorageServer +from ..storage.http_server import HTTPServer +from ..storage.http_client import StorageClient + + +class HTTPTests(TestCase): + """ + Tests of HTTP client talking to the HTTP server. + """ + + def setUp(self): + self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) + # TODO what should the swissnum _actually_ be? + self._http_server = HTTPServer(self._storage_server, b"abcd") + self.client = StorageClient( + DecodedURL.from_text("http://example.com"), + b"abcd", + treq=StubTreq(self._http_server.get_resource()), + ) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + """ + version = yield self.client.get_version() + expected_version = self.storage_server.remote_get_version() + self.assertEqual(version, expected_version) From c101dd4dc9e33190da63daedd1963a1fb0e9f7cf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 13:13:19 -0500 Subject: [PATCH 168/916] Closer to first passing test. --- src/allmydata/storage/http_client.py | 20 +++++++++++++------- src/allmydata/storage/http_server.py | 20 +++++++++++++++----- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ca80b704e..e593fd379 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,18 +2,23 @@ HTTP client that talks to the HTTP storage server. """ -# Make sure to import Python version: -from cbor2.encoder import loads -from cbor2.decoder import loads +# TODO Make sure to import Python version? +from cbor2 import loads, dumps -from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.defer import inlineCallbacks, returnValue, fail from hyperlink import DecodedURL import treq +class ClientException(Exception): + """An unexpected error.""" + + def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" - return treq.content(response).addCallback(loads) + if response.code > 199 and response.code < 300: + return treq.content(response).addCallback(loads) + return fail(ClientException(response.code, response.phrase)) class StorageClient(object): @@ -32,5 +37,6 @@ class StorageClient(object): Return the version metadata for the server. """ url = self._base_url.child("v1", "version") - response = _decode_cbor((yield self._treq.get(url))) - returnValue(response) + response = yield self._treq.get(url) + decoded_response = yield _decode_cbor(response) + returnValue(decoded_response) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 87edda999..b862fe7b1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -7,9 +7,8 @@ from functools import wraps from klein import Klein from twisted.web import http -# Make sure to use pure Python versions: -from cbor2.encoder import dumps -from cbor2.decoder import loads +# TODO Make sure to use pure Python versions? +from cbor2 import loads, dumps from .server import StorageServer @@ -22,14 +21,19 @@ def _authorization_decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if request.headers["Authorization"] != self._swissnum: + if ( + request.requestHeaders.getRawHeaders("Authorization", [None])[0] + != self._swissnum + ): request.setResponseCode(http.NOT_ALLOWED) return b"" - # authorization = request.headers.getRawHeaders("X-Tahoe-Authorization", []) + # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) # For now, just a placeholder: authorization = None return f(self, request, authorization, *args, **kwargs) + return route + def _route(app, *route_args, **route_kwargs): """ @@ -53,6 +57,8 @@ def _route(app, *route_args, **route_kwargs): class HTTPServer(object): """ A HTTP interface to the storage server. + + TODO returning CBOR should set CBOR content-type """ _app = Klein() @@ -61,6 +67,10 @@ class HTTPServer(object): self._storage_server = storage_server self._swissnum = swissnum + def get_resource(self): + """Return twisted.web Resource for this object.""" + return self._app.resource() + @_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): return dumps(self._storage_server.remote_get_version()) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 589cfdddf..663675f40 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -21,7 +21,7 @@ class HTTPTests(TestCase): def setUp(self): self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) # TODO what should the swissnum _actually_ be? - self._http_server = HTTPServer(self._storage_server, b"abcd") + self._http_server = HTTPServer(self.storage_server, b"abcd") self.client = StorageClient( DecodedURL.from_text("http://example.com"), b"abcd", From c3cb0ebaeaa196c24272ac1fd834ed3c30baa377 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:20:27 -0500 Subject: [PATCH 169/916] Switch to per-call parameter for controlling lease renewal behavior This is closer to an implementation where you could have two frontends, say a Foolscap frontend and an HTTP frontend or even just two different HTTP frontends, which had different opinions about what the behaviour should be. --- src/allmydata/storage/server.py | 51 ++++++------------------ src/allmydata/test/test_storage.py | 63 +++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 53 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 36cf06d0e..70d71f841 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -59,21 +59,10 @@ DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 class StorageServer(service.MultiService, Referenceable): """ A filesystem-based implementation of ``RIStorageServer``. - - :ivar bool _implicit_bucket_lease_renewal: If and only if this is ``True`` - then ``allocate_buckets`` will renew leases on existing shares - associated with the storage index it operates on. - - :ivar bool _implicit_slot_lease_renewal: If and only if this is ``True`` - then ``slot_testv_and_readv_and_writev`` will renew leases on shares - associated with the slot it operates on. """ name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler - _implicit_bucket_lease_renewal = True - _implicit_slot_lease_renewal = True - def __init__(self, storedir, nodeid, reserved_space=0, discard_storage=False, readonly_storage=False, stats_provider=None, @@ -149,29 +138,6 @@ class StorageServer(service.MultiService, Referenceable): def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) - def set_implicit_bucket_lease_renewal(self, enabled): - # type: (bool) -> None - """ - Control the behavior of implicit lease renewal by *allocate_buckets*. - - :param enabled: If and only if ``True`` then future *allocate_buckets* - calls will renew leases on shares that already exist in the bucket. - """ - self._implicit_bucket_lease_renewal = enabled - - def set_implicit_slot_lease_renewal(self, enabled): - # type: (bool) -> None - """ - Control the behavior of implicit lease renewal by - *slot_testv_and_readv_and_writev*. - - :param enabled: If and only if ``True`` then future - *slot_testv_and_readv_and_writev* calls will renew leases on - shares that still exist in the slot after the writev is applied - and which were touched by the writev. - """ - self._implicit_slot_lease_renewal = enabled - def have_shares(self): # quick test to decide if we need to commit to an implicit # permutation-seed or if we should use a new one @@ -314,9 +280,12 @@ class StorageServer(service.MultiService, Referenceable): def _allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - owner_num=0): + owner_num=0, renew_leases=True): """ Generic bucket allocation API. + + :param bool renew_leases: If and only if this is ``True`` then + renew leases on existing shares in this bucket. """ # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated @@ -356,7 +325,7 @@ class StorageServer(service.MultiService, Referenceable): # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot.add(shnum) - if self._implicit_bucket_lease_renewal: + if renew_leases: sf = ShareFile(fn) sf.add_or_renew_lease(lease_info) @@ -399,7 +368,7 @@ class StorageServer(service.MultiService, Referenceable): """Foolscap-specific ``allocate_buckets()`` API.""" alreadygot, bucketwriters = self._allocate_buckets( storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - owner_num=owner_num, + owner_num=owner_num, renew_leases=True, ) # Abort BucketWriters if disconnection happens. for bw in bucketwriters.values(): @@ -661,12 +630,17 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, + renew_leases, ): """ Read data from shares and conditionally write some data to them. See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. + + :param bool renew_leases: If and only if this is ``True`` then renew + leases on all shares mentioned in ``test_and_write_vectors` that + still exist after the changes are made. """ start = self._get_current_time() self.count("writev") @@ -704,7 +678,7 @@ class StorageServer(service.MultiService, Referenceable): test_and_write_vectors, shares, ) - if self._implicit_slot_lease_renewal: + if renew_leases: lease_info = self._make_lease_info(renew_secret, cancel_secret) self._add_or_renew_leases(remaining_shares, lease_info) @@ -721,6 +695,7 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, + renew_leases=True, ) def _allocate_slot_share(self, bucketdir, secrets, sharenum, diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index efa889f8d..a6c1ac2c2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -468,14 +468,19 @@ class Server(unittest.TestCase): sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnlessIn(b'available-space', sv1) - def allocate(self, ss, storage_index, sharenums, size, canary=None): + def allocate(self, ss, storage_index, sharenums, size, renew_leases=True): + """ + Call directly into the storage server's allocate_buckets implementation, + skipping the Foolscap layer. + """ renew_secret = hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)) cancel_secret = hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret)) - if not canary: - canary = FakeCanary() - return ss.remote_allocate_buckets(storage_index, - renew_secret, cancel_secret, - sharenums, size, canary) + return ss._allocate_buckets( + storage_index, + renew_secret, cancel_secret, + sharenums, size, + renew_leases=renew_leases, + ) def test_large_share(self): syslow = platform.system().lower() @@ -611,7 +616,7 @@ class Server(unittest.TestCase): def test_allocate_without_lease_renewal(self): """ ``remote_allocate_buckets`` does not renew leases on existing shares if - ``set_implicit_bucket_lease_renewal(False)`` is called first. + ``renew_leases`` is ``False``. """ first_lease = 456 second_lease = 543 @@ -623,10 +628,11 @@ class Server(unittest.TestCase): "test_allocate_without_lease_renewal", get_current_time=clock.seconds, ) - ss.set_implicit_bucket_lease_renewal(False) # Put a share on there - already, writers = self.allocate(ss, storage_index, [0], 1) + already, writers = self.allocate( + ss, storage_index, [0], 1, renew_leases=False, + ) (writer,) = writers.values() writer.remote_write(0, b"x") writer.remote_close() @@ -647,7 +653,9 @@ class Server(unittest.TestCase): clock.advance(second_lease) # Put another share on there. - already, writers = self.allocate(ss, storage_index, [1], 1) + already, writers = self.allocate( + ss, storage_index, [1], 1, renew_leases=False, + ) (writer,) = writers.values() writer.remote_write(0, b"x") writer.remote_close() @@ -684,8 +692,17 @@ class Server(unittest.TestCase): def test_disconnect(self): # simulate a disconnection ss = self.create("test_disconnect") + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 canary = FakeCanary() - already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75, canary) + already,writers = ss.remote_allocate_buckets( + b"disconnect", + renew_secret, + cancel_secret, + sharenums=[0,1,2], + allocated_size=75, + canary=canary, + ) self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) for (f,args,kwargs) in list(canary.disconnectors.values()): @@ -717,8 +734,17 @@ class Server(unittest.TestCase): # the size we request. OVERHEAD = 3*4 LEASE_SIZE = 4+32+32+4 + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 canary = FakeCanary() - already, writers = self.allocate(ss, b"vid1", [0,1,2], 1000, canary) + already, writers = ss.remote_allocate_buckets( + b"vid1", + renew_secret, + cancel_secret, + sharenums=[0,1,2], + allocated_size=1000, + canary=canary, + ) self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed @@ -751,7 +777,14 @@ class Server(unittest.TestCase): # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and # 5000-1085=3915 free, therefore we can fit 39 100byte shares canary3 = FakeCanary() - already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary3) + already3, writers3 = ss.remote_allocate_buckets( + b"vid3", + renew_secret, + cancel_secret, + sharenums=list(range(100)), + allocated_size=100, + canary=canary3, + ) self.failUnlessEqual(len(writers3), 39) self.failUnlessEqual(len(ss._bucket_writers), 39) @@ -1463,10 +1496,9 @@ class MutableServer(unittest.TestCase): def test_writev_without_renew_lease(self): """ The helper method ``slot_testv_and_readv_and_writev`` does not renew - leases if ``set_implicit_bucket_lease_renewal(False)`` is called first. + leases if ``renew_leases```` is ``False``. """ ss = self.create("test_writev_without_renew_lease") - ss.set_implicit_slot_lease_renewal(False) storage_index = b"si2" secrets = ( @@ -1485,6 +1517,7 @@ class MutableServer(unittest.TestCase): sharenum: ([], datav, None), }, read_vector=[], + renew_leases=False, ) leases = list(ss.get_slot_leases(storage_index)) self.assertEqual([], leases) From 85977e48a7dde8ea29e196a6d466ae8685c2f6fc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:23:15 -0500 Subject: [PATCH 170/916] put this comment back and merge info from the two versions --- src/allmydata/storage/server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 70d71f841..9b73963ae 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -635,12 +635,13 @@ class StorageServer(service.MultiService, Referenceable): """ Read data from shares and conditionally write some data to them. + :param bool renew_leases: If and only if this is ``True`` and the test + vectors pass then shares mentioned in ``test_and_write_vectors`` + that still exist after the changes are made will also have an + updated lease applied to them. + See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. - - :param bool renew_leases: If and only if this is ``True`` then renew - leases on all shares mentioned in ``test_and_write_vectors` that - still exist after the changes are made. """ start = self._get_current_time() self.count("writev") From dece67ee3ac8d2bd06b42e07a01492e3c4497ae6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:24:29 -0500 Subject: [PATCH 171/916] it is not the remote interface that varies anymore --- src/allmydata/test/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index a6c1ac2c2..076e9f3d1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -615,8 +615,8 @@ class Server(unittest.TestCase): def test_allocate_without_lease_renewal(self): """ - ``remote_allocate_buckets`` does not renew leases on existing shares if - ``renew_leases`` is ``False``. + ``StorageServer._allocate_buckets`` does not renew leases on existing + shares if ``renew_leases`` is ``False``. """ first_lease = 456 second_lease = 543 From 6c2e85e99145652625ff7a4d6791a410ce13c742 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:25:36 -0500 Subject: [PATCH 172/916] put the comment back --- src/allmydata/test/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 076e9f3d1..4e40a76a5 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1496,7 +1496,7 @@ class MutableServer(unittest.TestCase): def test_writev_without_renew_lease(self): """ The helper method ``slot_testv_and_readv_and_writev`` does not renew - leases if ``renew_leases```` is ``False``. + leases if ``False`` is passed for the ``renew_leases`` parameter. """ ss = self.create("test_writev_without_renew_lease") From ad6017e63df94dbac7916f4673332b33deb8d5be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 15 Nov 2021 08:08:14 -0500 Subject: [PATCH 173/916] clarify renew_leases docs on allocate_buckets --- src/allmydata/storage/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9b73963ae..bfbc10b59 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -284,8 +284,10 @@ class StorageServer(service.MultiService, Referenceable): """ Generic bucket allocation API. - :param bool renew_leases: If and only if this is ``True`` then - renew leases on existing shares in this bucket. + :param bool renew_leases: If and only if this is ``True`` then renew a + secret-matching lease on (or, if none match, add a new lease to) + existing shares in this bucket. Any *new* shares are given a new + lease regardless. """ # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated From 84c19f5468b04279e1826ed77dc1e7d4b4ae00e8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 15 Nov 2021 08:12:07 -0500 Subject: [PATCH 174/916] clarify renew_leases docs on slot_testv_and_readv_and_writev --- src/allmydata/storage/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index bfbc10b59..ee2ea1c61 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -639,8 +639,9 @@ class StorageServer(service.MultiService, Referenceable): :param bool renew_leases: If and only if this is ``True`` and the test vectors pass then shares mentioned in ``test_and_write_vectors`` - that still exist after the changes are made will also have an - updated lease applied to them. + that still exist after the changes are made will also have a + secret-matching lease renewed (or, if none match, a new lease + added). See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. From fcd634fc43c42c838ac415767bd6eeb05172c82b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 15 Nov 2021 13:34:46 -0500 Subject: [PATCH 175/916] some direct tests for the new utility function --- src/allmydata/test/common.py | 7 ++- src/allmydata/test/test_common_util.py | 78 +++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 8e97fa598..76127fb57 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1220,8 +1220,13 @@ def disable_modules(*names): A context manager which makes modules appear to be missing while it is active. - :param *names: The names of the modules to disappear. + :param *names: The names of the modules to disappear. Only top-level + modules are supported (that is, "." is not allowed in any names). + This is an implementation shortcoming which could be lifted if + desired. """ + if any("." in name for name in names): + raise ValueError("Names containing '.' are not supported.") missing = object() modules = list(sys.modules.get(n, missing) for n in names) for n in names: diff --git a/src/allmydata/test/test_common_util.py b/src/allmydata/test/test_common_util.py index 55986d123..c141adc8d 100644 --- a/src/allmydata/test/test_common_util.py +++ b/src/allmydata/test/test_common_util.py @@ -10,16 +10,30 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import sys import random -import unittest +from hypothesis import given +from hypothesis.strategies import lists, sampled_from +from testtools.matchers import Equals +from twisted.python.reflect import ( + ModuleNotFound, + namedAny, +) + +from .common import ( + SyncTestCase, + disable_modules, +) from allmydata.test.common_util import flip_one_bit -class TestFlipOneBit(unittest.TestCase): +class TestFlipOneBit(SyncTestCase): def setUp(self): - random.seed(42) # I tried using version=1 on PY3 to avoid the if below, to no avail. + super(TestFlipOneBit, self).setUp() + # I tried using version=1 on PY3 to avoid the if below, to no avail. + random.seed(42) def test_accepts_byte_string(self): actual = flip_one_bit(b'foo') @@ -27,3 +41,61 @@ class TestFlipOneBit(unittest.TestCase): def test_rejects_unicode_string(self): self.assertRaises(AssertionError, flip_one_bit, u'foo') + + + +def some_existing_modules(): + """ + Build the names of modules (as native strings) that exist and can be + imported. + """ + candidates = sorted( + name + for name + in sys.modules + if "." not in name + and sys.modules[name] is not None + ) + return sampled_from(candidates) + +class DisableModulesTests(SyncTestCase): + """ + Tests for ``disable_modules``. + """ + def setup_example(self): + return sys.modules.copy() + + def teardown_example(self, safe_modules): + sys.modules.update(safe_modules) + + @given(lists(some_existing_modules(), unique=True)) + def test_importerror(self, module_names): + """ + While the ``disable_modules`` context manager is active any import of the + modules identified by the names passed to it result in ``ImportError`` + being raised. + """ + def get_modules(): + return list( + namedAny(name) + for name + in module_names + ) + before_modules = get_modules() + + with disable_modules(*module_names): + for name in module_names: + with self.assertRaises(ModuleNotFound): + namedAny(name) + + after_modules = get_modules() + self.assertThat(before_modules, Equals(after_modules)) + + def test_dotted_names_rejected(self): + """ + If names with "." in them are passed to ``disable_modules`` then + ``ValueError`` is raised. + """ + with self.assertRaises(ValueError): + with disable_modules("foo.bar"): + pass From 304b0269e3afe6499eaa1a92abd4856c970da60b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 10:14:04 -0500 Subject: [PATCH 176/916] Apply suggestions from code review Co-authored-by: Jean-Paul Calderone --- docs/proposed/http-storage-node-protocol.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 44bda1205..bc109ac7e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -400,8 +400,8 @@ Either renew or create a new lease on the bucket addressed by ``storage_index``. The renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. For example:: - X-Tahoe-Authorization: lease-renew-secret - X-Tahoe-Authorization: lease-cancel-secret + X-Tahoe-Authorization: lease-renew-secret + X-Tahoe-Authorization: lease-cancel-secret If the ``lease-renew-secret`` value matches an existing lease then the expiration time of that lease will be changed to 31 days after the time of this operation. @@ -457,7 +457,6 @@ For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. -Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. For example:: X-Tahoe-Authorization: lease-renew-secret @@ -499,7 +498,7 @@ Regarding upload secrets, the goal is for uploading and aborting (see next sections) to be authenticated by more than just the storage index. In the future, we may want to generate them in a way that allows resuming/canceling when the client has issues. In the short term, they can just be a random byte string. -The key security constraint is that each upload to each server has its own, unique upload key, +The primary security constraint is that each upload to each server has its own unique upload key, tied to uploading that particular storage index to this particular server. Rejected designs for upload secrets: @@ -527,7 +526,7 @@ The server must recognize when all of the data has been received and mark the sh The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - X-Tahoe-Authorization: upload-secret + X-Tahoe-Authorization: upload-secret Responses: @@ -557,7 +556,7 @@ This cancels an *in-progress* upload. The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - X-Tahoe-Authorization: upload-secret + X-Tahoe-Authorization: upload-secret The response code: @@ -658,7 +657,7 @@ there is no separate "create this storage index" operation as there is for the i The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: - X-Tahoe-Authorization: write-enabler + X-Tahoe-Authorization: write-enabler X-Tahoe-Authorization: lease-cancel-secret X-Tahoe-Authorization: lease-renew-secret From 7caffce8d509e1293248cb83d89e81e030b88e16 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 10:14:19 -0500 Subject: [PATCH 177/916] Another review suggestion Co-authored-by: Jean-Paul Calderone --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bc109ac7e..490d3f3ca 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -780,7 +780,7 @@ Immutable Data PUT /v1/lease/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-cancel-secret jjkl - X-Tahoe-Authorization: upload-secret xyzf + X-Tahoe-Authorization: lease-renew-secret efgh 204 NO CONTENT From 41ec63f7586124eaaf9ca65bb4d6c4884e16b48f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 10:56:21 -0500 Subject: [PATCH 178/916] Passing first tests. --- src/allmydata/storage/http_client.py | 22 ++++++++++++++++++++-- src/allmydata/storage/http_server.py | 8 ++++---- src/allmydata/test/test_storage_http.py | 19 +++++++++++++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e593fd379..412bf9cec 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,9 +2,13 @@ HTTP client that talks to the HTTP storage server. """ +import base64 + # TODO Make sure to import Python version? from cbor2 import loads, dumps + +from twisted.web.http_headers import Headers from twisted.internet.defer import inlineCallbacks, returnValue, fail from hyperlink import DecodedURL import treq @@ -21,6 +25,11 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) +def swissnum_auth_header(swissnum): + """Return value for ``Authentication`` header.""" + return b"Tahoe-LAFS " + base64.encodestring(swissnum).strip() + + class StorageClient(object): """ HTTP client that talks to the HTTP storage server. @@ -31,12 +40,21 @@ class StorageClient(object): self._swissnum = swissnum self._treq = treq + def _get_headers(self): + """Return the basic headers to be used by default.""" + headers = Headers() + headers.addRawHeader( + "Authorization", + swissnum_auth_header(self._swissnum), + ) + return headers + @inlineCallbacks def get_version(self): """ Return the version metadata for the server. """ - url = self._base_url.child("v1", "version") - response = yield self._treq.get(url) + url = self._base_url.click("/v1/version") + response = yield self._treq.get(url, headers=self._get_headers()) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b862fe7b1..2d6308baf 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,6 +11,7 @@ from twisted.web import http from cbor2 import loads, dumps from .server import StorageServer +from .http_client import swissnum_auth_header def _authorization_decorator(f): @@ -21,11 +22,10 @@ def _authorization_decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if ( - request.requestHeaders.getRawHeaders("Authorization", [None])[0] - != self._swissnum + if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( + swissnum_auth_header(self._swissnum), "ascii" ): - request.setResponseCode(http.NOT_ALLOWED) + request.setResponseCode(http.UNAUTHORIZED) return b"" # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) # For now, just a placeholder: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 663675f40..b659a6ace 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -10,7 +10,7 @@ from hyperlink import DecodedURL from ..storage.server import StorageServer from ..storage.http_server import HTTPServer -from ..storage.http_client import StorageClient +from ..storage.http_client import StorageClient, ClientException class HTTPTests(TestCase): @@ -23,11 +23,26 @@ class HTTPTests(TestCase): # TODO what should the swissnum _actually_ be? self._http_server = HTTPServer(self.storage_server, b"abcd") self.client = StorageClient( - DecodedURL.from_text("http://example.com"), + DecodedURL.from_text("http://127.0.0.1"), b"abcd", treq=StubTreq(self._http_server.get_resource()), ) + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self._http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + @inlineCallbacks def test_version(self): """ From 671b670154f62cb6c7876c707f254a6c7b3a2f4f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:09:08 -0500 Subject: [PATCH 179/916] Some type annotations. --- src/allmydata/storage/http_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 412bf9cec..8e14d1137 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -25,7 +25,7 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) -def swissnum_auth_header(swissnum): +def swissnum_auth_header(swissnum): # type: (bytes) -> bytes """Return value for ``Authentication`` header.""" return b"Tahoe-LAFS " + base64.encodestring(swissnum).strip() @@ -40,7 +40,7 @@ class StorageClient(object): self._swissnum = swissnum self._treq = treq - def _get_headers(self): + def _get_headers(self): # type: () -> Headers """Return the basic headers to be used by default.""" headers = Headers() headers.addRawHeader( From 171d1053ec803f2d2de57f0970fbad049d49f2da Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:09:17 -0500 Subject: [PATCH 180/916] CBOR content-type on responses. --- src/allmydata/storage/http_server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2d6308baf..91387c58f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -57,8 +57,6 @@ def _route(app, *route_args, **route_kwargs): class HTTPServer(object): """ A HTTP interface to the storage server. - - TODO returning CBOR should set CBOR content-type """ _app = Klein() @@ -71,6 +69,12 @@ class HTTPServer(object): """Return twisted.web Resource for this object.""" return self._app.resource() + def _cbor(self, request, data): + """Return CBOR-encoded data.""" + request.setHeader("Content-Type", "application/cbor") + # TODO if data is big, maybe want to use a temporary file eventually... + return dumps(data) + @_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): - return dumps(self._storage_server.remote_get_version()) + return self._cbor(request, self._storage_server.remote_get_version()) From c195f895db7bd3ec7a8618956a71e67152e32df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:16:26 -0500 Subject: [PATCH 181/916] Python 2 support. --- src/allmydata/storage/http_client.py | 19 ++++++++++++++++++- src/allmydata/storage/http_server.py | 18 ++++++++++++++++-- src/allmydata/test/test_storage_http.py | 12 ++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8e14d1137..4a143a60b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,6 +2,21 @@ HTTP client that talks to the HTTP storage server. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on +else: + from typing import Union + from treq.testing import StubTreq + import base64 # TODO Make sure to import Python version? @@ -35,7 +50,9 @@ class StorageClient(object): HTTP client that talks to the HTTP storage server. """ - def __init__(self, url: DecodedURL, swissnum, treq=treq): + def __init__( + self, url, swissnum, treq=treq + ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None self._base_url = url self._swissnum = swissnum self._treq = treq diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 91387c58f..373d31e2e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,6 +2,18 @@ HTTP server for storage. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + from functools import wraps from klein import Klein @@ -61,12 +73,14 @@ class HTTPServer(object): _app = Klein() - def __init__(self, storage_server: StorageServer, swissnum): + def __init__( + self, storage_server, swissnum + ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum def get_resource(self): - """Return twisted.web Resource for this object.""" + """Return twisted.web ``Resource`` for this object.""" return self._app.resource() def _cbor(self, request, data): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b659a6ace..9ba8adf21 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -2,6 +2,18 @@ Tests for HTTP storage client + server. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks From a64778ddb0fb774ea43fa8a3c59be67b84e957ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:28:13 -0500 Subject: [PATCH 182/916] Flakes. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4a143a60b..d5ca6caec 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,7 +20,7 @@ else: import base64 # TODO Make sure to import Python version? -from cbor2 import loads, dumps +from cbor2 import loads from twisted.web.http_headers import Headers diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 373d31e2e..3baa336fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -20,7 +20,7 @@ from klein import Klein from twisted.web import http # TODO Make sure to use pure Python versions? -from cbor2 import loads, dumps +from cbor2 import dumps from .server import StorageServer from .http_client import swissnum_auth_header From e5b5b50602268314e035c89e56b740c745b85c84 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:28:19 -0500 Subject: [PATCH 183/916] Duplicate package. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index a6a8a69ec..8092dfaa7 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -95,7 +95,7 @@ EOF propagatedBuildInputs = with python.pkgs; [ twisted foolscap zfec appdirs setuptoolsTrial pyasn1 zope_interface - service-identity pyyaml magic-wormhole treq + service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended ]; From a1424e90e18ae1dfbed245277120fdf3f0aaedc8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:34:44 -0500 Subject: [PATCH 184/916] Another duplicate. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 8092dfaa7..df12f21d4 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, treq, cbor2 +, html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.16.0). From f549488bb508a8377d968d16addb07a98559d8fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:47:09 -0500 Subject: [PATCH 185/916] Don't use a deprecated API. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d5ca6caec..e1743343d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -42,7 +42,7 @@ def _decode_cbor(response): def swissnum_auth_header(swissnum): # type: (bytes) -> bytes """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + base64.encodestring(swissnum).strip() + return b"Tahoe-LAFS " + base64.b64encode(swissnum).strip() class StorageClient(object): From 3b69df36b0604a0981c92a4d4c0da0611bc04535 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 23 Oct 2021 15:38:51 -0600 Subject: [PATCH 186/916] crawler: pickle -> json --- src/allmydata/storage/crawler.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index bd4f4f432..d7dee78dc 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -11,15 +11,12 @@ from __future__ import print_function from future.utils import PY2, PY3 if PY2: - # We don't import bytes, object, dict, and list just in case they're used, - # so as not to create brittle pickles with random magic objects. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min -import os, time, struct -try: - import cPickle as pickle -except ImportError: - import pickle # type: ignore +import os +import time +import json +import struct from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a @@ -214,7 +211,7 @@ class ShareCrawler(service.MultiService): # None if we are sleeping between cycles try: with open(self.statefile, "rb") as f: - state = pickle.load(f) + state = json.load(f) except Exception: state = {"version": 1, "last-cycle-finished": None, @@ -252,9 +249,7 @@ class ShareCrawler(service.MultiService): self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" with open(tmpfile, "wb") as f: - # Newer protocols won't work in Python 2; when it is dropped, - # protocol v4 can be used (added in Python 3.4). - pickle.dump(self.state, f, protocol=2) + json.dump(self.state, f) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): From 758dcea2d4a73d472050d5f21bc84217a71802b8 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 23 Oct 2021 16:19:27 -0600 Subject: [PATCH 187/916] news --- newsfragments/3825.security | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 newsfragments/3825.security diff --git a/newsfragments/3825.security b/newsfragments/3825.security new file mode 100644 index 000000000..b16418d2b --- /dev/null +++ b/newsfragments/3825.security @@ -0,0 +1,5 @@ +The lease-checker now uses JSON instead of pickle to serialize its state. + +Once you have run this version the lease state files will be stored in JSON +and an older version of the software won't load them (it simply won't notice +them so it will appear to have never run). From f7b385f9544f48ebfc7da69b314d3d1dda30ee2c Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 24 Oct 2021 22:27:59 -0600 Subject: [PATCH 188/916] play nice with subclasses --- src/allmydata/storage/crawler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index d7dee78dc..b931f1ab5 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -248,8 +248,12 @@ class ShareCrawler(service.MultiService): last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" + + # Note: we use self.get_state() here because e.g + # LeaseCheckingCrawler stores non-JSON-able state in + # self.state() but converts it in self.get_state() with open(tmpfile, "wb") as f: - json.dump(self.state, f) + json.dump(self.get_state(), f) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): From bb70e00065ab42f0e9f8faabff50d587532f49f7 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 24 Oct 2021 23:47:24 -0600 Subject: [PATCH 189/916] Make internal state JSON-able for lease-crawler --- src/allmydata/storage/expirer.py | 53 +++++++++++++------------- src/allmydata/test/test_storage_web.py | 32 ++++++++-------- src/allmydata/web/storage.py | 11 +++--- 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 7c6cd8218..4513dadb2 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -5,10 +5,11 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - # We omit anything that might end up in pickle, just in case. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min # noqa: F401 - -import time, os, pickle, struct + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import json +import time +import os +import struct from allmydata.storage.crawler import ShareCrawler from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ @@ -95,9 +96,7 @@ class LeaseCheckingCrawler(ShareCrawler): if not os.path.exists(self.historyfile): history = {} # cyclenum -> dict with open(self.historyfile, "wb") as f: - # Newer protocols won't work in Python 2; when it is dropped, - # protocol v4 can be used (added in Python 3.4). - pickle.dump(history, f, protocol=2) + json.dump(history, f) def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() @@ -142,7 +141,7 @@ class LeaseCheckingCrawler(ShareCrawler): struct.error): twlog.msg("lease-checker error processing %s" % sharefile) twlog.err() - which = (storage_index_b32, shnum) + which = [storage_index_b32, shnum] self.state["cycle-to-date"]["corrupt-shares"].append(which) wks = (1, 1, 1, "unknown") would_keep_shares.append(wks) @@ -212,7 +211,7 @@ class LeaseCheckingCrawler(ShareCrawler): num_valid_leases_configured += 1 so_far = self.state["cycle-to-date"] - self.increment(so_far["leases-per-share-histogram"], num_leases, 1) + self.increment(so_far["leases-per-share-histogram"], str(num_leases), 1) self.increment_space("examined", s, sharetype) would_keep_share = [1, 1, 1, sharetype] @@ -291,12 +290,14 @@ class LeaseCheckingCrawler(ShareCrawler): start = self.state["current-cycle-start-time"] now = time.time() - h["cycle-start-finish-times"] = (start, now) + h["cycle-start-finish-times"] = [start, now] h["expiration-enabled"] = self.expiration_enabled - h["configured-expiration-mode"] = (self.mode, - self.override_lease_duration, - self.cutoff_date, - self.sharetypes_to_expire) + h["configured-expiration-mode"] = [ + self.mode, + self.override_lease_duration, + self.cutoff_date, + self.sharetypes_to_expire, + ] s = self.state["cycle-to-date"] @@ -315,15 +316,13 @@ class LeaseCheckingCrawler(ShareCrawler): h["space-recovered"] = s["space-recovered"].copy() with open(self.historyfile, "rb") as f: - history = pickle.load(f) - history[cycle] = h + history = json.load(f) + history[str(cycle)] = h while len(history) > 10: - oldcycles = sorted(history.keys()) - del history[oldcycles[0]] + oldcycles = sorted(int(k) for k in history.keys()) + del history[str(oldcycles[0])] with open(self.historyfile, "wb") as f: - # Newer protocols won't work in Python 2; when it is dropped, - # protocol v4 can be used (added in Python 3.4). - pickle.dump(history, f, protocol=2) + json.dump(history, f) def get_state(self): """In addition to the crawler state described in @@ -393,7 +392,7 @@ class LeaseCheckingCrawler(ShareCrawler): state = ShareCrawler.get_state(self) # does a shallow copy with open(self.historyfile, "rb") as f: - history = pickle.load(f) + history = json.load(f) state["history"] = history if not progress["cycle-in-progress"]: @@ -406,10 +405,12 @@ class LeaseCheckingCrawler(ShareCrawler): lah = so_far["lease-age-histogram"] so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah) so_far["expiration-enabled"] = self.expiration_enabled - so_far["configured-expiration-mode"] = (self.mode, - self.override_lease_duration, - self.cutoff_date, - self.sharetypes_to_expire) + so_far["configured-expiration-mode"] = [ + self.mode, + self.override_lease_duration, + self.cutoff_date, + self.sharetypes_to_expire, + ] so_far_sr = so_far["space-recovered"] remaining_sr = {} diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 38e380223..b9fa548d3 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -376,7 +376,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(type(lah), list) self.failUnlessEqual(len(lah), 1) self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) - self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) + self.failUnlessEqual(so_far["leases-per-share-histogram"], {"1": 1}) self.failUnlessEqual(so_far["corrupt-shares"], []) sr1 = so_far["space-recovered"] self.failUnlessEqual(sr1["examined-buckets"], 1) @@ -427,9 +427,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failIf("cycle-to-date" in s) self.failIf("estimated-remaining-cycle" in s) self.failIf("estimated-current-cycle" in s) - last = s["history"][0] + last = s["history"]["0"] self.failUnlessIn("cycle-start-finish-times", last) - self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple) + self.failUnlessEqual(type(last["cycle-start-finish-times"]), list) self.failUnlessEqual(last["expiration-enabled"], False) self.failUnlessIn("configured-expiration-mode", last) @@ -437,9 +437,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): lah = last["lease-age-histogram"] self.failUnlessEqual(type(lah), list) self.failUnlessEqual(len(lah), 1) - self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) + self.failUnlessEqual(lah, [ [0.0, DAY, 6] ] ) - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + self.failUnlessEqual(last["leases-per-share-histogram"], {"1": 2, "2": 2}) self.failUnlessEqual(last["corrupt-shares"], []) rec = last["space-recovered"] @@ -587,12 +587,12 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(count_leases(mutable_si_3), 1) s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], - ("age", 2000, None, ("mutable", "immutable"))) - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + ["age", 2000, None, ["mutable", "immutable"]]) + self.failUnlessEqual(last["leases-per-share-histogram"], {"1": 2, "2": 2}) rec = last["space-recovered"] self.failUnlessEqual(rec["examined-buckets"], 4) @@ -731,14 +731,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(count_leases(mutable_si_3), 1) s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], - ("cutoff-date", None, then, - ("mutable", "immutable"))) + ["cutoff-date", None, then, + ["mutable", "immutable"]]) self.failUnlessEqual(last["leases-per-share-histogram"], - {1: 2, 2: 2}) + {"1": 2, "2": 2}) rec = last["space-recovered"] self.failUnlessEqual(rec["examined-buckets"], 4) @@ -924,8 +924,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): s = lc.get_state() h = s["history"] self.failUnlessEqual(len(h), 10) - self.failUnlessEqual(max(h.keys()), 15) - self.failUnlessEqual(min(h.keys()), 6) + self.failUnlessEqual(max(int(k) for k in h.keys()), 15) + self.failUnlessEqual(min(int(k) for k in h.keys()), 6) d.addCallback(_check) return d @@ -1014,7 +1014,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def _check(ignored): s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] rec = last["space-recovered"] self.failUnlessEqual(rec["configured-buckets"], 4) self.failUnlessEqual(rec["configured-shares"], 4) @@ -1110,7 +1110,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def _after_first_cycle(ignored): s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] rec = last["space-recovered"] self.failUnlessEqual(rec["examined-buckets"], 5) self.failUnlessEqual(rec["examined-shares"], 3) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index f2f021a15..e568d5ed5 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -256,8 +256,8 @@ class StorageStatusElement(Element): if so_far["corrupt-shares"]: add("Corrupt shares:", - T.ul( (T.li( ["SI %s shnum %d" % corrupt_share - for corrupt_share in so_far["corrupt-shares"] ] + T.ul( (T.li( ["SI %s shnum %d" % (si, shnum) + for si, shnum in so_far["corrupt-shares"] ] )))) return tag("Current cycle:", p) @@ -267,7 +267,8 @@ class StorageStatusElement(Element): h = lc.get_state()["history"] if not h: return "" - last = h[max(h.keys())] + biggest = str(max(int(k) for k in h.keys())) + last = h[biggest] start, end = last["cycle-start-finish-times"] tag("Last complete cycle (which took %s and finished %s ago)" @@ -290,8 +291,8 @@ class StorageStatusElement(Element): if last["corrupt-shares"]: add("Corrupt shares:", - T.ul( (T.li( ["SI %s shnum %d" % corrupt_share - for corrupt_share in last["corrupt-shares"] ] + T.ul( (T.li( ["SI %s shnum %d" % (si, shnum) + for si, shnum in last["corrupt-shares"] ] )))) return tag(p) From fa6950f08dc054ec770af1c402e8eb5d3c581b3e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 12:18:28 -0600 Subject: [PATCH 190/916] an old pickle-format lease-checker state file --- src/allmydata/test/data/lease_checker.state | 545 ++++++++++++++++++++ 1 file changed, 545 insertions(+) create mode 100644 src/allmydata/test/data/lease_checker.state diff --git a/src/allmydata/test/data/lease_checker.state b/src/allmydata/test/data/lease_checker.state new file mode 100644 index 000000000..b32554434 --- /dev/null +++ b/src/allmydata/test/data/lease_checker.state @@ -0,0 +1,545 @@ +(dp1 +S'last-complete-prefix' +p2 +NsS'version' +p3 +I1 +sS'current-cycle-start-time' +p4 +F1635003106.611748 +sS'last-cycle-finished' +p5 +I312 +sS'cycle-to-date' +p6 +(dp7 +Vleases-per-share-histogram +p8 +(dp9 +I1 +I36793 +sI2 +I1 +ssVspace-recovered +p10 +(dp11 +Vexamined-buckets-immutable +p12 +I17183 +sVconfigured-buckets-mutable +p13 +I0 +sVexamined-shares-mutable +p14 +I1796 +sVoriginal-shares-mutable +p15 +I1563 +sVconfigured-buckets-immutable +p16 +I0 +sVoriginal-shares-immutable +p17 +I27926 +sVoriginal-diskbytes-immutable +p18 +I431149056 +sVexamined-shares-immutable +p19 +I34998 +sVoriginal-buckets +p20 +I14661 +sVactual-shares-immutable +p21 +I0 +sVconfigured-shares +p22 +I0 +sVoriginal-buckets-immutable +p23 +I13761 +sVactual-diskbytes +p24 +I4096 +sVactual-shares-mutable +p25 +I0 +sVconfigured-buckets +p26 +I1 +sVexamined-buckets-unknown +p27 +I14 +sVactual-sharebytes +p28 +I0 +sVoriginal-shares +p29 +I29489 +sVoriginal-sharebytes +p30 +I312664812 +sVexamined-sharebytes-immutable +p31 +I383801602 +sVactual-shares +p32 +I0 +sVactual-sharebytes-immutable +p33 +I0 +sVoriginal-diskbytes +p34 +I441643008 +sVconfigured-diskbytes-mutable +p35 +I0 +sVconfigured-sharebytes-immutable +p36 +I0 +sVconfigured-shares-mutable +p37 +I0 +sVactual-diskbytes-immutable +p38 +I0 +sVconfigured-diskbytes-immutable +p39 +I0 +sVoriginal-diskbytes-mutable +p40 +I10489856 +sVactual-sharebytes-mutable +p41 +I0 +sVconfigured-sharebytes +p42 +I0 +sVexamined-shares +p43 +I36794 +sVactual-diskbytes-mutable +p44 +I0 +sVactual-buckets +p45 +I1 +sVoriginal-buckets-mutable +p46 +I899 +sVconfigured-sharebytes-mutable +p47 +I0 +sVexamined-sharebytes +p48 +I390369660 +sVoriginal-sharebytes-immutable +p49 +I308125753 +sVoriginal-sharebytes-mutable +p50 +I4539059 +sVactual-buckets-mutable +p51 +I0 +sVexamined-diskbytes-mutable +p52 +I9154560 +sVexamined-buckets-mutable +p53 +I1043 +sVconfigured-shares-immutable +p54 +I0 +sVexamined-diskbytes +p55 +I476598272 +sVactual-buckets-immutable +p56 +I0 +sVexamined-sharebytes-mutable +p57 +I6568058 +sVexamined-buckets +p58 +I18241 +sVconfigured-diskbytes +p59 +I4096 +sVexamined-diskbytes-immutable +p60 +I467443712 +ssVcorrupt-shares +p61 +(lp62 +(V2dn6xnlnsqwtnapwxfdivpm3s4 +p63 +I4 +tp64 +a(g63 +I1 +tp65 +a(V2rrzthwsrrxolevmwdvbdy3rqi +p66 +I4 +tp67 +a(g66 +I1 +tp68 +a(V2skfngcto6h7eqmn4uo7ntk3ne +p69 +I4 +tp70 +a(g69 +I1 +tp71 +a(V32d5swqpqx2mwix7xmqzvhdwje +p72 +I4 +tp73 +a(g72 +I1 +tp74 +a(V5mmayp66yflmpon3o6unsnbaca +p75 +I4 +tp76 +a(g75 +I1 +tp77 +a(V6ixhpvbtre7fnrl6pehlrlflc4 +p78 +I4 +tp79 +a(g78 +I1 +tp80 +a(Vewzhvswjsz4vp2bqkb6mi3bz2u +p81 +I4 +tp82 +a(g81 +I1 +tp83 +a(Vfu7pazf6ogavkqj6z4q5qqex3u +p84 +I4 +tp85 +a(g84 +I1 +tp86 +a(Vhbyjtqvpcimwxiyqbcbbdn2i4a +p87 +I4 +tp88 +a(g87 +I1 +tp89 +a(Vpmcjbdkbjdl26k3e6yja77femq +p90 +I4 +tp91 +a(g90 +I1 +tp92 +a(Vr6swof4v2uttbiiqwj5pi32cm4 +p93 +I4 +tp94 +a(g93 +I1 +tp95 +a(Vt45v5akoktf53evc2fi6gwnv6y +p96 +I4 +tp97 +a(g96 +I1 +tp98 +a(Vy6zb4faar3rdvn3e6pfg4wlotm +p99 +I4 +tp100 +a(g99 +I1 +tp101 +a(Vz3yghutvqoqbchjao4lndnrh3a +p102 +I4 +tp103 +a(g102 +I1 +tp104 +asVlease-age-histogram +p105 +(dp106 +(I45619200 +I45705600 +tp107 +I4 +s(I12441600 +I12528000 +tp108 +I78 +s(I11923200 +I12009600 +tp109 +I89 +s(I33436800 +I33523200 +tp110 +I7 +s(I37411200 +I37497600 +tp111 +I4 +s(I38361600 +I38448000 +tp112 +I5 +s(I4665600 +I4752000 +tp113 +I256 +s(I11491200 +I11577600 +tp114 +I20 +s(I10713600 +I10800000 +tp115 +I183 +s(I42076800 +I42163200 +tp116 +I4 +s(I47865600 +I47952000 +tp117 +I7 +s(I3110400 +I3196800 +tp118 +I328 +s(I5788800 +I5875200 +tp119 +I954 +s(I9331200 +I9417600 +tp120 +I12 +s(I7430400 +I7516800 +tp121 +I7228 +s(I1555200 +I1641600 +tp122 +I492 +s(I37929600 +I38016000 +tp123 +I3 +s(I38880000 +I38966400 +tp124 +I3 +s(I12528000 +I12614400 +tp125 +I193 +s(I10454400 +I10540800 +tp126 +I1239 +s(I11750400 +I11836800 +tp127 +I7 +s(I950400 +I1036800 +tp128 +I4435 +s(I44409600 +I44496000 +tp129 +I13 +s(I12787200 +I12873600 +tp130 +I218 +s(I10368000 +I10454400 +tp131 +I117 +s(I3283200 +I3369600 +tp132 +I86 +s(I7516800 +I7603200 +tp133 +I993 +s(I42336000 +I42422400 +tp134 +I33 +s(I46310400 +I46396800 +tp135 +I1 +s(I39052800 +I39139200 +tp136 +I51 +s(I7603200 +I7689600 +tp137 +I2004 +s(I10540800 +I10627200 +tp138 +I16 +s(I36374400 +I36460800 +tp139 +I3 +s(I3369600 +I3456000 +tp140 +I79 +s(I12700800 +I12787200 +tp141 +I25 +s(I4838400 +I4924800 +tp142 +I386 +s(I10972800 +I11059200 +tp143 +I122 +s(I8812800 +I8899200 +tp144 +I57 +s(I38966400 +I39052800 +tp145 +I61 +s(I3196800 +I3283200 +tp146 +I628 +s(I9244800 +I9331200 +tp147 +I73 +s(I30499200 +I30585600 +tp148 +I5 +s(I12009600 +I12096000 +tp149 +I329 +s(I12960000 +I13046400 +tp150 +I8 +s(I12614400 +I12700800 +tp151 +I210 +s(I3801600 +I3888000 +tp152 +I32 +s(I10627200 +I10713600 +tp153 +I43 +s(I44928000 +I45014400 +tp154 +I2 +s(I8208000 +I8294400 +tp155 +I38 +s(I8640000 +I8726400 +tp156 +I32 +s(I7344000 +I7430400 +tp157 +I12689 +s(I49075200 +I49161600 +tp158 +I19 +s(I2764800 +I2851200 +tp159 +I76 +s(I2592000 +I2678400 +tp160 +I40 +s(I2073600 +I2160000 +tp161 +I388 +s(I37497600 +I37584000 +tp162 +I11 +s(I1641600 +I1728000 +tp163 +I78 +s(I12873600 +I12960000 +tp164 +I5 +s(I1814400 +I1900800 +tp165 +I1860 +s(I40176000 +I40262400 +tp166 +I1 +s(I3715200 +I3801600 +tp167 +I104 +s(I2332800 +I2419200 +tp168 +I12 +s(I2678400 +I2764800 +tp169 +I278 +s(I12268800 +I12355200 +tp170 +I2 +s(I28771200 +I28857600 +tp171 +I6 +s(I41990400 +I42076800 +tp172 +I10 +sssS'last-complete-bucket' +p173 +NsS'current-cycle' +p174 +Ns. \ No newline at end of file From f81e4e2d25e4d12362f05824075915d52b3878cc Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 13:15:38 -0600 Subject: [PATCH 191/916] refactor to use serializers / pickle->json upgraders --- src/allmydata/storage/crawler.py | 143 +++++++++++++++++++++++-- src/allmydata/storage/expirer.py | 82 +++++++++++--- src/allmydata/storage/server.py | 1 + src/allmydata/test/test_storage_web.py | 10 +- 4 files changed, 212 insertions(+), 24 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index b931f1ab5..48b03ec8b 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,12 +19,145 @@ import json import struct from twisted.internet import reactor from twisted.application import service +from twisted.python.filepath import FilePath from allmydata.storage.common import si_b2a from allmydata.util import fileutil class TimeSliceExceeded(Exception): pass + +def _convert_pickle_state_to_json(state): + """ + :param dict state: the pickled state + + :return dict: the state in the JSON form + """ + # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list + # ["leases-per-share-histogram"] gets str keys instead of int + # ["cycle-start-finish-times"] from 2-tuple to list + # ["configured-expiration-mode"] from 4-tuple to list + # ["history"] keys are strings + if state["version"] != 1: + raise ValueError( + "Unknown version {version} in pickle state".format(**state) + ) + + def convert_lpsh(value): + return { + str(k): v + for k, v in value.items() + } + + def convert_cem(value): + # original is a 4-tuple, with the last element being a 2-tuple + # .. convert both to lists + return [ + value[0], + value[1], + value[2], + list(value[3]), + ] + + def convert_history(value): + print("convert history") + print(value) + return { + str(k): v + for k, v in value + } + + converters = { + "cycle-to-date": list, + "leases-per-share-histogram": convert_lpsh, + "cycle-starte-finish-times": list, + "configured-expiration-mode": convert_cem, + "history": convert_history, + } + + def convert_value(key, value): + converter = converters.get(key, None) + if converter is None: + return value + return converter(value) + + new_state = { + k: convert_value(k, v) + for k, v in state.items() + } + return new_state + + +def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): + """ + :param FilePath state_path: the filepath to ensure is json + + :param Callable[dict] convert_pickle: function to change + pickle-style state into JSON-style state + + :returns unicode: the local path where the state is stored + + If this state path is JSON, simply return it. + + If this state is pickle, convert to the JSON format and return the + JSON path. + """ + if state_path.path.endswith(".json"): + return state_path.path + + json_state_path = state_path.siblingExtension(".json") + + # if there's no file there at all, we're done because there's + # nothing to upgrade + if not state_path.exists(): + return json_state_path.path + + # upgrade the pickle data to JSON + import pickle + with state_path.open("r") as f: + state = pickle.load(f) + state = convert_pickle(state) + json_state_path = state_path.siblingExtension(".json") + with json_state_path.open("w") as f: + json.dump(state, f) + # we've written the JSON, delete the pickle + state_path.remove() + return json_state_path.path + + +class _LeaseStateSerializer(object): + """ + Read and write state for LeaseCheckingCrawler. This understands + how to read the legacy pickle format files and upgrade them to the + new JSON format (which will occur automatically). + """ + + def __init__(self, state_path): + self._path = FilePath( + _maybe_upgrade_pickle_to_json( + FilePath(state_path), + _convert_pickle_state_to_json, + ) + ) + # XXX want this to .. load and save the state + # - if the state is pickle-only: + # - load it and convert to json format + # - save json + # - delete pickle + # - if the state is json, load it + + def load(self): + with self._path.open("r") as f: + return json.load(f) + + def save(self, data): + tmpfile = self._path.siblingExtension(".tmp") + with tmpfile.open("wb") as f: + json.dump(data, f) + fileutil.move_into_place(tmpfile.path, self._path.path) + return None + + class ShareCrawler(service.MultiService): """A ShareCrawler subclass is attached to a StorageServer, and periodically walks all of its shares, processing each one in some @@ -87,7 +220,7 @@ class ShareCrawler(service.MultiService): self.allowed_cpu_percentage = allowed_cpu_percentage self.server = server self.sharedir = server.sharedir - self.statefile = statefile + self._state_serializer = _LeaseStateSerializer(statefile) self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] if PY3: @@ -210,8 +343,7 @@ class ShareCrawler(service.MultiService): # of the last bucket to be processed, or # None if we are sleeping between cycles try: - with open(self.statefile, "rb") as f: - state = json.load(f) + state = self._state_serializer.load() except Exception: state = {"version": 1, "last-cycle-finished": None, @@ -247,14 +379,11 @@ class ShareCrawler(service.MultiService): else: last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix - tmpfile = self.statefile + ".tmp" # Note: we use self.get_state() here because e.g # LeaseCheckingCrawler stores non-JSON-able state in # self.state() but converts it in self.get_state() - with open(tmpfile, "wb") as f: - json.dump(self.get_state(), f) - fileutil.move_into_place(tmpfile, self.statefile) + self._state_serializer.save(self.get_state()) def startService(self): # arrange things to look like we were just sleeping, so diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 4513dadb2..d2f48004a 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -10,11 +10,72 @@ import json import time import os import struct -from allmydata.storage.crawler import ShareCrawler +from allmydata.storage.crawler import ( + ShareCrawler, + _maybe_upgrade_pickle_to_json, +) from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ UnknownImmutableContainerVersionError from twisted.python import log as twlog +from twisted.python.filepath import FilePath + + +def _convert_pickle_state_to_json(state): + """ + :param dict state: the pickled state + + :return dict: the state in the JSON form + """ + print("CONVERT", state) + for k, v in state.items(): + print(k, v) + if state["version"] != 1: + raise ValueError( + "Unknown version {version} in pickle state".format(**state) + ) + + return state + + +class _HistorySerializer(object): + """ + Serialize the 'history' file of the lease-crawler state. This is + "storage/history.state" for the pickle or + "storage/history.state.json" for the new JSON format. + """ + + def __init__(self, history_path): + self._path = FilePath( + _maybe_upgrade_pickle_to_json( + FilePath(history_path), + _convert_pickle_state_to_json, + ) + ) + if not self._path.exists(): + with self._path.open("wb") as f: + json.dump({}, f) + + def read(self): + """ + Deserialize the existing data. + + :return dict: the existing history state + """ + assert self._path is not None, "Not initialized" + with self._path.open("rb") as f: + history = json.load(f) + return history + + def write(self, new_history): + """ + Serialize the existing data as JSON. + """ + assert self._path is not None, "Not initialized" + with self._path.open("wb") as f: + json.dump(new_history, f) + return None + class LeaseCheckingCrawler(ShareCrawler): """I examine the leases on all shares, determining which are still valid @@ -64,7 +125,8 @@ class LeaseCheckingCrawler(ShareCrawler): override_lease_duration, # used if expiration_mode=="age" cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): - self.historyfile = historyfile + self._history_serializer = _HistorySerializer(historyfile) + ##self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None @@ -92,12 +154,6 @@ class LeaseCheckingCrawler(ShareCrawler): for k in so_far: self.state["cycle-to-date"].setdefault(k, so_far[k]) - # initialize history - if not os.path.exists(self.historyfile): - history = {} # cyclenum -> dict - with open(self.historyfile, "wb") as f: - json.dump(history, f) - def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() so_far = {"corrupt-shares": [], @@ -315,14 +371,12 @@ class LeaseCheckingCrawler(ShareCrawler): # copy() needs to become a deepcopy h["space-recovered"] = s["space-recovered"].copy() - with open(self.historyfile, "rb") as f: - history = json.load(f) + history = self._history_serializer.read() history[str(cycle)] = h while len(history) > 10: oldcycles = sorted(int(k) for k in history.keys()) del history[str(oldcycles[0])] - with open(self.historyfile, "wb") as f: - json.dump(history, f) + self._history_serializer.write(history) def get_state(self): """In addition to the crawler state described in @@ -391,9 +445,7 @@ class LeaseCheckingCrawler(ShareCrawler): progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy - with open(self.historyfile, "rb") as f: - history = json.load(f) - state["history"] = history + state["history"] = self._history_serializer.read() if not progress["cycle-in-progress"]: del state["cycle-to-date"] diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 49cb7fa82..9211535b7 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -57,6 +57,7 @@ DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 @implementer(RIStorageServer, IStatsProducer) class StorageServer(service.MultiService, Referenceable): name = 'storage' + # only the tests change this to anything else LeaseCheckerClass = LeaseCheckingCrawler def __init__(self, storedir, nodeid, reserved_space=0, diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index b9fa548d3..d91242449 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -25,14 +25,20 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString +from twisted.python.filepath import FilePath from foolscap.api import fireEventually from allmydata.util import fileutil, hashutil, base32, pollmixin from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError from allmydata.storage.server import StorageServer -from allmydata.storage.crawler import BucketCountingCrawler -from allmydata.storage.expirer import LeaseCheckingCrawler +from allmydata.storage.crawler import ( + BucketCountingCrawler, + _LeaseStateSerializer, +) +from allmydata.storage.expirer import ( + LeaseCheckingCrawler, +) from allmydata.web.storage import ( StorageStatus, StorageStatusElement, From bf5e682d71e086f351126129c2e586a80442c2bb Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 13:17:46 -0600 Subject: [PATCH 192/916] test upgrade of main state works --- src/allmydata/test/test_storage_web.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index d91242449..0b287d667 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1145,6 +1145,22 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d + def test_deserialize_pickle(self): + """ + The crawler can read existing state from the old pickle format + """ + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state") + test_pickle = FilePath("lease_checker.state") + with test_pickle.open("w") as local, original_pickle.open("r") as remote: + local.write(remote.read()) + + serial = _LeaseStateSerializer(test_pickle.path) + + # the (existing) state file should have been upgraded to JSON + self.assertNot(test_pickle.exists()) + self.assertTrue(test_pickle.siblingExtension(".json").exists()) + + class WebStatus(unittest.TestCase, pollmixin.PollMixin): From 89c2aacadca5cc7ba13f4afda2c4d3817ea9b5c0 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 15:52:01 -0600 Subject: [PATCH 193/916] working test of 'in the wild' data, working converters --- src/allmydata/storage/crawler.py | 36 +++--- src/allmydata/test/test_storage_web.py | 165 +++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 48b03ec8b..548864e06 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -34,7 +34,7 @@ def _convert_pickle_state_to_json(state): :return dict: the state in the JSON form """ # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list - # ["leases-per-share-histogram"] gets str keys instead of int + # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int # ["cycle-start-finish-times"] from 2-tuple to list # ["configured-expiration-mode"] from 4-tuple to list # ["history"] keys are strings @@ -43,12 +43,6 @@ def _convert_pickle_state_to_json(state): "Unknown version {version} in pickle state".format(**state) ) - def convert_lpsh(value): - return { - str(k): v - for k, v in value.items() - } - def convert_cem(value): # original is a 4-tuple, with the last element being a 2-tuple # .. convert both to lists @@ -59,20 +53,28 @@ def _convert_pickle_state_to_json(state): list(value[3]), ] - def convert_history(value): - print("convert history") - print(value) + def convert_ctd(value): + ctd_converter = { + "lease-age-histogram": lambda value: { + "{},{}".format(k[0], k[1]): v + for k, v in value.items() + }, + "corrupt-shares": lambda value: [ + list(x) + for x in value + ], + } return { - str(k): v - for k, v in value + k: ctd_converter.get(k, lambda z: z)(v) + for k, v in value.items() } + # we don't convert "history" here because that's in a separate + # file; see expirer.py converters = { - "cycle-to-date": list, - "leases-per-share-histogram": convert_lpsh, + "cycle-to-date": convert_ctd, "cycle-starte-finish-times": list, "configured-expiration-mode": convert_cem, - "history": convert_history, } def convert_value(key, value): @@ -116,10 +118,10 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): import pickle with state_path.open("r") as f: state = pickle.load(f) - state = convert_pickle(state) + new_state = convert_pickle(state) json_state_path = state_path.siblingExtension(".json") with json_state_path.open("w") as f: - json.dump(state, f) + json.dump(new_state, f) # we've written the JSON, delete the pickle state_path.remove() return json_state_path.path diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 0b287d667..5cdf02a25 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1160,6 +1160,171 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.assertNot(test_pickle.exists()) self.assertTrue(test_pickle.siblingExtension(".json").exists()) + self.assertEqual( + serial.load(), + { + u'last-complete-prefix': None, + u'version': 1, + u'current-cycle-start-time': 1635003106.611748, + u'last-cycle-finished': 312, + u'cycle-to-date': { + u'leases-per-share-histogram': { + u'1': 36793, + u'2': 1, + }, + u'space-recovered': { + u'examined-buckets-immutable': 17183, + u'configured-buckets-mutable': 0, + u'examined-shares-mutable': 1796, + u'original-shares-mutable': 1563, + u'configured-buckets-immutable': 0, + u'original-shares-immutable': 27926, + u'original-diskbytes-immutable': 431149056, + u'examined-shares-immutable': 34998, + u'original-buckets': 14661, + u'actual-shares-immutable': 0, + u'configured-shares': 0, + u'original-buckets-mutable': 899, + u'actual-diskbytes': 4096, + u'actual-shares-mutable': 0, + u'configured-buckets': 1, + u'examined-buckets-unknown': 14, + u'actual-sharebytes': 0, + u'original-shares': 29489, + u'actual-buckets-immutable': 0, + u'original-sharebytes': 312664812, + u'examined-sharebytes-immutable': 383801602, + u'actual-shares': 0, + u'actual-sharebytes-immutable': 0, + u'original-diskbytes': 441643008, + u'configured-diskbytes-mutable': 0, + u'configured-sharebytes-immutable': 0, + u'configured-shares-mutable': 0, + u'actual-diskbytes-immutable': 0, + u'configured-diskbytes-immutable': 0, + u'original-diskbytes-mutable': 10489856, + u'actual-sharebytes-mutable': 0, + u'configured-sharebytes': 0, + u'examined-shares': 36794, + u'actual-diskbytes-mutable': 0, + u'actual-buckets': 1, + u'original-buckets-immutable': 13761, + u'configured-sharebytes-mutable': 0, + u'examined-sharebytes': 390369660, + u'original-sharebytes-immutable': 308125753, + u'original-sharebytes-mutable': 4539059, + u'actual-buckets-mutable': 0, + u'examined-buckets-mutable': 1043, + u'configured-shares-immutable': 0, + u'examined-diskbytes': 476598272, + u'examined-diskbytes-mutable': 9154560, + u'examined-sharebytes-mutable': 6568058, + u'examined-buckets': 18241, + u'configured-diskbytes': 4096, + u'examined-diskbytes-immutable': 467443712}, + u'corrupt-shares': [ + [u'2dn6xnlnsqwtnapwxfdivpm3s4', 4], + [u'2dn6xnlnsqwtnapwxfdivpm3s4', 1], + [u'2rrzthwsrrxolevmwdvbdy3rqi', 4], + [u'2rrzthwsrrxolevmwdvbdy3rqi', 1], + [u'2skfngcto6h7eqmn4uo7ntk3ne', 4], + [u'2skfngcto6h7eqmn4uo7ntk3ne', 1], + [u'32d5swqpqx2mwix7xmqzvhdwje', 4], + [u'32d5swqpqx2mwix7xmqzvhdwje', 1], + [u'5mmayp66yflmpon3o6unsnbaca', 4], + [u'5mmayp66yflmpon3o6unsnbaca', 1], + [u'6ixhpvbtre7fnrl6pehlrlflc4', 4], + [u'6ixhpvbtre7fnrl6pehlrlflc4', 1], + [u'ewzhvswjsz4vp2bqkb6mi3bz2u', 4], + [u'ewzhvswjsz4vp2bqkb6mi3bz2u', 1], + [u'fu7pazf6ogavkqj6z4q5qqex3u', 4], + [u'fu7pazf6ogavkqj6z4q5qqex3u', 1], + [u'hbyjtqvpcimwxiyqbcbbdn2i4a', 4], + [u'hbyjtqvpcimwxiyqbcbbdn2i4a', 1], + [u'pmcjbdkbjdl26k3e6yja77femq', 4], + [u'pmcjbdkbjdl26k3e6yja77femq', 1], + [u'r6swof4v2uttbiiqwj5pi32cm4', 4], + [u'r6swof4v2uttbiiqwj5pi32cm4', 1], + [u't45v5akoktf53evc2fi6gwnv6y', 4], + [u't45v5akoktf53evc2fi6gwnv6y', 1], + [u'y6zb4faar3rdvn3e6pfg4wlotm', 4], + [u'y6zb4faar3rdvn3e6pfg4wlotm', 1], + [u'z3yghutvqoqbchjao4lndnrh3a', 4], + [u'z3yghutvqoqbchjao4lndnrh3a', 1], + ], + u'lease-age-histogram': { + "1641600,1728000": 78, + "12441600,12528000": 78, + "8640000,8726400": 32, + "1814400,1900800": 1860, + "2764800,2851200": 76, + "11491200,11577600": 20, + "10713600,10800000": 183, + "47865600,47952000": 7, + "3110400,3196800": 328, + "10627200,10713600": 43, + "45619200,45705600": 4, + "12873600,12960000": 5, + "7430400,7516800": 7228, + "1555200,1641600": 492, + "38880000,38966400": 3, + "12528000,12614400": 193, + "7344000,7430400": 12689, + "2678400,2764800": 278, + "2332800,2419200": 12, + "9244800,9331200": 73, + "12787200,12873600": 218, + "49075200,49161600": 19, + "10368000,10454400": 117, + "4665600,4752000": 256, + "7516800,7603200": 993, + "42336000,42422400": 33, + "10972800,11059200": 122, + "39052800,39139200": 51, + "12614400,12700800": 210, + "7603200,7689600": 2004, + "10540800,10627200": 16, + "950400,1036800": 4435, + "42076800,42163200": 4, + "8812800,8899200": 57, + "5788800,5875200": 954, + "36374400,36460800": 3, + "9331200,9417600": 12, + "30499200,30585600": 5, + "12700800,12787200": 25, + "2073600,2160000": 388, + "12960000,13046400": 8, + "11923200,12009600": 89, + "3369600,3456000": 79, + "3196800,3283200": 628, + "37497600,37584000": 11, + "33436800,33523200": 7, + "44928000,45014400": 2, + "37929600,38016000": 3, + "38966400,39052800": 61, + "3283200,3369600": 86, + "11750400,11836800": 7, + "3801600,3888000": 32, + "46310400,46396800": 1, + "4838400,4924800": 386, + "8208000,8294400": 38, + "37411200,37497600": 4, + "12009600,12096000": 329, + "10454400,10540800": 1239, + "40176000,40262400": 1, + "3715200,3801600": 104, + "44409600,44496000": 13, + "38361600,38448000": 5, + "12268800,12355200": 2, + "28771200,28857600": 6, + "41990400,42076800": 10, + "2592000,2678400": 40, + }, + }, + 'current-cycle': None, + 'last-complete-bucket': None, + } + ) class WebStatus(unittest.TestCase, pollmixin.PollMixin): From d4fc14f9ada372e94d68667da64643b52de9923c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 19:42:08 -0600 Subject: [PATCH 194/916] docstring --- src/allmydata/storage/expirer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index d2f48004a..ce126b6a4 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -23,6 +23,9 @@ from twisted.python.filepath import FilePath def _convert_pickle_state_to_json(state): """ + Convert a pickle-serialized crawler-history state to the new JSON + format. + :param dict state: the pickled state :return dict: the state in the JSON form From 75410e51f04f90007efce9fa2cba6504c54fe9ac Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:10:43 -0600 Subject: [PATCH 195/916] refactor --- src/allmydata/storage/crawler.py | 86 ++++++++++++++++++-------------- src/allmydata/storage/expirer.py | 14 ++---- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 548864e06..2e9bafd13 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -27,23 +27,14 @@ class TimeSliceExceeded(Exception): pass -def _convert_pickle_state_to_json(state): +def _convert_cycle_data(state): """ - :param dict state: the pickled state + :param dict state: cycle-to-date or history-item state :return dict: the state in the JSON form """ - # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list - # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int - # ["cycle-start-finish-times"] from 2-tuple to list - # ["configured-expiration-mode"] from 4-tuple to list - # ["history"] keys are strings - if state["version"] != 1: - raise ValueError( - "Unknown version {version} in pickle state".format(**state) - ) - def convert_cem(value): + def _convert_expiration_mode(value): # original is a 4-tuple, with the last element being a 2-tuple # .. convert both to lists return [ @@ -53,41 +44,60 @@ def _convert_pickle_state_to_json(state): list(value[3]), ] - def convert_ctd(value): - ctd_converter = { - "lease-age-histogram": lambda value: { + def _convert_lease_age(value): + # if we're in cycle-to-date, this is a dict + if isinstance(value, dict): + return { "{},{}".format(k[0], k[1]): v for k, v in value.items() - }, - "corrupt-shares": lambda value: [ - list(x) - for x in value - ], - } - return { - k: ctd_converter.get(k, lambda z: z)(v) - for k, v in value.items() - } + } + # otherwise, it's a history-item and they're 3-tuples + return [ + list(v) + for v in value + ] - # we don't convert "history" here because that's in a separate - # file; see expirer.py converters = { - "cycle-to-date": convert_ctd, - "cycle-starte-finish-times": list, - "configured-expiration-mode": convert_cem, + "configured-expiration-mode": _convert_expiration_mode, + "cycle-start-finish-times": list, + "lease-age-histogram": _convert_lease_age, + "corrupt-shares": lambda value: [ + list(x) + for x in value + ], + "leases-per-share-histogram": lambda value: { + str(k): v + for k, v in value.items() + }, + } + return { + k: converters.get(k, lambda z: z)(v) + for k, v in state.items() } - def convert_value(key, value): - converter = converters.get(key, None) - if converter is None: - return value - return converter(value) - new_state = { - k: convert_value(k, v) +def _convert_pickle_state_to_json(state): + """ + :param dict state: the pickled state + + :return dict: the state in the JSON form + """ + # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list + # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int + # ["cycle-start-finish-times"] from 2-tuple to list + # ["history"] keys are strings + if state["version"] != 1: + raise ValueError( + "Unknown version {version} in pickle state".format(**state) + ) + + converters = { + "cycle-to-date": _convert_cycle_data, + } + return { + k: converters.get(k, lambda x: x)(v) for k, v in state.items() } - return new_state def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index ce126b6a4..946498eaf 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -13,6 +13,7 @@ import struct from allmydata.storage.crawler import ( ShareCrawler, _maybe_upgrade_pickle_to_json, + _convert_cycle_data, ) from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ @@ -30,15 +31,10 @@ def _convert_pickle_state_to_json(state): :return dict: the state in the JSON form """ - print("CONVERT", state) - for k, v in state.items(): - print(k, v) - if state["version"] != 1: - raise ValueError( - "Unknown version {version} in pickle state".format(**state) - ) - - return state + return { + str(k): _convert_cycle_data(v) + for k, v in state.items() + } class _HistorySerializer(object): From a867294e00addbf4a0f4426820beb07895b81441 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:12:17 -0600 Subject: [PATCH 196/916] dead --- src/allmydata/storage/expirer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 946498eaf..254264e38 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -61,7 +61,6 @@ class _HistorySerializer(object): :return dict: the existing history state """ - assert self._path is not None, "Not initialized" with self._path.open("rb") as f: history = json.load(f) return history @@ -70,7 +69,6 @@ class _HistorySerializer(object): """ Serialize the existing data as JSON. """ - assert self._path is not None, "Not initialized" with self._path.open("wb") as f: json.dump(new_history, f) return None From 94670461f1d93bef6766652a03a2b9bd85916224 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:37:51 -0600 Subject: [PATCH 197/916] tests --- src/allmydata/storage/expirer.py | 10 +- src/allmydata/test/test_storage_web.py | 168 +++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 254264e38..9ba71539c 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -55,7 +55,7 @@ class _HistorySerializer(object): with self._path.open("wb") as f: json.dump({}, f) - def read(self): + def load(self): """ Deserialize the existing data. @@ -65,7 +65,7 @@ class _HistorySerializer(object): history = json.load(f) return history - def write(self, new_history): + def save(self, new_history): """ Serialize the existing data as JSON. """ @@ -368,12 +368,12 @@ class LeaseCheckingCrawler(ShareCrawler): # copy() needs to become a deepcopy h["space-recovered"] = s["space-recovered"].copy() - history = self._history_serializer.read() + history = self._history_serializer.load() history[str(cycle)] = h while len(history) > 10: oldcycles = sorted(int(k) for k in history.keys()) del history[str(oldcycles[0])] - self._history_serializer.write(history) + self._history_serializer.save(history) def get_state(self): """In addition to the crawler state described in @@ -442,7 +442,7 @@ class LeaseCheckingCrawler(ShareCrawler): progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy - state["history"] = self._history_serializer.read() + state["history"] = self._history_serializer.load() if not progress["cycle-in-progress"]: del state["cycle-to-date"] diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 5cdf02a25..033462d46 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -38,6 +38,7 @@ from allmydata.storage.crawler import ( ) from allmydata.storage.expirer import ( LeaseCheckingCrawler, + _HistorySerializer, ) from allmydata.web.storage import ( StorageStatus, @@ -1149,6 +1150,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): """ The crawler can read existing state from the old pickle format """ + # this file came from an "in the wild" tahoe version 1.16.0 original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state") test_pickle = FilePath("lease_checker.state") with test_pickle.open("w") as local, original_pickle.open("r") as remote: @@ -1326,6 +1328,172 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): } ) + def test_deserialize_history_pickle(self): + """ + The crawler can read existing history state from the old pickle + format + """ + # this file came from an "in the wild" tahoe version 1.16.0 + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history") + test_pickle = FilePath("lease_checker.history") + with test_pickle.open("w") as local, original_pickle.open("r") as remote: + local.write(remote.read()) + + serial = _HistorySerializer(test_pickle.path) + + self.maxDiff = None + self.assertEqual( + serial.load(), + { + "363": { + 'configured-expiration-mode': ['age', None, None, ['immutable', 'mutable']], + 'expiration-enabled': False, + 'leases-per-share-histogram': { + '1': 39774, + }, + 'lease-age-histogram': [ + [0, 86400, 3125], + [345600, 432000, 4175], + [950400, 1036800, 141], + [1036800, 1123200, 345], + [1123200, 1209600, 81], + [1296000, 1382400, 1832], + [1555200, 1641600, 390], + [1728000, 1814400, 12], + [2073600, 2160000, 84], + [2160000, 2246400, 228], + [2246400, 2332800, 75], + [2592000, 2678400, 644], + [2678400, 2764800, 273], + [2764800, 2851200, 94], + [2851200, 2937600, 97], + [3196800, 3283200, 143], + [3283200, 3369600, 48], + [4147200, 4233600, 374], + [4320000, 4406400, 534], + [5270400, 5356800, 1005], + [6739200, 6825600, 8704], + [6825600, 6912000, 3986], + [6912000, 6998400, 7592], + [6998400, 7084800, 2607], + [7689600, 7776000, 35], + [8035200, 8121600, 33], + [8294400, 8380800, 54], + [8640000, 8726400, 45], + [8726400, 8812800, 27], + [8812800, 8899200, 12], + [9763200, 9849600, 77], + [9849600, 9936000, 91], + [9936000, 10022400, 1210], + [10022400, 10108800, 45], + [10108800, 10195200, 186], + [10368000, 10454400, 113], + [10972800, 11059200, 21], + [11232000, 11318400, 5], + [11318400, 11404800, 19], + [11404800, 11491200, 238], + [11491200, 11577600, 159], + [11750400, 11836800, 1], + [11836800, 11923200, 32], + [11923200, 12009600, 192], + [12009600, 12096000, 222], + [12096000, 12182400, 18], + [12182400, 12268800, 224], + [12268800, 12355200, 9], + [12355200, 12441600, 9], + [12441600, 12528000, 10], + [12528000, 12614400, 6], + [12614400, 12700800, 6], + [12700800, 12787200, 18], + [12787200, 12873600, 6], + [12873600, 12960000, 62], + ], + 'cycle-start-finish-times': [1634446505.241972, 1634446666.055401], + 'space-recovered': { + 'examined-buckets-immutable': 17896, + 'configured-buckets-mutable': 0, + 'examined-shares-mutable': 2473, + 'original-shares-mutable': 1185, + 'configured-buckets-immutable': 0, + 'original-shares-immutable': 27457, + 'original-diskbytes-immutable': 2810982400, + 'examined-shares-immutable': 37301, + 'original-buckets': 14047, + 'actual-shares-immutable': 0, + 'configured-shares': 0, + 'original-buckets-mutable': 691, + 'actual-diskbytes': 4096, + 'actual-shares-mutable': 0, + 'configured-buckets': 1, + 'examined-buckets-unknown': 14, + 'actual-sharebytes': 0, + 'original-shares': 28642, + 'actual-buckets-immutable': 0, + 'original-sharebytes': 2695552941, + 'examined-sharebytes-immutable': 2754798505, + 'actual-shares': 0, + 'actual-sharebytes-immutable': 0, + 'original-diskbytes': 2818981888, + 'configured-diskbytes-mutable': 0, + 'configured-sharebytes-immutable': 0, + 'configured-shares-mutable': 0, + 'actual-diskbytes-immutable': 0, + 'configured-diskbytes-immutable': 0, + 'original-diskbytes-mutable': 7995392, + 'actual-sharebytes-mutable': 0, + 'configured-sharebytes': 0, + 'examined-shares': 39774, + 'actual-diskbytes-mutable': 0, + 'actual-buckets': 1, + 'original-buckets-immutable': 13355, + 'configured-sharebytes-mutable': 0, + 'examined-sharebytes': 2763646972, + 'original-sharebytes-immutable': 2692076909, + 'original-sharebytes-mutable': 3476032, + 'actual-buckets-mutable': 0, + 'examined-buckets-mutable': 1286, + 'configured-shares-immutable': 0, + 'examined-diskbytes': 2854801408, + 'examined-diskbytes-mutable': 12161024, + 'examined-sharebytes-mutable': 8848467, + 'examined-buckets': 19197, + 'configured-diskbytes': 4096, + 'examined-diskbytes-immutable': 2842640384 + }, + 'corrupt-shares': [ + ['2dn6xnlnsqwtnapwxfdivpm3s4', 3], + ['2dn6xnlnsqwtnapwxfdivpm3s4', 0], + ['2rrzthwsrrxolevmwdvbdy3rqi', 3], + ['2rrzthwsrrxolevmwdvbdy3rqi', 0], + ['2skfngcto6h7eqmn4uo7ntk3ne', 3], + ['2skfngcto6h7eqmn4uo7ntk3ne', 0], + ['32d5swqpqx2mwix7xmqzvhdwje', 3], + ['32d5swqpqx2mwix7xmqzvhdwje', 0], + ['5mmayp66yflmpon3o6unsnbaca', 3], + ['5mmayp66yflmpon3o6unsnbaca', 0], + ['6ixhpvbtre7fnrl6pehlrlflc4', 3], + ['6ixhpvbtre7fnrl6pehlrlflc4', 0], + ['ewzhvswjsz4vp2bqkb6mi3bz2u', 3], + ['ewzhvswjsz4vp2bqkb6mi3bz2u', 0], + ['fu7pazf6ogavkqj6z4q5qqex3u', 3], + ['fu7pazf6ogavkqj6z4q5qqex3u', 0], + ['hbyjtqvpcimwxiyqbcbbdn2i4a', 3], + ['hbyjtqvpcimwxiyqbcbbdn2i4a', 0], + ['pmcjbdkbjdl26k3e6yja77femq', 3], + ['pmcjbdkbjdl26k3e6yja77femq', 0], + ['r6swof4v2uttbiiqwj5pi32cm4', 3], + ['r6swof4v2uttbiiqwj5pi32cm4', 0], + ['t45v5akoktf53evc2fi6gwnv6y', 3], + ['t45v5akoktf53evc2fi6gwnv6y', 0], + ['y6zb4faar3rdvn3e6pfg4wlotm', 3], + ['y6zb4faar3rdvn3e6pfg4wlotm', 0], + ['z3yghutvqoqbchjao4lndnrh3a', 3], + ['z3yghutvqoqbchjao4lndnrh3a', 0], + ] + } + } + ) + class WebStatus(unittest.TestCase, pollmixin.PollMixin): From 069c332a6815c6c67b77a7af328e7bb2993d175d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:49:25 -0600 Subject: [PATCH 198/916] straight assert --- src/allmydata/storage/crawler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 2e9bafd13..d1366765e 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -86,10 +86,7 @@ def _convert_pickle_state_to_json(state): # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int # ["cycle-start-finish-times"] from 2-tuple to list # ["history"] keys are strings - if state["version"] != 1: - raise ValueError( - "Unknown version {version} in pickle state".format(**state) - ) + assert state["version"] == 1, "Only known version is 1" converters = { "cycle-to-date": _convert_cycle_data, From 9b3c55e4aa856e48b6d6fb5ee0252d69aeb64110 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:59:29 -0600 Subject: [PATCH 199/916] test a second deserialzation --- src/allmydata/test/test_storage_web.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 033462d46..70866cba9 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1327,6 +1327,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): 'last-complete-bucket': None, } ) + second_serial = _LeaseStateSerializer(serial._path.path) + self.assertEqual( + serial.load(), + second_serial.load(), + ) def test_deserialize_history_pickle(self): """ From 4f64bbaa0086af61745f7f55fd04fb716f018960 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 22:15:49 -0600 Subject: [PATCH 200/916] data --- src/allmydata/test/data/lease_checker.history | 501 ++++++++++++++++++ 1 file changed, 501 insertions(+) create mode 100644 src/allmydata/test/data/lease_checker.history diff --git a/src/allmydata/test/data/lease_checker.history b/src/allmydata/test/data/lease_checker.history new file mode 100644 index 000000000..0c27a5ad0 --- /dev/null +++ b/src/allmydata/test/data/lease_checker.history @@ -0,0 +1,501 @@ +(dp0 +I363 +(dp1 +Vconfigured-expiration-mode +p2 +(S'age' +p3 +NN(S'immutable' +p4 +S'mutable' +p5 +tp6 +tp7 +sVexpiration-enabled +p8 +I00 +sVleases-per-share-histogram +p9 +(dp10 +I1 +I39774 +ssVlease-age-histogram +p11 +(lp12 +(I0 +I86400 +I3125 +tp13 +a(I345600 +I432000 +I4175 +tp14 +a(I950400 +I1036800 +I141 +tp15 +a(I1036800 +I1123200 +I345 +tp16 +a(I1123200 +I1209600 +I81 +tp17 +a(I1296000 +I1382400 +I1832 +tp18 +a(I1555200 +I1641600 +I390 +tp19 +a(I1728000 +I1814400 +I12 +tp20 +a(I2073600 +I2160000 +I84 +tp21 +a(I2160000 +I2246400 +I228 +tp22 +a(I2246400 +I2332800 +I75 +tp23 +a(I2592000 +I2678400 +I644 +tp24 +a(I2678400 +I2764800 +I273 +tp25 +a(I2764800 +I2851200 +I94 +tp26 +a(I2851200 +I2937600 +I97 +tp27 +a(I3196800 +I3283200 +I143 +tp28 +a(I3283200 +I3369600 +I48 +tp29 +a(I4147200 +I4233600 +I374 +tp30 +a(I4320000 +I4406400 +I534 +tp31 +a(I5270400 +I5356800 +I1005 +tp32 +a(I6739200 +I6825600 +I8704 +tp33 +a(I6825600 +I6912000 +I3986 +tp34 +a(I6912000 +I6998400 +I7592 +tp35 +a(I6998400 +I7084800 +I2607 +tp36 +a(I7689600 +I7776000 +I35 +tp37 +a(I8035200 +I8121600 +I33 +tp38 +a(I8294400 +I8380800 +I54 +tp39 +a(I8640000 +I8726400 +I45 +tp40 +a(I8726400 +I8812800 +I27 +tp41 +a(I8812800 +I8899200 +I12 +tp42 +a(I9763200 +I9849600 +I77 +tp43 +a(I9849600 +I9936000 +I91 +tp44 +a(I9936000 +I10022400 +I1210 +tp45 +a(I10022400 +I10108800 +I45 +tp46 +a(I10108800 +I10195200 +I186 +tp47 +a(I10368000 +I10454400 +I113 +tp48 +a(I10972800 +I11059200 +I21 +tp49 +a(I11232000 +I11318400 +I5 +tp50 +a(I11318400 +I11404800 +I19 +tp51 +a(I11404800 +I11491200 +I238 +tp52 +a(I11491200 +I11577600 +I159 +tp53 +a(I11750400 +I11836800 +I1 +tp54 +a(I11836800 +I11923200 +I32 +tp55 +a(I11923200 +I12009600 +I192 +tp56 +a(I12009600 +I12096000 +I222 +tp57 +a(I12096000 +I12182400 +I18 +tp58 +a(I12182400 +I12268800 +I224 +tp59 +a(I12268800 +I12355200 +I9 +tp60 +a(I12355200 +I12441600 +I9 +tp61 +a(I12441600 +I12528000 +I10 +tp62 +a(I12528000 +I12614400 +I6 +tp63 +a(I12614400 +I12700800 +I6 +tp64 +a(I12700800 +I12787200 +I18 +tp65 +a(I12787200 +I12873600 +I6 +tp66 +a(I12873600 +I12960000 +I62 +tp67 +asVcycle-start-finish-times +p68 +(F1634446505.241972 +F1634446666.055401 +tp69 +sVspace-recovered +p70 +(dp71 +Vexamined-buckets-immutable +p72 +I17896 +sVconfigured-buckets-mutable +p73 +I0 +sVexamined-shares-mutable +p74 +I2473 +sVoriginal-shares-mutable +p75 +I1185 +sVconfigured-buckets-immutable +p76 +I0 +sVoriginal-shares-immutable +p77 +I27457 +sVoriginal-diskbytes-immutable +p78 +I2810982400 +sVexamined-shares-immutable +p79 +I37301 +sVoriginal-buckets +p80 +I14047 +sVactual-shares-immutable +p81 +I0 +sVconfigured-shares +p82 +I0 +sVoriginal-buckets-mutable +p83 +I691 +sVactual-diskbytes +p84 +I4096 +sVactual-shares-mutable +p85 +I0 +sVconfigured-buckets +p86 +I1 +sVexamined-buckets-unknown +p87 +I14 +sVactual-sharebytes +p88 +I0 +sVoriginal-shares +p89 +I28642 +sVactual-buckets-immutable +p90 +I0 +sVoriginal-sharebytes +p91 +I2695552941 +sVexamined-sharebytes-immutable +p92 +I2754798505 +sVactual-shares +p93 +I0 +sVactual-sharebytes-immutable +p94 +I0 +sVoriginal-diskbytes +p95 +I2818981888 +sVconfigured-diskbytes-mutable +p96 +I0 +sVconfigured-sharebytes-immutable +p97 +I0 +sVconfigured-shares-mutable +p98 +I0 +sVactual-diskbytes-immutable +p99 +I0 +sVconfigured-diskbytes-immutable +p100 +I0 +sVoriginal-diskbytes-mutable +p101 +I7995392 +sVactual-sharebytes-mutable +p102 +I0 +sVconfigured-sharebytes +p103 +I0 +sVexamined-shares +p104 +I39774 +sVactual-diskbytes-mutable +p105 +I0 +sVactual-buckets +p106 +I1 +sVoriginal-buckets-immutable +p107 +I13355 +sVconfigured-sharebytes-mutable +p108 +I0 +sVexamined-sharebytes +p109 +I2763646972 +sVoriginal-sharebytes-immutable +p110 +I2692076909 +sVoriginal-sharebytes-mutable +p111 +I3476032 +sVactual-buckets-mutable +p112 +I0 +sVexamined-buckets-mutable +p113 +I1286 +sVconfigured-shares-immutable +p114 +I0 +sVexamined-diskbytes +p115 +I2854801408 +sVexamined-diskbytes-mutable +p116 +I12161024 +sVexamined-sharebytes-mutable +p117 +I8848467 +sVexamined-buckets +p118 +I19197 +sVconfigured-diskbytes +p119 +I4096 +sVexamined-diskbytes-immutable +p120 +I2842640384 +ssVcorrupt-shares +p121 +(lp122 +(V2dn6xnlnsqwtnapwxfdivpm3s4 +p123 +I3 +tp124 +a(g123 +I0 +tp125 +a(V2rrzthwsrrxolevmwdvbdy3rqi +p126 +I3 +tp127 +a(g126 +I0 +tp128 +a(V2skfngcto6h7eqmn4uo7ntk3ne +p129 +I3 +tp130 +a(g129 +I0 +tp131 +a(V32d5swqpqx2mwix7xmqzvhdwje +p132 +I3 +tp133 +a(g132 +I0 +tp134 +a(V5mmayp66yflmpon3o6unsnbaca +p135 +I3 +tp136 +a(g135 +I0 +tp137 +a(V6ixhpvbtre7fnrl6pehlrlflc4 +p138 +I3 +tp139 +a(g138 +I0 +tp140 +a(Vewzhvswjsz4vp2bqkb6mi3bz2u +p141 +I3 +tp142 +a(g141 +I0 +tp143 +a(Vfu7pazf6ogavkqj6z4q5qqex3u +p144 +I3 +tp145 +a(g144 +I0 +tp146 +a(Vhbyjtqvpcimwxiyqbcbbdn2i4a +p147 +I3 +tp148 +a(g147 +I0 +tp149 +a(Vpmcjbdkbjdl26k3e6yja77femq +p150 +I3 +tp151 +a(g150 +I0 +tp152 +a(Vr6swof4v2uttbiiqwj5pi32cm4 +p153 +I3 +tp154 +a(g153 +I0 +tp155 +a(Vt45v5akoktf53evc2fi6gwnv6y +p156 +I3 +tp157 +a(g156 +I0 +tp158 +a(Vy6zb4faar3rdvn3e6pfg4wlotm +p159 +I3 +tp160 +a(g159 +I0 +tp161 +a(Vz3yghutvqoqbchjao4lndnrh3a +p162 +I3 +tp163 +a(g162 +I0 +tp164 +ass. \ No newline at end of file From 1c93175583365966f0b476ecc95720efcbf2f827 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 22:42:33 -0600 Subject: [PATCH 201/916] cleanup --- src/allmydata/storage/crawler.py | 22 ++++------------------ src/allmydata/storage/expirer.py | 1 - 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index d1366765e..a06806d17 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -82,10 +82,6 @@ def _convert_pickle_state_to_json(state): :return dict: the state in the JSON form """ - # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list - # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int - # ["cycle-start-finish-times"] from 2-tuple to list - # ["history"] keys are strings assert state["version"] == 1, "Only known version is 1" converters = { @@ -123,12 +119,12 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): # upgrade the pickle data to JSON import pickle - with state_path.open("r") as f: + with state_path.open("rb") as f: state = pickle.load(f) new_state = convert_pickle(state) - json_state_path = state_path.siblingExtension(".json") - with json_state_path.open("w") as f: + with json_state_path.open("wb") as f: json.dump(new_state, f) + # we've written the JSON, delete the pickle state_path.remove() return json_state_path.path @@ -148,15 +144,9 @@ class _LeaseStateSerializer(object): _convert_pickle_state_to_json, ) ) - # XXX want this to .. load and save the state - # - if the state is pickle-only: - # - load it and convert to json format - # - save json - # - delete pickle - # - if the state is json, load it def load(self): - with self._path.open("r") as f: + with self._path.open("rb") as f: return json.load(f) def save(self, data): @@ -388,10 +378,6 @@ class ShareCrawler(service.MultiService): else: last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix - - # Note: we use self.get_state() here because e.g - # LeaseCheckingCrawler stores non-JSON-able state in - # self.state() but converts it in self.get_state() self._state_serializer.save(self.get_state()) def startService(self): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 9ba71539c..ad1343ef5 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -123,7 +123,6 @@ class LeaseCheckingCrawler(ShareCrawler): cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): self._history_serializer = _HistorySerializer(historyfile) - ##self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None From 23ff1b2430334d376abd426421039562f94bcfc8 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 22:45:08 -0600 Subject: [PATCH 202/916] noqa --- src/allmydata/storage/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index a06806d17..129659d27 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2, PY3 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import time From 2fe686135bf9513cbdbbe700e5faa63ee1743e83 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 23:33:54 -0600 Subject: [PATCH 203/916] rename data to appease distutils --- .../data/{lease_checker.history => lease_checker.history.txt} | 0 .../data/{lease_checker.state => lease_checker.state.txt} | 0 src/allmydata/test/test_storage_web.py | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/allmydata/test/data/{lease_checker.history => lease_checker.history.txt} (100%) rename src/allmydata/test/data/{lease_checker.state => lease_checker.state.txt} (100%) diff --git a/src/allmydata/test/data/lease_checker.history b/src/allmydata/test/data/lease_checker.history.txt similarity index 100% rename from src/allmydata/test/data/lease_checker.history rename to src/allmydata/test/data/lease_checker.history.txt diff --git a/src/allmydata/test/data/lease_checker.state b/src/allmydata/test/data/lease_checker.state.txt similarity index 100% rename from src/allmydata/test/data/lease_checker.state rename to src/allmydata/test/data/lease_checker.state.txt diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 70866cba9..269af2203 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1151,7 +1151,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): The crawler can read existing state from the old pickle format """ # this file came from an "in the wild" tahoe version 1.16.0 - original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state") + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state.txt") test_pickle = FilePath("lease_checker.state") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) @@ -1339,7 +1339,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): format """ # this file came from an "in the wild" tahoe version 1.16.0 - original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history") + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history.txt") test_pickle = FilePath("lease_checker.history") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) From a208502e18c4f4faf85d500e86e3b2093d219ecf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 16 Nov 2021 18:29:01 -0500 Subject: [PATCH 204/916] whitespace --- src/allmydata/storage/lease.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 63dba15e8..bc94ca6d5 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -272,7 +272,7 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign Hash the candidate secret and compare the result to the stored hashed secret. """ - if isinstance(candidate_secret, _HashedCancelSecret): + if isinstance(candidate_secret, _HashedCancelSecret): # Someone read it off of this object in this project - probably # the lease crawler - and is just trying to use it to identify # which lease it wants to operate on. Avoid re-hashing the value. From 3a8432713fb0885f3795d4501c77e80a21caea5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 16 Nov 2021 18:29:05 -0500 Subject: [PATCH 205/916] a note about what's happening with proxyForInterface --- src/allmydata/storage/lease.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index bc94ca6d5..0c3b219f6 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -260,6 +260,10 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign _lease_info = attr.ib() _hash = attr.ib() + # proxyForInterface will take care of forwarding all methods on ILeaseInfo + # to `_lease_info`. Here we override a few of those methods to adjust + # their behavior to make them suitable for use with hashed secrets. + def is_renew_secret(self, candidate_secret): """ Hash the candidate secret and compare the result to the stored hashed From e8adca40abdfa9f4c8616194bbe0bf1fe8817f1f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 16 Nov 2021 18:32:35 -0500 Subject: [PATCH 206/916] give the ContainerVersionError exceptions a nice str --- src/allmydata/storage/common.py | 6 ++++++ src/allmydata/test/test_storage.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 48fc77840..17a3f41b7 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -21,6 +21,12 @@ class UnknownContainerVersionError(Exception): self.filename = filename self.version = version + def __str__(self): + return "sharefile {!r} had unexpected version {!r}".format( + self.filename, + self.version, + ) + class UnknownMutableContainerVersionError(UnknownContainerVersionError): pass diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 655395042..ba3d3598f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -651,6 +651,7 @@ class Server(unittest.TestCase): ss.remote_get_buckets, b"si1") self.assertEqual(e.filename, fn) self.assertEqual(e.version, 0) + self.assertIn("had unexpected version 0", str(e)) def test_disconnect(self): # simulate a disconnection @@ -1136,6 +1137,8 @@ class MutableServer(unittest.TestCase): read, b"si1", [0], [(0,10)]) self.assertEqual(e.filename, fn) self.assertTrue(e.version.startswith(b"BAD MAGIC")) + self.assertIn("had unexpected version", str(e)) + self.assertIn("BAD MAGIC", str(e)) def test_container_size(self): ss = self.create("test_container_size") From 6a78703675e9ce9e42a8401ac38410d78357a86e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 10:53:51 -0500 Subject: [PATCH 207/916] News file. --- newsfragments/3807.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3807.feature diff --git a/newsfragments/3807.feature b/newsfragments/3807.feature new file mode 100644 index 000000000..f82363ffd --- /dev/null +++ b/newsfragments/3807.feature @@ -0,0 +1 @@ +If uploading an immutable hasn't had a write for 30 minutes, the storage server will abort the upload. \ No newline at end of file From 92c36a67d8c98436e2cc2616d89ff0135307858c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:01:04 -0500 Subject: [PATCH 208/916] Use IReactorTime instead of ad-hoc solutions. --- src/allmydata/storage/server.py | 39 ++++++++++++----------- src/allmydata/test/test_istorageserver.py | 10 +++--- src/allmydata/test/test_storage.py | 16 +++++----- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ee2ea1c61..499d47276 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -20,6 +20,7 @@ import six from foolscap.api import Referenceable from foolscap.ipb import IRemoteReference from twisted.application import service +from twisted.internet import reactor from zope.interface import implementer from allmydata.interfaces import RIStorageServer, IStatsProducer @@ -71,7 +72,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_override_lease_duration=None, expiration_cutoff_date=None, expiration_sharetypes=("mutable", "immutable"), - get_current_time=time.time): + clock=reactor): service.MultiService.__init__(self) assert isinstance(nodeid, bytes) assert len(nodeid) == 20 @@ -122,7 +123,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_cutoff_date, expiration_sharetypes) self.lease_checker.setServiceParent(self) - self._get_current_time = get_current_time + self._clock = clock # Currently being-written Bucketwriters. For Foolscap, lifetime is tied # to connection: when disconnection happens, the BucketWriters are @@ -292,7 +293,7 @@ class StorageServer(service.MultiService, Referenceable): # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. - start = self._get_current_time() + start = self._clock.seconds() self.count("allocate") alreadygot = set() bucketwriters = {} # k: shnum, v: BucketWriter @@ -305,7 +306,7 @@ class StorageServer(service.MultiService, Referenceable): # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. - expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) @@ -360,7 +361,7 @@ class StorageServer(service.MultiService, Referenceable): if bucketwriters: fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) - self.add_latency("allocate", self._get_current_time() - start) + self.add_latency("allocate", self._clock.seconds() - start) return alreadygot, bucketwriters def remote_allocate_buckets(self, storage_index, @@ -395,26 +396,26 @@ class StorageServer(service.MultiService, Referenceable): def remote_add_lease(self, storage_index, renew_secret, cancel_secret, owner_num=1): - start = self._get_current_time() + start = self._clock.seconds() self.count("add-lease") - new_expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) for sf in self._iter_share_files(storage_index): sf.add_or_renew_lease(lease_info) - self.add_latency("add-lease", self._get_current_time() - start) + self.add_latency("add-lease", self._clock.seconds() - start) return None def remote_renew_lease(self, storage_index, renew_secret): - start = self._get_current_time() + start = self._clock.seconds() self.count("renew") - new_expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME found_buckets = False for sf in self._iter_share_files(storage_index): found_buckets = True sf.renew_lease(renew_secret, new_expire_time) - self.add_latency("renew", self._get_current_time() - start) + self.add_latency("renew", self._clock.seconds() - start) if not found_buckets: raise IndexError("no such lease to renew") @@ -441,7 +442,7 @@ class StorageServer(service.MultiService, Referenceable): pass def remote_get_buckets(self, storage_index): - start = self._get_current_time() + start = self._clock.seconds() self.count("get") si_s = si_b2a(storage_index) log.msg("storage: get_buckets %r" % si_s) @@ -449,7 +450,7 @@ class StorageServer(service.MultiService, Referenceable): for shnum, filename in self._get_bucket_shares(storage_index): bucketreaders[shnum] = BucketReader(self, filename, storage_index, shnum) - self.add_latency("get", self._get_current_time() - start) + self.add_latency("get", self._clock.seconds() - start) return bucketreaders def get_leases(self, storage_index): @@ -608,7 +609,7 @@ class StorageServer(service.MultiService, Referenceable): :return LeaseInfo: Information for a new lease for a share. """ ownerid = 1 # TODO - expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) @@ -646,7 +647,7 @@ class StorageServer(service.MultiService, Referenceable): See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. """ - start = self._get_current_time() + start = self._clock.seconds() self.count("writev") si_s = si_b2a(storage_index) log.msg("storage: slot_writev %r" % si_s) @@ -687,7 +688,7 @@ class StorageServer(service.MultiService, Referenceable): self._add_or_renew_leases(remaining_shares, lease_info) # all done - self.add_latency("writev", self._get_current_time() - start) + self.add_latency("writev", self._clock.seconds() - start) return (testv_is_good, read_data) def remote_slot_testv_and_readv_and_writev(self, storage_index, @@ -713,7 +714,7 @@ class StorageServer(service.MultiService, Referenceable): return share def remote_slot_readv(self, storage_index, shares, readv): - start = self._get_current_time() + start = self._clock.seconds() self.count("readv") si_s = si_b2a(storage_index) lp = log.msg("storage: slot_readv %r %r" % (si_s, shares), @@ -722,7 +723,7 @@ class StorageServer(service.MultiService, Referenceable): # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): - self.add_latency("readv", self._get_current_time() - start) + self.add_latency("readv", self._clock.seconds() - start) return {} datavs = {} for sharenum_s in os.listdir(bucketdir): @@ -736,7 +737,7 @@ class StorageServer(service.MultiService, Referenceable): datavs[sharenum] = msf.readv(readv) log.msg("returning shares %s" % (list(datavs.keys()),), facility="tahoe.storage", level=log.NOISY, parent=lp) - self.add_latency("readv", self._get_current_time() - start) + self.add_latency("readv", self._clock.seconds() - start) return datavs def remote_advise_corrupt_share(self, share_type, storage_index, shnum, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fe494a9d4..a17264713 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -21,6 +21,7 @@ if PY2: from random import Random from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.task import Clock from foolscap.api import Referenceable, RemoteException @@ -1017,16 +1018,17 @@ class _FoolscapMixin(SystemTestMixin): self.server = s break assert self.server is not None, "Couldn't find StorageServer" - self._current_time = 123456 - self.server._get_current_time = self.fake_time + self._clock = Clock() + self._clock.advance(123456) + self.server._clock = self._clock def fake_time(self): """Return the current fake, test-controlled, time.""" - return self._current_time + return self._clock.seconds() def fake_sleep(self, seconds): """Advance the fake time by the given number of seconds.""" - self._current_time += seconds + self._clock.advance(seconds) @inlineCallbacks def tearDown(self): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 4e40a76a5..e143bec63 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -23,7 +23,7 @@ from uuid import uuid4 from twisted.trial import unittest -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.internet.task import Clock from hypothesis import given, strategies @@ -438,11 +438,11 @@ class Server(unittest.TestCase): basedir = os.path.join("storage", "Server", name) return basedir - def create(self, name, reserved_space=0, klass=StorageServer, get_current_time=time.time): + def create(self, name, reserved_space=0, klass=StorageServer, clock=reactor): workdir = self.workdir(name) ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space, stats_provider=FakeStatsProvider(), - get_current_time=get_current_time) + clock=clock) ss.setServiceParent(self.sparent) return ss @@ -626,7 +626,7 @@ class Server(unittest.TestCase): clock.advance(first_lease) ss = self.create( "test_allocate_without_lease_renewal", - get_current_time=clock.seconds, + clock=clock, ) # Put a share on there @@ -918,7 +918,7 @@ class Server(unittest.TestCase): """ clock = Clock() clock.advance(123) - ss = self.create("test_immutable_add_lease_renews", get_current_time=clock.seconds) + ss = self.create("test_immutable_add_lease_renews", clock=clock) # Start out with single lease created with bucket: renewal_secret, cancel_secret = self.create_bucket_5_shares(ss, b"si0") @@ -1032,10 +1032,10 @@ class MutableServer(unittest.TestCase): basedir = os.path.join("storage", "MutableServer", name) return basedir - def create(self, name, get_current_time=time.time): + def create(self, name, clock=reactor): workdir = self.workdir(name) ss = StorageServer(workdir, b"\x00" * 20, - get_current_time=get_current_time) + clock=clock) ss.setServiceParent(self.sparent) return ss @@ -1420,7 +1420,7 @@ class MutableServer(unittest.TestCase): clock = Clock() clock.advance(235) ss = self.create("test_mutable_add_lease_renews", - get_current_time=clock.seconds) + clock=clock) def secrets(n): return ( self.write_enabler(b"we1"), self.renew_secret(b"we1-%d" % n), From bf7d03310fc1bd730ba12449112144f3e6935020 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:09:45 -0500 Subject: [PATCH 209/916] Hide all _trial_temp. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 50a1352a2..7c7fa2afd 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,7 @@ zope.interface-*.egg .pc /src/allmydata/test/plugins/dropin.cache -/_trial_temp* +**/_trial_temp* /tmp* /*.patch /dist/ From 45c00e93c9709e216fe87410783b0a6125e159e7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:12:40 -0500 Subject: [PATCH 210/916] Use clock in BucketWriter. --- src/allmydata/storage/immutable.py | 11 ++++++----- src/allmydata/storage/server.py | 3 ++- src/allmydata/test/test_storage.py | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 8a7a5a966..7cfb7a1bf 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -233,7 +233,7 @@ class ShareFile(object): @implementer(RIBucketWriter) class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 - def __init__(self, ss, incominghome, finalhome, max_size, lease_info): + def __init__(self, ss, incominghome, finalhome, max_size, lease_info, clock): self.ss = ss self.incominghome = incominghome self.finalhome = finalhome @@ -245,12 +245,13 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 # added by simultaneous uploaders self._sharefile.add_lease(lease_info) self._already_written = RangeMap() + self._clock = clock def allocated_size(self): return self._max_size def remote_write(self, offset, data): - start = time.time() + start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: return @@ -268,12 +269,12 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._sharefile.write_share_data(offset, data) self._already_written.set(True, offset, end) - self.ss.add_latency("write", time.time() - start) + self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") def remote_close(self): precondition(not self.closed) - start = time.time() + start = self._clock.seconds() fileutil.make_dirs(os.path.dirname(self.finalhome)) fileutil.rename(self.incominghome, self.finalhome) @@ -306,7 +307,7 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 filelen = os.stat(self.finalhome)[stat.ST_SIZE] self.ss.bucket_writer_closed(self, filelen) - self.ss.add_latency("close", time.time() - start) + self.ss.add_latency("close", self._clock.seconds() - start) self.ss.count("close") def disconnected(self): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 499d47276..080c1aea1 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -347,7 +347,8 @@ class StorageServer(service.MultiService, Referenceable): elif (not limited) or (remaining_space >= max_space_per_bucket): # ok! we need to create the new share file. bw = BucketWriter(self, incominghome, finalhome, - max_space_per_bucket, lease_info) + max_space_per_bucket, lease_info, + clock=self._clock) if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e143bec63..36c776fba 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -128,7 +128,7 @@ class Bucket(unittest.TestCase): def test_create(self): incoming, final = self.make_workdir("test_create") - bw = BucketWriter(self, incoming, final, 200, self.make_lease()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) bw.remote_write(0, b"a"*25) bw.remote_write(25, b"b"*25) bw.remote_write(50, b"c"*25) @@ -137,7 +137,7 @@ class Bucket(unittest.TestCase): def test_readwrite(self): incoming, final = self.make_workdir("test_readwrite") - bw = BucketWriter(self, incoming, final, 200, self.make_lease()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) bw.remote_write(0, b"a"*25) bw.remote_write(25, b"b"*25) bw.remote_write(50, b"c"*7) # last block may be short @@ -155,7 +155,7 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir( "test_write_past_size_errors-{}".format(i) ) - bw = BucketWriter(self, incoming, final, 200, self.make_lease()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) with self.assertRaises(DataTooLargeError): bw.remote_write(offset, b"a" * length) @@ -174,7 +174,7 @@ class Bucket(unittest.TestCase): expected_data = b"".join(bchr(i) for i in range(100)) incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), + self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. bw.remote_write(10, expected_data[10:20]) @@ -212,7 +212,7 @@ class Bucket(unittest.TestCase): length = 100 incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), + self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. bw.remote_write(10, b"1" * 10) @@ -312,7 +312,7 @@ class BucketProxy(unittest.TestCase): final = os.path.join(basedir, "bucket") fileutil.make_dirs(basedir) fileutil.make_dirs(os.path.join(basedir, "tmp")) - bw = BucketWriter(self, incoming, final, size, self.make_lease()) + bw = BucketWriter(self, incoming, final, size, self.make_lease(), Clock()) rb = RemoteBucket(bw) return bw, rb, final From 5e341ad43a85444ffc3c12c685463171a53838ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:29:34 -0500 Subject: [PATCH 211/916] New tests to write. --- src/allmydata/test/test_storage.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 36c776fba..93779bb29 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -285,6 +285,22 @@ class Bucket(unittest.TestCase): result_of_read = br.remote_read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) + def test_bucket_expires_if_no_writes_for_30_minutes(self): + pass + + def test_bucket_writes_delay_timeout(self): + pass + + def test_bucket_finishing_writiing_cancels_timeout(self): + pass + + def test_bucket_closing_cancels_timeout(self): + pass + + def test_bucket_aborting_cancels_timeout(self): + pass + + class RemoteBucket(object): def __init__(self, target): @@ -559,7 +575,6 @@ class Server(unittest.TestCase): writer.remote_abort() self.failUnlessEqual(ss.allocated_size(), 0) - def test_allocate(self): ss = self.create("test_allocate") From 92e8d78a3db530f76c11188a9dc8a543d03b6fbb Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 18 Nov 2021 11:46:01 +0100 Subject: [PATCH 212/916] stronger language for adding contributors & make commands stand out Signed-off-by: fenn-cs --- README.rst | 5 +++-- docs/release-checklist.rst | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 20748a8db..0b73b520e 100644 --- a/README.rst +++ b/README.rst @@ -95,12 +95,13 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of - `Patch reviews `__ -Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. +Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. + 🥳 First Contribution? ---------------------- -If you are committing to Tahoe for the very first time, consider adding your name to our contributor list in `CREDITS `__ +If you are committing to Tahoe for the very first time, it's required that you add your name to our contributor list in `CREDITS `__. Please ensure that this addition has it's own commit within your first contribution. 🤝 Supporters diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 403a6f933..3f075dd34 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -17,7 +17,7 @@ This checklist is based on the original instructions (in old revisions in the fi Any Contributor -``````````````` +--------------- Anyone who can create normal PRs should be able to complete this portion of the release process. @@ -50,7 +50,11 @@ previously generated files. practice to give it the release name. You MAY also discard this directory once the release process is complete.* -- ``cd`` into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]`` +Get into the release directory and install dependencies by running + +- ``cd ../tahoe-release-x.x.x`` (assuming you are still in your original clone) +- ``python -m venv venv`` +- ``./venv/bin/pip install --editable .[test]`` Create Branch and Apply Updates @@ -137,16 +141,18 @@ they will need to evaluate which contributors' signatures they trust. - build all code locally - these should all pass: - - tox -e py27,codechecks,docs,integration + - ``tox -e py27,codechecks,docs,integration`` - these can fail (ideally they should not of course): - - tox -e deprecations,upcoming-deprecations + - ``tox -e deprecations,upcoming-deprecations`` - build tarballs - tox -e tarballs - - Confirm that release tarballs exist by runnig: ``ls dist/ | grep 1.16.0rc0`` + - Confirm that release tarballs exist by runnig: + + - ``ls dist/ | grep 1.16.0rc0`` - inspect and test the tarballs @@ -160,7 +166,7 @@ they will need to evaluate which contributors' signatures they trust. Privileged Contributor -`````````````````````` +----------------------- Steps in this portion require special access to keys or infrastructure. For example, **access to tahoe-lafs.org** to upload From 767948759d2a1a88acf61f0f5a843c7183c46778 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 18 Nov 2021 11:48:32 +0100 Subject: [PATCH 213/916] correct indent Signed-off-by: fenn-cs --- docs/release-checklist.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3f075dd34..0ba94df3a 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -139,6 +139,7 @@ they will need to evaluate which contributors' signatures they trust. *Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0`* - build all code locally + - these should all pass: - ``tox -e py27,codechecks,docs,integration`` From e9ae3aa885227f4ba8fa0f33ede779d1c698ae13 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 18 Nov 2021 12:04:56 +0100 Subject: [PATCH 214/916] move gpg signing instructions to seperate file Signed-off-by: fenn-cs --- docs/gpg-setup.rst | 18 ++++++++++++++++++ docs/release-checklist.rst | 22 +--------------------- 2 files changed, 19 insertions(+), 21 deletions(-) create mode 100644 docs/gpg-setup.rst diff --git a/docs/gpg-setup.rst b/docs/gpg-setup.rst new file mode 100644 index 000000000..cb8cbfd20 --- /dev/null +++ b/docs/gpg-setup.rst @@ -0,0 +1,18 @@ +Preparing to Authenticate Release (Setting up GPG) +-------------------------------------------------- + +In other to keep releases authentic it's required that releases are signed before being +published. This ensure's that users of Tahoe are able to verify that the version of Tahoe +they are using is coming from a trusted or at the very least known source. + +The authentication is done using the ``GPG`` implementation of ``OpenGPG`` to be able to complete +the release steps you would have to download the ``GPG`` software and setup a key(identity). + +- `Download `__ and install GPG for your operating system. +- Generate a key pair using ``gpg --gen-key``. *Some questions would be asked to personalize your key configuration.* + +You might take additional steps including: + +- Setting up a revocation certificate (Incase you lose your secret key) +- Backing up your key pair +- Upload your fingerprint to a keyserver such as `openpgp.org `__ diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 0ba94df3a..3b313da61 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -34,6 +34,7 @@ Tuesday if you want to get anything in"). - Create a ticket for the release in Trac - Ticket number needed in next section +- Making first release? See `GPG Setup Instructions `__ to make sure you can sign releases. [One time setup] Get a clean checkout ```````````````````` @@ -96,27 +97,6 @@ Create Branch and Apply Updates - Confirm CI runs successfully on all platforms -Preparing to Authenticate Release (Setting up GPG) -`````````````````````````````````````````````````` -*Skip the section if you already have GPG setup.* - -In other to keep releases authentic it's required that releases are signed before being -published. This ensure's that users of Tahoe are able to verify that the version of Tahoe -they are using is coming from a trusted or at the very least known source. - -The authentication is done using the ``GPG`` implementation of ``OpenGPG`` to be able to complete -the release steps you would have to download the ``GPG`` software and setup a key(identity). - -- `Download `__ and install GPG for your operating system. -- Generate a key pair using ``gpg --gen-key``. *Some questions would be asked to personalize your key configuration.* - -You might take additional steps including: - -- Setting up a revocation certificate (Incase you lose your secret key) -- Backing up your key pair -- Upload your fingerprint to a keyserver such as `openpgp.org `__ - - Create Release Candidate ```````````````````````` From 8c8e377466bcf2659029f7d59636d5039e12abf7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 14:35:04 -0500 Subject: [PATCH 215/916] Implement timeout and corresponding tests. --- src/allmydata/storage/immutable.py | 28 +++++++++++++-- src/allmydata/test/test_storage.py | 58 ++++++++++++++++++++++++++---- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 7cfb7a1bf..8a7519b7b 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -246,11 +246,17 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._sharefile.add_lease(lease_info) self._already_written = RangeMap() self._clock = clock + self._timeout = clock.callLater(30 * 60, self._abort_due_to_timeout) def allocated_size(self): return self._max_size def remote_write(self, offset, data): + self.write(offset, data) + + def write(self, offset, data): + # Delay the timeout, since we received data: + self._timeout.reset(30 * 60) start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: @@ -273,7 +279,11 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self.ss.count("write") def remote_close(self): + self.close() + + def close(self): precondition(not self.closed) + self._timeout.cancel() start = self._clock.seconds() fileutil.make_dirs(os.path.dirname(self.finalhome)) @@ -312,15 +322,23 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def disconnected(self): if not self.closed: - self._abort() + self.abort() + + def _abort_due_to_timeout(self): + """ + Called if we run out of time. + """ + log.msg("storage: aborting sharefile %s due to timeout" % self.incominghome, + facility="tahoe.storage", level=log.UNUSUAL) + self.abort() def remote_abort(self): log.msg("storage: aborting sharefile %s" % self.incominghome, facility="tahoe.storage", level=log.UNUSUAL) - self._abort() + self.abort() self.ss.count("abort") - def _abort(self): + def abort(self): if self.closed: return @@ -338,6 +356,10 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self.closed = True self.ss.bucket_writer_closed(self, 0) + # Cancel timeout if it wasn't already cancelled. + if self._timeout.active(): + self._timeout.cancel() + @implementer(RIBucketReader) class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 93779bb29..18dca9856 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -285,20 +285,64 @@ class Bucket(unittest.TestCase): result_of_read = br.remote_read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) + def _assert_timeout_only_after_30_minutes(self, clock, bw): + """ + The ``BucketWriter`` times out and is closed after 30 minutes, but not + sooner. + """ + self.assertFalse(bw.closed) + # 29 minutes pass. Everything is fine. + for i in range(29): + clock.advance(60) + self.assertFalse(bw.closed, "Bucket closed after only %d minutes" % (i + 1,)) + # After the 30th minute, the bucket is closed due to lack of writes. + clock.advance(60) + self.assertTrue(bw.closed) + def test_bucket_expires_if_no_writes_for_30_minutes(self): - pass + """ + If a ``BucketWriter`` receives no writes for 30 minutes, it is removed. + """ + incoming, final = self.make_workdir("test_bucket_expires") + clock = Clock() + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), clock) + self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_writes_delay_timeout(self): - pass - - def test_bucket_finishing_writiing_cancels_timeout(self): - pass + """ + So long as the ``BucketWriter`` receives writes, the the removal + timeout is put off. + """ + incoming, final = self.make_workdir("test_bucket_writes_delay_timeout") + clock = Clock() + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), clock) + # 20 minutes pass, getting close to the timeout... + clock.advance(29 * 60) + # .. but we receive a write! So that should delay the timeout. + bw.write(0, b"hello") + self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_closing_cancels_timeout(self): - pass + """ + Closing cancels the ``BucketWriter`` timeout. + """ + incoming, final = self.make_workdir("test_bucket_close_timeout") + clock = Clock() + bw = BucketWriter(self, incoming, final, 10, self.make_lease(), clock) + self.assertTrue(clock.getDelayedCalls()) + bw.close() + self.assertFalse(clock.getDelayedCalls()) def test_bucket_aborting_cancels_timeout(self): - pass + """ + Closing cancels the ``BucketWriter`` timeout. + """ + incoming, final = self.make_workdir("test_bucket_abort_timeout") + clock = Clock() + bw = BucketWriter(self, incoming, final, 10, self.make_lease(), clock) + self.assertTrue(clock.getDelayedCalls()) + bw.abort() + self.assertFalse(clock.getDelayedCalls()) class RemoteBucket(object): From 1827faf36b60751811302d1d20ba87348b7e32c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 14:45:44 -0500 Subject: [PATCH 216/916] Fix issue with leaked-past-end-of-test DelayedCalls. --- src/allmydata/test/test_storage.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 18dca9856..977ed768f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -498,7 +498,9 @@ class Server(unittest.TestCase): basedir = os.path.join("storage", "Server", name) return basedir - def create(self, name, reserved_space=0, klass=StorageServer, clock=reactor): + def create(self, name, reserved_space=0, klass=StorageServer, clock=None): + if clock is None: + clock = Clock() workdir = self.workdir(name) ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space, stats_provider=FakeStatsProvider(), @@ -1091,8 +1093,10 @@ class MutableServer(unittest.TestCase): basedir = os.path.join("storage", "MutableServer", name) return basedir - def create(self, name, clock=reactor): + def create(self, name, clock=None): workdir = self.workdir(name) + if clock is None: + clock = Clock() ss = StorageServer(workdir, b"\x00" * 20, clock=clock) ss.setServiceParent(self.sparent) From 5d915afe1c00d5832fec59d9a1599482d66a9e85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:42:54 -0500 Subject: [PATCH 217/916] Clean up BucketWriters on shutdown (also preventing DelayedCalls leaks in tests). --- src/allmydata/storage/server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 080c1aea1..6e3d6f683 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -136,6 +136,12 @@ class StorageServer(service.MultiService, Referenceable): # Canaries and disconnect markers for BucketWriters created via Foolscap: self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,(IRemoteReference, object)] + def stopService(self): + # Cancel any in-progress uploads: + for bw in list(self._bucket_writers.values()): + bw.disconnected() + return service.MultiService.stopService(self) + def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) From bd645edd9e68efef5c21b5864957b6e2858acc12 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:44:51 -0500 Subject: [PATCH 218/916] Fix flake. --- src/allmydata/test/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 977ed768f..92de63f0d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -23,7 +23,7 @@ from uuid import uuid4 from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.task import Clock from hypothesis import given, strategies From e2636466b584fff59dcd513866e254443417e771 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:47:25 -0500 Subject: [PATCH 219/916] Fix a flake. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 6e3d6f683..7dc277e39 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -14,7 +14,7 @@ if PY2: else: from typing import Dict -import os, re, time +import os, re import six from foolscap.api import Referenceable From 4c111773876bb97192dc2604e6a32e14f7898c16 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:58:55 -0500 Subject: [PATCH 220/916] Fix a problem with typechecking. Using remote_write() isn't quite right given move to HTTP, but can fight that battle another day. --- src/allmydata/storage/immutable.py | 3 --- src/allmydata/test/test_storage.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 8a7519b7b..08b83cd87 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -252,9 +252,6 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 return self._max_size def remote_write(self, offset, data): - self.write(offset, data) - - def write(self, offset, data): # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 92de63f0d..7fbe8f87b 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -319,7 +319,7 @@ class Bucket(unittest.TestCase): # 20 minutes pass, getting close to the timeout... clock.advance(29 * 60) # .. but we receive a write! So that should delay the timeout. - bw.write(0, b"hello") + bw.remote_write(0, b"hello") self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_closing_cancels_timeout(self): From eeb1d90e7a83cb85e685c73bfd7a960140a16ff4 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 20 Nov 2021 18:26:02 +0100 Subject: [PATCH 221/916] Leveled headings and rst semantics for sidenotes Signed-off-by: fenn-cs --- docs/release-checklist.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3b313da61..796be75ba 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -17,7 +17,7 @@ This checklist is based on the original instructions (in old revisions in the fi Any Contributor ---------------- +=============== Anyone who can create normal PRs should be able to complete this portion of the release process. @@ -46,10 +46,10 @@ previously generated files. - Inside the tahoe root dir run ``git clone . ../tahoe-release-x.x.x`` where (x.x.x is the release number such as 1.16.0). -*The above command would create a new directory at the same level as your original clone named -``tahoe-release-x.x.x``. You could name the folder however you want but it would be a good -practice to give it the release name. You MAY also discard this directory once the release -process is complete.* +.. note:: + The above command would create a new directory at the same level as your original clone named ``tahoe-release-x.x.x``. You can name this folder however you want but it would be a good + practice to give it the release name. You MAY also discard this directory once the release + process is complete. Get into the release directory and install dependencies by running @@ -115,8 +115,9 @@ they will need to evaluate which contributors' signatures they trust. - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` -*Replace the key-id above with your own, which can simply be your email if's attached your fingerprint.* -*Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0`* +.. note:: + - Replace the key-id above with your own, which can simply be your email if's attached your fingerprint. + - Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0` - build all code locally @@ -147,7 +148,7 @@ they will need to evaluate which contributors' signatures they trust. Privileged Contributor ------------------------ +====================== Steps in this portion require special access to keys or infrastructure. For example, **access to tahoe-lafs.org** to upload From 04e45f065ab496acf8aadb9f4cca0f60f55c41a2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Nov 2021 07:56:51 -0500 Subject: [PATCH 222/916] document `compare_leases_without_timestamps` --- src/allmydata/test/test_storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 92176ce52..91c7adb7f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1361,6 +1361,10 @@ class MutableServer(unittest.TestCase): 2: [b"2"*10]}) def compare_leases_without_timestamps(self, leases_a, leases_b): + """ + Assert that, except for expiration times, ``leases_a`` contains the same + lease information as ``leases_b``. + """ for a, b in zip(leases_a, leases_b): # The leases aren't always of the same type (though of course # corresponding elements in the two lists should be of the same From b92343c664c15605df4a4244208d614f2b3390b2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Nov 2021 08:36:12 -0500 Subject: [PATCH 223/916] some more docstrings --- src/allmydata/storage/lease_schema.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py index 697ac9e34..c09a9279b 100644 --- a/src/allmydata/storage/lease_schema.py +++ b/src/allmydata/storage/lease_schema.py @@ -28,11 +28,17 @@ from .lease import ( @attr.s(frozen=True) class CleartextLeaseSerializer(object): + """ + Serialize and unserialize leases with cleartext secrets. + """ _to_data = attr.ib() _from_data = attr.ib() def serialize(self, lease): # type: (LeaseInfo) -> bytes + """ + Represent the given lease as bytes with cleartext secrets. + """ if isinstance(lease, LeaseInfo): return self._to_data(lease) raise ValueError( @@ -42,6 +48,9 @@ class CleartextLeaseSerializer(object): ) def unserialize(self, data): + """ + Load a lease with cleartext secrets from the given bytes representation. + """ # type: (bytes) -> LeaseInfo # In v1 of the immutable schema lease secrets are stored plaintext. # So load the data into a plain LeaseInfo which works on plaintext From d1839187f148f0e8f265123149c5fc2bc3c9d143 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Nov 2021 08:45:10 -0500 Subject: [PATCH 224/916] "misplaced type annotation" --- src/allmydata/storage/lease_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py index c09a9279b..7e604388e 100644 --- a/src/allmydata/storage/lease_schema.py +++ b/src/allmydata/storage/lease_schema.py @@ -48,10 +48,10 @@ class CleartextLeaseSerializer(object): ) def unserialize(self, data): + # type: (bytes) -> LeaseInfo """ Load a lease with cleartext secrets from the given bytes representation. """ - # type: (bytes) -> LeaseInfo # In v1 of the immutable schema lease secrets are stored plaintext. # So load the data into a plain LeaseInfo which works on plaintext # secrets. From c341a86abdd4aa2b4244d0adeff53d5893be9a03 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:01:03 -0500 Subject: [PATCH 225/916] Correct the comment. --- src/allmydata/test/test_storage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 7fbe8f87b..bc87e168d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -316,9 +316,10 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir("test_bucket_writes_delay_timeout") clock = Clock() bw = BucketWriter(self, incoming, final, 200, self.make_lease(), clock) - # 20 minutes pass, getting close to the timeout... + # 29 minutes pass, getting close to the timeout... clock.advance(29 * 60) - # .. but we receive a write! So that should delay the timeout. + # .. but we receive a write! So that should delay the timeout again to + # another 30 minutes. bw.remote_write(0, b"hello") self._assert_timeout_only_after_30_minutes(clock, bw) From 6c514dfda57bfc2ede45719db24acecfbfee3ed1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:33:45 -0500 Subject: [PATCH 226/916] Add klein. --- nix/klein.nix | 18 ++++++++++++++++++ nix/overlays.nix | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 nix/klein.nix diff --git a/nix/klein.nix b/nix/klein.nix new file mode 100644 index 000000000..aa109e3d1 --- /dev/null +++ b/nix/klein.nix @@ -0,0 +1,18 @@ +{ lib, buildPythonPackage, fetchPypi }: +buildPythonPackage rec { + pname = "klein"; + version = "21.8.0"; + + src = fetchPypi { + sha256 = "09i1x5ppan3kqsgclbz8xdnlvzvp3amijbmdzv0kik8p5l5zswxa"; + inherit pname version; + }; + + doCheck = false; + + meta = with lib; { + homepage = https://github.com/twisted/klein; + description = "Nicer web server for Twisted"; + license = licenses.mit; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index fbd0ce3bb..011d8dd6b 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -28,6 +28,9 @@ self: super: { packageOverrides = python-self: python-super: { # collections-extended is not part of nixpkgs at this time. collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; + + # klein is not in nixpkgs 21.05, at least: + klein = python-super.pythonPackages.callPackage ./klein.nix { }; }; }; } From c921b153f4990e98a32dde1286d3a9c11d5fd2e4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:39:15 -0500 Subject: [PATCH 227/916] A better name for the API. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3baa336fa..327892ecd 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -47,7 +47,7 @@ def _authorization_decorator(f): return route -def _route(app, *route_args, **route_kwargs): +def _authorized_route(app, *route_args, **route_kwargs): """ Like Klein's @route, but with additional support for checking the ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The @@ -89,6 +89,6 @@ class HTTPServer(object): # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) - @_route(_app, "/v1/version", methods=["GET"]) + @_authorized_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): return self._cbor(request, self._storage_server.remote_get_version()) From a593095dc935b6719e266e0bf3a996b39047d9c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:39:53 -0500 Subject: [PATCH 228/916] Explain why it's a conditional import. --- src/allmydata/storage/http_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e1743343d..f8a7590aa 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -14,6 +14,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on else: + # typing module not available in Python 2, and we only do type checking in + # Python 3 anyway. from typing import Union from treq.testing import StubTreq From 8abc1ad8f4e43a244d0bcead201a133f3cf8b0c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:44:45 -0500 Subject: [PATCH 229/916] cbor2 for Python 2 on Nix. --- nix/cbor2.nix | 18 ++++++++++++++++++ nix/overlays.nix | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 nix/cbor2.nix diff --git a/nix/cbor2.nix b/nix/cbor2.nix new file mode 100644 index 000000000..02c810e1e --- /dev/null +++ b/nix/cbor2.nix @@ -0,0 +1,18 @@ +{ lib, buildPythonPackage, fetchPypi }: +buildPythonPackage rec { + pname = "cbor2"; + version = "5.2.0"; + + src = fetchPypi { + sha256 = "1mmmncfbsx7cbdalcrsagp9hx7wqfawaz9361gjkmsk3lp6chd5w"; + inherit pname version; + }; + + doCheck = false; + + meta = with lib; { + homepage = https://github.com/agronholm/cbor2; + description = "CBOR encoder/decoder"; + license = licenses.mit; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index 011d8dd6b..5cfab200c 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -21,6 +21,9 @@ self: super: { # collections-extended is not part of nixpkgs at this time. collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; + + # cbor2 is not part of nixpkgs at this time. + cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { }; }; }; From 30511ea8502dc04848f9ed3715b5517c51444c96 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 11:39:51 -0500 Subject: [PATCH 230/916] Add more build inputs. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index df12f21d4..59864d36d 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -98,6 +98,7 @@ EOF service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended + klein cbor2 treq ]; checkInputs = with python.pkgs; [ From 5855a30e34e1f18d44cbd898dee1b128be2cd976 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Nov 2021 14:01:43 -0700 Subject: [PATCH 231/916] add docstrings --- src/allmydata/storage/crawler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 129659d27..dcbea909a 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -146,10 +146,17 @@ class _LeaseStateSerializer(object): ) def load(self): + """ + :returns: deserialized JSON state + """ with self._path.open("rb") as f: return json.load(f) def save(self, data): + """ + Serialize the given data as JSON into the state-path + :returns: None + """ tmpfile = self._path.siblingExtension(".tmp") with tmpfile.open("wb") as f: json.dump(data, f) From 54c032d0d7f467ff3861e5dcefa4c0024b415b34 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 27 Nov 2021 00:59:13 +0100 Subject: [PATCH 232/916] change assertTrue -> assertEquals for non bools Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_problems.py | 8 ++++---- src/allmydata/test/mutable/test_repair.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 9abee560d..40105142a 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -241,7 +241,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): # that ought to work def _got_node(n): d = n.download_best_version() - d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 1")) # now break the second peer def _break_peer1(res): self.g.break_server(self.server1.get_serverid()) @@ -249,7 +249,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 2")) def _explain_error(f): print(f) if f.check(NotEnoughServersError): @@ -281,7 +281,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d = nm.create_mutable_file(MutableData(b"contents 1")) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 1")) # now break one of the remaining servers def _break_second_server(res): self.g.break_server(peerids[1]) @@ -289,7 +289,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 2")) return d d.addCallback(_created) return d diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index 987b21cc3..deddb8d92 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -229,7 +229,7 @@ class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin): new_versionid = smap.best_recoverable_version() self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.assertTrue, expected_contents) + d2.addCallback(self.assertEquals, expected_contents) return d2 d.addCallback(_check_smap) return d From c02d8cab3ab7ebee5cc57a07dde5d4b52393eb03 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 08:56:05 -0500 Subject: [PATCH 233/916] change one more assertTrue to assertEquals --- src/allmydata/test/mutable/test_problems.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 40105142a..4bcb8161b 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -420,7 +420,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): return self._node.download_version(servermap, ver) d.addCallback(_then) d.addCallback(lambda data: - self.assertTrue(data, CONTENTS)) + self.assertEquals(data, CONTENTS)) return d def test_1654(self): From 5fef83078d863843ef1f5f8a35990bfc3fcdb338 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:08:11 -0500 Subject: [PATCH 234/916] news fragment --- newsfragments/3847.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3847.minor diff --git a/newsfragments/3847.minor b/newsfragments/3847.minor new file mode 100644 index 000000000..e69de29bb From 66a0c6f3f43ecd018cdbb571ebd6eab740b6cca7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:43:06 -0500 Subject: [PATCH 235/916] add a direct test for the non-utf-8 bytestring behavior --- src/allmydata/test/test_eliotutil.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 3f915ecd2..00110530c 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -78,6 +78,9 @@ from .common import ( class EliotLoggedTestTests(AsyncTestCase): + """ + Tests for the automatic log-related provided by ``EliotLoggedRunTest``. + """ def test_returns_none(self): Message.log(hello="world") @@ -95,6 +98,12 @@ class EliotLoggedTestTests(AsyncTestCase): # We didn't start an action. We're not finishing an action. return d.result + def test_logs_non_utf_8_byte(self): + """ + If an Eliot message is emitted that contains a non-UTF-8 byte string then + the test nevertheless passes. + """ + Message.log(hello=b"\xFF") class ParseDestinationDescriptionTests(SyncTestCase): From f40da7dc27d089e3bdfdca48e51aec25aea282c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:23:59 -0500 Subject: [PATCH 236/916] Put the choice of JSON encoder for Eliot into its own module and use it in a few places --- src/allmydata/test/__init__.py | 4 ++-- src/allmydata/test/test_eliotutil.py | 4 ++-- src/allmydata/util/_eliot_updates.py | 28 ++++++++++++++++++++++++++++ src/allmydata/util/eliotutil.py | 7 ++++--- 4 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/allmydata/util/_eliot_updates.py diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 893aa15ce..ad245ca77 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -125,5 +125,5 @@ if sys.platform == "win32": initialize() from eliot import to_file -from allmydata.util.jsonbytes import AnyBytesJSONEncoder -to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder) +from allmydata.util.eliotutil import eliot_json_encoder +to_file(open("eliot.log", "wb"), encoder=eliot_json_encoder) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 00110530c..0be02b277 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -65,11 +65,11 @@ from twisted.internet.task import deferLater from twisted.internet import reactor from ..util.eliotutil import ( + eliot_json_encoder, log_call_deferred, _parse_destination_description, _EliotLogging, ) -from ..util.jsonbytes import AnyBytesJSONEncoder from .common import ( SyncTestCase, @@ -118,7 +118,7 @@ class ParseDestinationDescriptionTests(SyncTestCase): reactor = object() self.assertThat( _parse_destination_description("file:-")(reactor), - Equals(FileDestination(stdout, encoder=AnyBytesJSONEncoder)), + Equals(FileDestination(stdout, encoder=eliot_json_encoder)), ) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py new file mode 100644 index 000000000..4300f2be8 --- /dev/null +++ b/src/allmydata/util/_eliot_updates.py @@ -0,0 +1,28 @@ +""" +Bring in some Eliot updates from newer versions of Eliot than we can +depend on in Python 2. + +Every API in this module (except ``eliot_json_encoder``) should be obsolete as +soon as we depend on Eliot 1.14 or newer. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from .jsonbytes import AnyBytesJSONEncoder + +# There are currently a number of log messages that include non-UTF-8 bytes. +# Allow these, at least for now. Later when the whole test suite has been +# converted to our SyncTestCase or AsyncTestCase it will be easier to turn +# this off and then attribute log failures to specific codepaths so they can +# be fixed (and then not regressed later) because those instances will result +# in test failures instead of only garbage being written to the eliot log. +eliot_json_encoder = AnyBytesJSONEncoder diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 4e48fbb9f..ff858531d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,8 +87,9 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import AnyBytesJSONEncoder - +from ._eliot_updates import ( + eliot_json_encoder, +) def validateInstanceOf(t): """ @@ -306,7 +307,7 @@ class _DestinationParser(object): rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) - return lambda reactor: FileDestination(get_file(), AnyBytesJSONEncoder) + return lambda reactor: FileDestination(get_file(), eliot_json_encoder) _parse_destination_description = _DestinationParser().parse From 3eb1a5e7cb6bc227feda6f254b43e35b1807446d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:25:03 -0500 Subject: [PATCH 237/916] Add a MemoryLogger that prefers our encoder and use it instead of Eliot's --- src/allmydata/test/eliotutil.py | 16 ++----- src/allmydata/util/_eliot_updates.py | 62 +++++++++++++++++++++++++++- src/allmydata/util/eliotutil.py | 2 + 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 1685744fd..dd21f1e9d 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -42,7 +42,6 @@ from zope.interface import ( from eliot import ( ActionType, Field, - MemoryLogger, ILogger, ) from eliot.testing import ( @@ -54,8 +53,9 @@ from twisted.python.monkey import ( MonkeyPatcher, ) -from ..util.jsonbytes import AnyBytesJSONEncoder - +from ..util.eliotutil import ( + MemoryLogger, +) _NAME = Field.for_types( u"name", @@ -71,14 +71,6 @@ RUN_TEST = ActionType( ) -# On Python 3, we want to use our custom JSON encoder when validating messages -# can be encoded to JSON: -if PY2: - _memory_logger = MemoryLogger -else: - _memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder) - - @attr.s class EliotLoggedRunTest(object): """ @@ -170,7 +162,7 @@ def with_logging( """ @wraps(test_method) def run_with_logging(*args, **kwargs): - validating_logger = _memory_logger() + validating_logger = MemoryLogger() original = swap_logger(None) try: swap_logger(_TwoLoggers(original, validating_logger)) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py index 4300f2be8..4ff0caf4d 100644 --- a/src/allmydata/util/_eliot_updates.py +++ b/src/allmydata/util/_eliot_updates.py @@ -1,6 +1,7 @@ """ Bring in some Eliot updates from newer versions of Eliot than we can -depend on in Python 2. +depend on in Python 2. The implementations are copied from Eliot 1.14 and +only changed enough to add Python 2 compatibility. Every API in this module (except ``eliot_json_encoder``) should be obsolete as soon as we depend on Eliot 1.14 or newer. @@ -17,6 +18,13 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import json as pyjson +from functools import partial + +from eliot import ( + MemoryLogger as _MemoryLogger, +) + from .jsonbytes import AnyBytesJSONEncoder # There are currently a number of log messages that include non-UTF-8 bytes. @@ -26,3 +34,55 @@ from .jsonbytes import AnyBytesJSONEncoder # be fixed (and then not regressed later) because those instances will result # in test failures instead of only garbage being written to the eliot log. eliot_json_encoder = AnyBytesJSONEncoder + +class _CustomEncoderMemoryLogger(_MemoryLogger): + """ + Override message validation from the Eliot-supplied ``MemoryLogger`` to + use our chosen JSON encoder. + + This is only necessary on Python 2 where we use an old version of Eliot + that does not parameterize the encoder. + """ + def __init__(self, encoder=eliot_json_encoder): + """ + @param encoder: A JSONEncoder subclass to use when encoding JSON. + """ + self._encoder = encoder + super(_CustomEncoderMemoryLogger, self).__init__() + + def _validate_message(self, dictionary, serializer): + """Validate an individual message. + + As a side-effect, the message is replaced with its serialized contents. + + @param dictionary: A message C{dict} to be validated. Might be mutated + by the serializer! + + @param serializer: C{None} or a serializer. + + @raises TypeError: If a field name is not unicode, or the dictionary + fails to serialize to JSON. + + @raises eliot.ValidationError: If serializer was given and validation + failed. + """ + if serializer is not None: + serializer.validate(dictionary) + for key in dictionary: + if not isinstance(key, str): + if isinstance(key, bytes): + key.decode("utf-8") + else: + raise TypeError(dictionary, "%r is not unicode" % (key,)) + if serializer is not None: + serializer.serialize(dictionary) + + try: + pyjson.dumps(dictionary, cls=self._encoder) + except Exception as e: + raise TypeError("Message %s doesn't encode to JSON: %s" % (dictionary, e)) + +if PY2: + MemoryLogger = partial(_CustomEncoderMemoryLogger, encoder=eliot_json_encoder) +else: + MemoryLogger = partial(_MemoryLogger, encoder=eliot_json_encoder) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ff858531d..5067876c5 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -16,6 +16,7 @@ from __future__ import ( ) __all__ = [ + "MemoryLogger", "inline_callbacks", "eliot_logging_service", "opt_eliot_destination", @@ -88,6 +89,7 @@ from twisted.internet.defer import ( from twisted.application.service import Service from ._eliot_updates import ( + MemoryLogger, eliot_json_encoder, ) From 20e0626e424276c83cd1f2eb42fdddb7cf56072e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:27:17 -0500 Subject: [PATCH 238/916] add capture_logging that parameterizes JSON encoder --- src/allmydata/util/_eliot_updates.py | 100 ++++++++++++++++++++++++++- src/allmydata/util/eliotutil.py | 13 +--- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py index 4ff0caf4d..8e3beca45 100644 --- a/src/allmydata/util/_eliot_updates.py +++ b/src/allmydata/util/_eliot_updates.py @@ -19,12 +19,17 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import json as pyjson -from functools import partial +from functools import wraps, partial from eliot import ( MemoryLogger as _MemoryLogger, ) +from eliot.testing import ( + check_for_errors, + swap_logger, +) + from .jsonbytes import AnyBytesJSONEncoder # There are currently a number of log messages that include non-UTF-8 bytes. @@ -86,3 +91,96 @@ if PY2: MemoryLogger = partial(_CustomEncoderMemoryLogger, encoder=eliot_json_encoder) else: MemoryLogger = partial(_MemoryLogger, encoder=eliot_json_encoder) + +def validateLogging( + assertion, *assertionArgs, **assertionKwargs +): + """ + Decorator factory for L{unittest.TestCase} methods to add logging + validation. + + 1. The decorated test method gets a C{logger} keyword argument, a + L{MemoryLogger}. + 2. All messages logged to this logger will be validated at the end of + the test. + 3. Any unflushed logged tracebacks will cause the test to fail. + + For example: + + from unittest import TestCase + from eliot.testing import assertContainsFields, validateLogging + + class MyTests(TestCase): + def assertFooLogging(self, logger): + assertContainsFields(self, logger.messages[0], {"key": 123}) + + + @param assertion: A callable that will be called with the + L{unittest.TestCase} instance, the logger and C{assertionArgs} and + C{assertionKwargs} once the actual test has run, allowing for extra + logging-related assertions on the effects of the test. Use L{None} if you + want the cleanup assertions registered but no custom assertions. + + @param assertionArgs: Additional positional arguments to pass to + C{assertion}. + + @param assertionKwargs: Additional keyword arguments to pass to + C{assertion}. + + @param encoder_: C{json.JSONEncoder} subclass to use when validating JSON. + """ + encoder_ = assertionKwargs.pop("encoder_", eliot_json_encoder) + def decorator(function): + @wraps(function) + def wrapper(self, *args, **kwargs): + skipped = False + + kwargs["logger"] = logger = MemoryLogger(encoder=encoder_) + self.addCleanup(check_for_errors, logger) + # TestCase runs cleanups in reverse order, and we want this to + # run *before* tracebacks are checked: + if assertion is not None: + self.addCleanup( + lambda: skipped + or assertion(self, logger, *assertionArgs, **assertionKwargs) + ) + try: + return function(self, *args, **kwargs) + except self.skipException: + skipped = True + raise + + return wrapper + + return decorator + +# PEP 8 variant: +validate_logging = validateLogging + +def capture_logging( + assertion, *assertionArgs, **assertionKwargs +): + """ + Capture and validate all logging that doesn't specify a L{Logger}. + + See L{validate_logging} for details on the rest of its behavior. + """ + encoder_ = assertionKwargs.pop("encoder_", eliot_json_encoder) + def decorator(function): + @validate_logging( + assertion, *assertionArgs, encoder_=encoder_, **assertionKwargs + ) + @wraps(function) + def wrapper(self, *args, **kwargs): + logger = kwargs["logger"] + previous_logger = swap_logger(logger) + + def cleanup(): + swap_logger(previous_logger) + + self.addCleanup(cleanup) + return function(self, *args, **kwargs) + + return wrapper + + return decorator diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 5067876c5..789ef38ff 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -23,6 +23,7 @@ __all__ = [ "opt_help_eliot_destinations", "validateInstanceOf", "validateSetMembership", + "capture_logging", ] from future.utils import PY2 @@ -33,7 +34,7 @@ from six import ensure_text from sys import ( stdout, ) -from functools import wraps, partial +from functools import wraps from logging import ( INFO, Handler, @@ -67,8 +68,6 @@ from eliot.twisted import ( DeferredContext, inline_callbacks, ) -from eliot.testing import capture_logging as eliot_capture_logging - from twisted.python.usage import ( UsageError, ) @@ -91,6 +90,7 @@ from twisted.application.service import Service from ._eliot_updates import ( MemoryLogger, eliot_json_encoder, + capture_logging, ) def validateInstanceOf(t): @@ -330,10 +330,3 @@ def log_call_deferred(action_type): return DeferredContext(d).addActionFinish() return logged_f return decorate_log_call_deferred - -# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON -# encoder we want to use when validating messages. -if PY2: - capture_logging = eliot_capture_logging -else: - capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) From 7626a02bdb84013e4f45bad81c8a3f5ba4586401 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:27:28 -0500 Subject: [PATCH 239/916] remove redundant assertion --- src/allmydata/test/test_util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a03845ed6..9a0af1e06 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -553,11 +553,6 @@ class JSONBytes(unittest.TestCase): o, cls=jsonbytes.AnyBytesJSONEncoder)), expected, ) - self.assertEqual( - json.loads(jsonbytes.dumps(o, any_bytes=True)), - expected - ) - class FakeGetVersion(object): From b01478659ea9868164aa9f7b7368f295f2d47921 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:18:18 -0500 Subject: [PATCH 240/916] Apparently I generated wrong hashes. --- nix/cbor2.nix | 2 +- nix/klein.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 02c810e1e..1bd9920e6 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -4,7 +4,7 @@ buildPythonPackage rec { version = "5.2.0"; src = fetchPypi { - sha256 = "1mmmncfbsx7cbdalcrsagp9hx7wqfawaz9361gjkmsk3lp6chd5w"; + sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3"; inherit pname version; }; diff --git a/nix/klein.nix b/nix/klein.nix index aa109e3d1..0bb025cf8 100644 --- a/nix/klein.nix +++ b/nix/klein.nix @@ -4,7 +4,7 @@ buildPythonPackage rec { version = "21.8.0"; src = fetchPypi { - sha256 = "09i1x5ppan3kqsgclbz8xdnlvzvp3amijbmdzv0kik8p5l5zswxa"; + sha256 = "1mpydmz90d0n9dwa7mr6pgj5v0kczfs05ykssrasdq368dssw7ch"; inherit pname version; }; From 1fc77504aeec738acac315d65b68b4a7e01db095 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:39:42 -0500 Subject: [PATCH 241/916] List dependencies. --- nix/klein.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/klein.nix b/nix/klein.nix index 0bb025cf8..196f95e88 100644 --- a/nix/klein.nix +++ b/nix/klein.nix @@ -10,6 +10,8 @@ buildPythonPackage rec { doCheck = false; + propagatedBuildInputs = [ attrs hyperlink incremental Tubes Twisted typing_extensions Werkzeug zope.interface ]; + meta = with lib; { homepage = https://github.com/twisted/klein; description = "Nicer web server for Twisted"; From c65a13e63228fada255a94b99ca4e61a1e9e58dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:47:28 -0500 Subject: [PATCH 242/916] Rip out klein, maybe not necessary. --- nix/klein.nix | 20 -------------------- nix/overlays.nix | 3 --- 2 files changed, 23 deletions(-) delete mode 100644 nix/klein.nix diff --git a/nix/klein.nix b/nix/klein.nix deleted file mode 100644 index 196f95e88..000000000 --- a/nix/klein.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi }: -buildPythonPackage rec { - pname = "klein"; - version = "21.8.0"; - - src = fetchPypi { - sha256 = "1mpydmz90d0n9dwa7mr6pgj5v0kczfs05ykssrasdq368dssw7ch"; - inherit pname version; - }; - - doCheck = false; - - propagatedBuildInputs = [ attrs hyperlink incremental Tubes Twisted typing_extensions Werkzeug zope.interface ]; - - meta = with lib; { - homepage = https://github.com/twisted/klein; - description = "Nicer web server for Twisted"; - license = licenses.mit; - }; -} diff --git a/nix/overlays.nix b/nix/overlays.nix index 5cfab200c..92f36e93e 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -31,9 +31,6 @@ self: super: { packageOverrides = python-self: python-super: { # collections-extended is not part of nixpkgs at this time. collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - - # klein is not in nixpkgs 21.05, at least: - klein = python-super.pythonPackages.callPackage ./klein.nix { }; }; }; } From 2f4d1079aa3b0621e4ad5991f810a5baf32c23db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:51:36 -0500 Subject: [PATCH 243/916] Needs setuptools_scm --- nix/cbor2.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 1bd9920e6..4d9734a8b 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi }: +{ lib, buildPythonPackage, fetchPypi , setuptools_scm }: buildPythonPackage rec { pname = "cbor2"; version = "5.2.0"; @@ -10,6 +10,8 @@ buildPythonPackage rec { doCheck = false; + nativeBuildInputs = [ setuptools_scm ]; + meta = with lib; { homepage = https://github.com/agronholm/cbor2; description = "CBOR encoder/decoder"; From 136bf95bdfcf0819285f7c4ed937f4de64a99125 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:58:02 -0500 Subject: [PATCH 244/916] Simpler way. --- nix/cbor2.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 4d9734a8b..ace5e13c6 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi , setuptools_scm }: +{ lib, buildPythonPackage, fetchPypi }: buildPythonPackage rec { pname = "cbor2"; version = "5.2.0"; @@ -10,7 +10,7 @@ buildPythonPackage rec { doCheck = false; - nativeBuildInputs = [ setuptools_scm ]; + buildInputs = [ setuptools_scm ]; meta = with lib; { homepage = https://github.com/agronholm/cbor2; From f2b52f368d63059ebe559109b4dbe8c4720bdd2f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:58:22 -0500 Subject: [PATCH 245/916] Another way. --- nix/cbor2.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index ace5e13c6..0544b1eb1 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -10,7 +10,7 @@ buildPythonPackage rec { doCheck = false; - buildInputs = [ setuptools_scm ]; + propagatedBuildInputs = [ setuptools_scm ]; meta = with lib; { homepage = https://github.com/agronholm/cbor2; From 49f24893219482a53a882c8139e3c46ffedcd48e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 15:59:27 -0700 Subject: [PATCH 246/916] explicit 'migrate pickle files' command --- src/allmydata/scripts/admin.py | 55 +++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index a9feed0dd..c125bc9e6 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -18,7 +18,17 @@ except ImportError: pass from twisted.python import usage -from allmydata.scripts.common import BaseOptions +from twisted.python.filepath import ( + FilePath, +) +from allmydata.scripts.common import ( + BaseOptions, + BasedirOptions, +) +from allmydata.storage import ( + crawler, + expirer, +) class GenerateKeypairOptions(BaseOptions): @@ -65,12 +75,54 @@ def derive_pubkey(options): print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 +class MigrateCrawlerOptions(BasedirOptions): + + def getSynopsis(self): + return "Usage: tahoe [global-options] admin migrate-crawler" + + def getUsage(self, width=None): + t = BasedirOptions.getUsage(self, width) + t += ( + "The crawler data is now stored as JSON to avoid" + " potential security issues with pickle files.\n\nIf" + " you are confident the state files in the 'storage/'" + " subdirectory of your node are trustworthy, run this" + " command to upgrade them to JSON.\n\nThe files are:" + " lease_checker.history, lease_checker.state, and" + " bucket_counter.state" + ) + return t + +def migrate_crawler(options): + out = options.stdout + storage = FilePath(options['basedir']).child("storage") + + conversions = [ + (storage.child("lease_checker.state"), crawler._convert_pickle_state_to_json), + (storage.child("bucket_counter.state"), crawler._convert_pickle_state_to_json), + (storage.child("lease_checker.history"), expirer._convert_pickle_state_to_json), + ] + + for fp, converter in conversions: + existed = fp.exists() + newfp = crawler._maybe_upgrade_pickle_to_json(fp, converter) + if existed: + print("Converted '{}' to '{}'".format(fp.path, newfp.path)) + else: + if newfp.exists(): + print("Already converted: '{}'".format(newfp.path)) + else: + print("Not found: '{}'".format(fp.path)) + + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, "Generate a public/private keypair, write to stdout."), ("derive-pubkey", None, DerivePubkeyOptions, "Derive a public key from a private key."), + ("migrate-crawler", None, MigrateCrawlerOptions, + "Write the crawler-history data as JSON."), ] def postOptions(self): if not hasattr(self, 'subOptions'): @@ -88,6 +140,7 @@ each subcommand. subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, + "migrate-crawler": migrate_crawler, } def do_admin(options): From ce25795e4e86b778ad6cc739fdc83d42d101101e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 16:00:19 -0700 Subject: [PATCH 247/916] new news --- newsfragments/3825.security | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/newsfragments/3825.security b/newsfragments/3825.security index b16418d2b..df83821de 100644 --- a/newsfragments/3825.security +++ b/newsfragments/3825.security @@ -1,5 +1,6 @@ The lease-checker now uses JSON instead of pickle to serialize its state. -Once you have run this version the lease state files will be stored in JSON -and an older version of the software won't load them (it simply won't notice -them so it will appear to have never run). +tahoe will now refuse to run until you either delete all pickle files or +migrate them using the new command: + + tahoe admin migrate-crawler From 3fd1ca8acbc3d046c97e3c520fdc6fab5d67541d Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 16:00:35 -0700 Subject: [PATCH 248/916] it's an error to have pickle-format files --- src/allmydata/scripts/tahoe_run.py | 16 +++++++++++++++- src/allmydata/storage/crawler.py | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 01f1a354c..51be32ee3 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -27,7 +27,9 @@ from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin - +from allmydata.storage.crawler import ( + MigratePickleFileError, +) from allmydata.node import ( PortAssignmentRequired, PrivacyError, @@ -164,6 +166,18 @@ class DaemonizeTheRealService(Service, HookMixin): self.stderr.write("\ntub.port cannot be 0: you must choose.\n\n") elif reason.check(PrivacyError): self.stderr.write("\n{}\n\n".format(reason.value)) + elif reason.check(MigratePickleFileError): + self.stderr.write( + "Error\nAt least one 'pickle' format file exists.\n" + "The file is {}\n" + "You must either delete the pickle-format files" + " or migrate them using the command:\n" + " tahoe admin migrate-crawler --basedir {}\n\n" + .format( + reason.value.args[0].path, + self.basedir, + ) + ) else: self.stderr.write("\nUnknown error\n") reason.printTraceback(self.stderr) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index dcbea909a..2b8cde230 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -27,6 +27,14 @@ class TimeSliceExceeded(Exception): pass +class MigratePickleFileError(Exception): + """ + A pickle-format file exists (the FilePath to the file will be the + single arg). + """ + pass + + def _convert_cycle_data(state): """ :param dict state: cycle-to-date or history-item state From 1b8ae8039e79bf288916edb9f7949f76f943aef4 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 16:01:15 -0700 Subject: [PATCH 249/916] no auto-migrate; produce error if pickle-files exist --- src/allmydata/storage/crawler.py | 31 +++++++++++++++++++++---------- src/allmydata/storage/expirer.py | 14 ++++++-------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 2b8cde230..f63754e10 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -108,7 +108,7 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): :param Callable[dict] convert_pickle: function to change pickle-style state into JSON-style state - :returns unicode: the local path where the state is stored + :returns FilePath: the local path where the state is stored If this state path is JSON, simply return it. @@ -116,14 +116,14 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): JSON path. """ if state_path.path.endswith(".json"): - return state_path.path + return state_path json_state_path = state_path.siblingExtension(".json") # if there's no file there at all, we're done because there's # nothing to upgrade if not state_path.exists(): - return json_state_path.path + return json_state_path # upgrade the pickle data to JSON import pickle @@ -135,7 +135,23 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): # we've written the JSON, delete the pickle state_path.remove() - return json_state_path.path + return json_state_path + + +def _confirm_json_format(fp): + """ + :param FilePath fp: the original (pickle) name of a state file + + This confirms that we do _not_ have the pickle-version of a + state-file and _do_ either have nothing, or the JSON version. If + the pickle-version exists, an exception is raised. + + :returns FilePath: the JSON name of a state file + """ + jsonfp = fp.siblingExtension(".json") + if fp.exists(): + raise MigratePickleFileError(fp) + return jsonfp class _LeaseStateSerializer(object): @@ -146,12 +162,7 @@ class _LeaseStateSerializer(object): """ def __init__(self, state_path): - self._path = FilePath( - _maybe_upgrade_pickle_to_json( - FilePath(state_path), - _convert_pickle_state_to_json, - ) - ) + self._path = _confirm_json_format(FilePath(state_path)) def load(self): """ diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index ad1343ef5..cd0a9369a 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -12,6 +12,8 @@ import os import struct from allmydata.storage.crawler import ( ShareCrawler, + MigratePickleFileError, + _confirm_json_format, _maybe_upgrade_pickle_to_json, _convert_cycle_data, ) @@ -40,17 +42,13 @@ def _convert_pickle_state_to_json(state): class _HistorySerializer(object): """ Serialize the 'history' file of the lease-crawler state. This is - "storage/history.state" for the pickle or - "storage/history.state.json" for the new JSON format. + "storage/lease_checker.history" for the pickle or + "storage/lease_checker.history.json" for the new JSON format. """ def __init__(self, history_path): - self._path = FilePath( - _maybe_upgrade_pickle_to_json( - FilePath(history_path), - _convert_pickle_state_to_json, - ) - ) + self._path = _confirm_json_format(FilePath(history_path)) + if not self._path.exists(): with self._path.open("wb") as f: json.dump({}, f) From 0a4bc385c5ac7c5e9410e83703250b425608be57 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 18:00:58 -0700 Subject: [PATCH 250/916] fix tests to use migrate command --- src/allmydata/scripts/admin.py | 7 ++-- src/allmydata/storage/crawler.py | 2 ++ src/allmydata/test/test_storage_web.py | 45 +++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index c125bc9e6..a6e826174 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -93,6 +93,7 @@ class MigrateCrawlerOptions(BasedirOptions): ) return t + def migrate_crawler(options): out = options.stdout storage = FilePath(options['basedir']).child("storage") @@ -107,12 +108,12 @@ def migrate_crawler(options): existed = fp.exists() newfp = crawler._maybe_upgrade_pickle_to_json(fp, converter) if existed: - print("Converted '{}' to '{}'".format(fp.path, newfp.path)) + print("Converted '{}' to '{}'".format(fp.path, newfp.path), file=out) else: if newfp.exists(): - print("Already converted: '{}'".format(newfp.path)) + print("Already converted: '{}'".format(newfp.path), file=out) else: - print("Not found: '{}'".format(fp.path)) + print("Not found: '{}'".format(fp.path), file=out) class AdminCommand(BaseOptions): diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index f63754e10..a1f70f4e5 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -148,6 +148,8 @@ def _confirm_json_format(fp): :returns FilePath: the JSON name of a state file """ + if fp.path.endswith(".json"): + return fp jsonfp = fp.siblingExtension(".json") if fp.exists(): raise MigratePickleFileError(fp) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 269af2203..86c2382f0 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -19,6 +19,7 @@ import time import os.path import re import json +from six.moves import StringIO from twisted.trial import unittest @@ -45,6 +46,13 @@ from allmydata.web.storage import ( StorageStatusElement, remove_prefix ) +from allmydata.scripts.admin import ( + MigrateCrawlerOptions, + migrate_crawler, +) +from allmydata.scripts.runner import ( + Options, +) from .common_util import FakeCanary from .common_web import ( @@ -1152,15 +1160,29 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): """ # this file came from an "in the wild" tahoe version 1.16.0 original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state.txt") - test_pickle = FilePath("lease_checker.state") + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + test_pickle = storage.child("lease_checker.state") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) - serial = _LeaseStateSerializer(test_pickle.path) + # convert from pickle format to JSON + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + options.stdout = StringIO() + migrate_crawler(options) # the (existing) state file should have been upgraded to JSON - self.assertNot(test_pickle.exists()) + self.assertFalse(test_pickle.exists()) self.assertTrue(test_pickle.siblingExtension(".json").exists()) + serial = _LeaseStateSerializer(test_pickle.path) self.assertEqual( serial.load(), @@ -1340,10 +1362,25 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): """ # this file came from an "in the wild" tahoe version 1.16.0 original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history.txt") - test_pickle = FilePath("lease_checker.history") + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + test_pickle = storage.child("lease_checker.history") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) + # convert from pickle format to JSON + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + options.stdout = StringIO() + migrate_crawler(options) + serial = _HistorySerializer(test_pickle.path) self.maxDiff = None From fc9671a8122bd085fa1d4ea74e2d4850abdf529f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 18:25:32 -0700 Subject: [PATCH 251/916] simplify, flake9 --- src/allmydata/scripts/admin.py | 2 +- src/allmydata/storage/crawler.py | 7 +------ src/allmydata/storage/expirer.py | 2 -- src/allmydata/test/test_storage_web.py | 1 - 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index a6e826174..e0dcc8821 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -106,7 +106,7 @@ def migrate_crawler(options): for fp, converter in conversions: existed = fp.exists() - newfp = crawler._maybe_upgrade_pickle_to_json(fp, converter) + newfp = crawler._upgrade_pickle_to_json(fp, converter) if existed: print("Converted '{}' to '{}'".format(fp.path, newfp.path), file=out) else: diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index a1f70f4e5..dbf4b1300 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -101,7 +101,7 @@ def _convert_pickle_state_to_json(state): } -def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): +def _upgrade_pickle_to_json(state_path, convert_pickle): """ :param FilePath state_path: the filepath to ensure is json @@ -110,14 +110,9 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): :returns FilePath: the local path where the state is stored - If this state path is JSON, simply return it. - If this state is pickle, convert to the JSON format and return the JSON path. """ - if state_path.path.endswith(".json"): - return state_path - json_state_path = state_path.siblingExtension(".json") # if there's no file there at all, we're done because there's diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index cd0a9369a..abe3c37b6 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -12,9 +12,7 @@ import os import struct from allmydata.storage.crawler import ( ShareCrawler, - MigratePickleFileError, _confirm_json_format, - _maybe_upgrade_pickle_to_json, _convert_cycle_data, ) from allmydata.storage.shares import get_share_file diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 86c2382f0..490a3f775 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -47,7 +47,6 @@ from allmydata.web.storage import ( remove_prefix ) from allmydata.scripts.admin import ( - MigrateCrawlerOptions, migrate_crawler, ) from allmydata.scripts.runner import ( From 679c46451764aae1213239bb90dd25b36bba324e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 18:43:06 -0700 Subject: [PATCH 252/916] tests --- src/allmydata/test/cli/test_admin.py | 86 ++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 src/allmydata/test/cli/test_admin.py diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py new file mode 100644 index 000000000..bdfc0a46f --- /dev/null +++ b/src/allmydata/test/cli/test_admin.py @@ -0,0 +1,86 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six.moves import StringIO + +from testtools.matchers import ( + Contains, +) + +from twisted.trial import unittest +from twisted.python.filepath import FilePath + +from allmydata.scripts.admin import ( + migrate_crawler, +) +from allmydata.scripts.runner import ( + Options, +) +from ..common import ( + SyncTestCase, +) + +class AdminMigrateCrawler(SyncTestCase): + """ + Tests related to 'tahoe admin migrate-crawler' + """ + + def test_already(self): + """ + We've already migrated; don't do it again. + """ + + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + with storage.child("lease_checker.state.json").open("w") as f: + f.write(b"{}\n") + + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + options.stdout = StringIO() + migrate_crawler(options) + + self.assertThat( + options.stdout.getvalue(), + Contains("Already converted:"), + ) + + def test_usage(self): + """ + We've already migrated; don't do it again. + """ + + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + with storage.child("lease_checker.state.json").open("w") as f: + f.write(b"{}\n") + + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + self.assertThat( + str(options), + Contains("security issues with pickle") + ) From b47381401c589c056afe89744df5b3f01f2ae5ae Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 19:01:09 -0700 Subject: [PATCH 253/916] flake8 --- src/allmydata/test/cli/test_admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index bdfc0a46f..082904652 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -16,8 +16,9 @@ from testtools.matchers import ( Contains, ) -from twisted.trial import unittest -from twisted.python.filepath import FilePath +from twisted.python.filepath import ( + FilePath, +) from allmydata.scripts.admin import ( migrate_crawler, From 85fa8fe32e05c68da46f15fe68b69781d05384a7 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 23:00:59 -0700 Subject: [PATCH 254/916] py2/py3 glue code for json dumping --- src/allmydata/storage/crawler.py | 18 ++++++++++++++---- src/allmydata/storage/expirer.py | 7 +++---- src/allmydata/test/test_storage_web.py | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index dbf4b1300..7516bc4e9 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -125,8 +125,7 @@ def _upgrade_pickle_to_json(state_path, convert_pickle): with state_path.open("rb") as f: state = pickle.load(f) new_state = convert_pickle(state) - with json_state_path.open("wb") as f: - json.dump(new_state, f) + _dump_json_to_file(new_state, json_state_path) # we've written the JSON, delete the pickle state_path.remove() @@ -151,6 +150,18 @@ def _confirm_json_format(fp): return jsonfp +def _dump_json_to_file(js, afile): + """ + Dump the JSON object `js` to the FilePath `afile` + """ + with afile.open("wb") as f: + data = json.dumps(js) + if PY2: + f.write(data) + else: + f.write(data.encode("utf8")) + + class _LeaseStateSerializer(object): """ Read and write state for LeaseCheckingCrawler. This understands @@ -174,8 +185,7 @@ class _LeaseStateSerializer(object): :returns: None """ tmpfile = self._path.siblingExtension(".tmp") - with tmpfile.open("wb") as f: - json.dump(data, f) + _dump_json_to_file(data, tmpfile) fileutil.move_into_place(tmpfile.path, self._path.path) return None diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index abe3c37b6..55ab51843 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -14,6 +14,7 @@ from allmydata.storage.crawler import ( ShareCrawler, _confirm_json_format, _convert_cycle_data, + _dump_json_to_file, ) from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ @@ -48,8 +49,7 @@ class _HistorySerializer(object): self._path = _confirm_json_format(FilePath(history_path)) if not self._path.exists(): - with self._path.open("wb") as f: - json.dump({}, f) + _dump_json_to_file({}, self._path) def load(self): """ @@ -65,8 +65,7 @@ class _HistorySerializer(object): """ Serialize the existing data as JSON. """ - with self._path.open("wb") as f: - json.dump(new_history, f) + _dump_json_to_file(new_history, self._path) return None diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 490a3f775..dff3b36f5 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1163,7 +1163,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.state") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: local.write(remote.read()) # convert from pickle format to JSON @@ -1365,7 +1365,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.history") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: local.write(remote.read()) # convert from pickle format to JSON From d985d1062295e2f816214bcbedb6746838d7a67d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Dec 2021 09:24:03 -0500 Subject: [PATCH 255/916] Update nix/cbor2.nix Co-authored-by: Jean-Paul Calderone --- nix/cbor2.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 0544b1eb1..16ca8ff63 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi }: +{ lib, buildPythonPackage, fetchPypi, setuptools_scm }: buildPythonPackage rec { pname = "cbor2"; version = "5.2.0"; From 18a5966f1d27791d3129690926791a623957472c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Dec 2021 09:38:56 -0500 Subject: [PATCH 256/916] Don't bother running HTTP server tests on Python 2, since it's going away any day now. --- src/allmydata/test/test_storage_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 9ba8adf21..442e154a0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -14,6 +14,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +from unittest import SkipTest + from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks @@ -31,6 +33,8 @@ class HTTPTests(TestCase): """ def setUp(self): + if PY2: + raise SkipTest("Not going to bother supporting Python 2") self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) # TODO what should the swissnum _actually_ be? self._http_server = HTTPServer(self.storage_server, b"abcd") From 50e21a90347a8a4bcc88487245c93f0379811dde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Dec 2021 09:55:44 -0500 Subject: [PATCH 257/916] Split StorageServer into generic part and Foolscap part. --- src/allmydata/storage/server.py | 127 ++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 49 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 7dc277e39..9d3ac4012 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -12,7 +12,7 @@ if PY2: # strings. Omit bytes so we don't leak future's custom bytes. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 else: - from typing import Dict + from typing import Dict, Tuple import os, re import six @@ -56,12 +56,11 @@ NUM_RE=re.compile("^[0-9]+$") DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 -@implementer(RIStorageServer, IStatsProducer) -class StorageServer(service.MultiService, Referenceable): +@implementer(IStatsProducer) +class StorageServer(service.MultiService): """ - A filesystem-based implementation of ``RIStorageServer``. + Implement the business logic for the storage server. """ - name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler def __init__(self, storedir, nodeid, reserved_space=0, @@ -125,16 +124,8 @@ class StorageServer(service.MultiService, Referenceable): self.lease_checker.setServiceParent(self) self._clock = clock - # Currently being-written Bucketwriters. For Foolscap, lifetime is tied - # to connection: when disconnection happens, the BucketWriters are - # removed. For HTTP, this makes no sense, so there will be - # timeout-based cleanup; see - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3807. - # Map in-progress filesystem path -> BucketWriter: self._bucket_writers = {} # type: Dict[str,BucketWriter] - # Canaries and disconnect markers for BucketWriters created via Foolscap: - self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,(IRemoteReference, object)] def stopService(self): # Cancel any in-progress uploads: @@ -263,7 +254,7 @@ class StorageServer(service.MultiService, Referenceable): space += bw.allocated_size() return space - def remote_get_version(self): + def get_version(self): remaining_space = self.get_available_space() if remaining_space is None: # We're on a platform that has no API to get disk stats. @@ -284,7 +275,7 @@ class StorageServer(service.MultiService, Referenceable): } return version - def _allocate_buckets(self, storage_index, + def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, owner_num=0, renew_leases=True): @@ -371,21 +362,6 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("allocate", self._clock.seconds() - start) return alreadygot, bucketwriters - def remote_allocate_buckets(self, storage_index, - renew_secret, cancel_secret, - sharenums, allocated_size, - canary, owner_num=0): - """Foolscap-specific ``allocate_buckets()`` API.""" - alreadygot, bucketwriters = self._allocate_buckets( - storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - owner_num=owner_num, renew_leases=True, - ) - # Abort BucketWriters if disconnection happens. - for bw in bucketwriters.values(): - disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) - self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) - return alreadygot, bucketwriters - def _iter_share_files(self, storage_index): for shnum, filename in self._get_bucket_shares(storage_index): with open(filename, 'rb') as f: @@ -401,8 +377,7 @@ class StorageServer(service.MultiService, Referenceable): continue # non-sharefile yield sf - def remote_add_lease(self, storage_index, renew_secret, cancel_secret, - owner_num=1): + def add_lease(self, storage_index, renew_secret, cancel_secret, owner_num=1): start = self._clock.seconds() self.count("add-lease") new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME @@ -414,7 +389,7 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("add-lease", self._clock.seconds() - start) return None - def remote_renew_lease(self, storage_index, renew_secret): + def renew_lease(self, storage_index, renew_secret): start = self._clock.seconds() self.count("renew") new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME @@ -448,7 +423,7 @@ class StorageServer(service.MultiService, Referenceable): # Commonly caused by there being no buckets at all. pass - def remote_get_buckets(self, storage_index): + def get_buckets(self, storage_index): start = self._clock.seconds() self.count("get") si_s = si_b2a(storage_index) @@ -698,18 +673,6 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("writev", self._clock.seconds() - start) return (testv_is_good, read_data) - def remote_slot_testv_and_readv_and_writev(self, storage_index, - secrets, - test_and_write_vectors, - read_vector): - return self.slot_testv_and_readv_and_writev( - storage_index, - secrets, - test_and_write_vectors, - read_vector, - renew_leases=True, - ) - def _allocate_slot_share(self, bucketdir, secrets, sharenum, owner_num=0): (write_enabler, renew_secret, cancel_secret) = secrets @@ -720,7 +683,7 @@ class StorageServer(service.MultiService, Referenceable): self) return share - def remote_slot_readv(self, storage_index, shares, readv): + def slot_readv(self, storage_index, shares, readv): start = self._clock.seconds() self.count("readv") si_s = si_b2a(storage_index) @@ -747,8 +710,8 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("readv", self._clock.seconds() - start) return datavs - def remote_advise_corrupt_share(self, share_type, storage_index, shnum, - reason): + def advise_corrupt_share(self, share_type, storage_index, shnum, + reason): # This is a remote API, I believe, so this has to be bytes for legacy # protocol backwards compatibility reasons. assert isinstance(share_type, bytes) @@ -774,3 +737,69 @@ class StorageServer(service.MultiService, Referenceable): share_type=share_type, si=si_s, shnum=shnum, reason=reason, level=log.SCARY, umid="SGx2fA") return None + + +@implementer(RIStorageServer) +class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 + """ + A filesystem-based implementation of ``RIStorageServer``. + + For Foolscap, BucketWriter lifetime is tied to connection: when + disconnection happens, the BucketWriters are removed. + """ + name = 'storage' + + def __init__(self, storage_server): # type: (StorageServer) -> None + self._server = storage_server + + # Canaries and disconnect markers for BucketWriters created via Foolscap: + self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]] + + + def remote_get_version(self): + return self.get_version() + + def remote_allocate_buckets(self, storage_index, + renew_secret, cancel_secret, + sharenums, allocated_size, + canary, owner_num=0): + """Foolscap-specific ``allocate_buckets()`` API.""" + alreadygot, bucketwriters = self._server.allocate_buckets( + storage_index, renew_secret, cancel_secret, sharenums, allocated_size, + owner_num=owner_num, renew_leases=True, + ) + # Abort BucketWriters if disconnection happens. + for bw in bucketwriters.values(): + disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) + self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) + return alreadygot, bucketwriters + + def remote_add_lease(self, storage_index, renew_secret, cancel_secret, + owner_num=1): + return self._server.add_lease(storage_index, renew_secret, cancel_secret) + + def remote_renew_lease(self, storage_index, renew_secret): + return self._server.renew_lease(storage_index, renew_secret) + + def remote_get_buckets(self, storage_index): + return self._server.get_buckets(storage_index) + + def remote_slot_testv_and_readv_and_writev(self, storage_index, + secrets, + test_and_write_vectors, + read_vector): + return self._server.slot_testv_and_readv_and_writev( + storage_index, + secrets, + test_and_write_vectors, + read_vector, + renew_leases=True, + ) + + def remote_slot_readv(self, storage_index, shares, readv): + return self._server.slot_readv(self, storage_index, shares, readv) + + def remote_advise_corrupt_share(self, share_type, storage_index, shnum, + reason): + return self._server.advise_corrupt_share(share_type, storage_index, shnum, + reason) From 25ca767095019f3f0d6288b2a870ef3b98eef112 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 11:49:52 -0700 Subject: [PATCH 258/916] an offering to the windows godesses --- src/allmydata/test/test_storage_web.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index dff3b36f5..282fb67e1 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -22,11 +22,11 @@ import json from six.moves import StringIO from twisted.trial import unittest - from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString from twisted.python.filepath import FilePath +from twisted.python.runtime import platform from foolscap.api import fireEventually from allmydata.util import fileutil, hashutil, base32, pollmixin @@ -1163,8 +1163,12 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.state") - with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - local.write(remote.read()) + with test_pickle.open("w") as local, original_pickle.open("r") as remote: + for line in remote.readlines(): + if platform.isWindows(): + local.write(line.replace("\n", "\r\n")) + else: + local.write(line.replace("\n", "\r\n")) # convert from pickle format to JSON top = Options() @@ -1366,7 +1370,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage.makedirs() test_pickle = storage.child("lease_checker.history") with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - local.write(remote.read()) + for line in remote.readlines(): + if platform.isWindows(): + local.write(line.replace("\n", "\r\n")) + else: + local.write(line) # convert from pickle format to JSON top = Options() From 7080ee6fc7747e1a9ca10ddb47fe81fd5e96a37b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 12:02:06 -0700 Subject: [PATCH 259/916] oops --- src/allmydata/test/test_storage_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 282fb67e1..1cf96d660 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1168,7 +1168,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): if platform.isWindows(): local.write(line.replace("\n", "\r\n")) else: - local.write(line.replace("\n", "\r\n")) + local.write(line) # convert from pickle format to JSON top = Options() From 940c6343cf32318b9ba72f1330fdd5371346ce1f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 12:02:42 -0700 Subject: [PATCH 260/916] consistency --- src/allmydata/test/test_storage_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 1cf96d660..961bbef98 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1369,7 +1369,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.history") - with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: + with test_pickle.open("w") as local, original_pickle.open("r") as remote: for line in remote.readlines(): if platform.isWindows(): local.write(line.replace("\n", "\r\n")) From 90d1e90a14b0a3455e2e5ac86cde814a8a81b378 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Dec 2021 15:05:29 -0500 Subject: [PATCH 261/916] rewrite the Eliot interaction tests to make expected behavior clearer and to have explicit assertions about that behavior --- src/allmydata/test/test_eliotutil.py | 113 ++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 0be02b277..cabe599b3 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -27,13 +27,12 @@ from fixtures import ( ) from testtools import ( TestCase, -) -from testtools import ( TestResult, ) from testtools.matchers import ( Is, IsInstance, + Not, MatchesStructure, Equals, HasLength, @@ -77,33 +76,105 @@ from .common import ( ) -class EliotLoggedTestTests(AsyncTestCase): +def passes(): """ - Tests for the automatic log-related provided by ``EliotLoggedRunTest``. + Create a matcher that matches a ``TestCase`` that runs without failures or + errors. """ - def test_returns_none(self): - Message.log(hello="world") + def run(case): + result = TestResult() + case.run(result) + return result.wasSuccessful() + return AfterPreprocessing(run, Equals(True)) - def test_returns_fired_deferred(self): - Message.log(hello="world") - return succeed(None) - def test_returns_unfired_deferred(self): - Message.log(hello="world") - # @eliot_logged_test automatically gives us an action context but it's - # still our responsibility to maintain it across stack-busting - # operations. - d = DeferredContext(deferLater(reactor, 0.0, lambda: None)) - d.addCallback(lambda ignored: Message.log(goodbye="world")) - # We didn't start an action. We're not finishing an action. - return d.result +class EliotLoggedTestTests(TestCase): + """ + Tests for the automatic log-related provided by ``AsyncTestCase``. + + This class uses ``testtools.TestCase`` because it is inconvenient to nest + ``AsyncTestCase`` inside ``AsyncTestCase`` (in particular, Eliot messages + emitted by the inner test case get observed by the outer test case and if + an inner case emits invalid messages they cause the outer test case to + fail). + """ + def test_fails(self): + """ + A test method of an ``AsyncTestCase`` subclass can fail. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + self.fail("make sure it can fail") + + self.assertThat(UnderTest("test_it"), Not(passes())) + + def test_unserializable_fails(self): + """ + A test method of an ``AsyncTestCase`` subclass that logs an unserializable + value with Eliot fails. + """ + class world(object): + """ + an unserializable object + """ + + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello=world) + + self.assertThat(UnderTest("test_it"), Not(passes())) def test_logs_non_utf_8_byte(self): """ - If an Eliot message is emitted that contains a non-UTF-8 byte string then - the test nevertheless passes. + A test method of an ``AsyncTestCase`` subclass can log a message that + contains a non-UTF-8 byte string and return ``None`` and pass. """ - Message.log(hello=b"\xFF") + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello=b"\xFF") + + self.assertThat(UnderTest("test_it"), passes()) + + def test_returns_none(self): + """ + A test method of an ``AsyncTestCase`` subclass can log a message and + return ``None`` and pass. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello="world") + + self.assertThat(UnderTest("test_it"), passes()) + + def test_returns_fired_deferred(self): + """ + A test method of an ``AsyncTestCase`` subclass can log a message and + return an already-fired ``Deferred`` and pass. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello="world") + return succeed(None) + + self.assertThat(UnderTest("test_it"), passes()) + + def test_returns_unfired_deferred(self): + """ + A test method of an ``AsyncTestCase`` subclass can log a message and + return an unfired ``Deferred`` and pass when the ``Deferred`` fires. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello="world") + # @eliot_logged_test automatically gives us an action context + # but it's still our responsibility to maintain it across + # stack-busting operations. + d = DeferredContext(deferLater(reactor, 0.0, lambda: None)) + d.addCallback(lambda ignored: Message.log(goodbye="world")) + # We didn't start an action. We're not finishing an action. + return d.result + + self.assertThat(UnderTest("test_it"), passes()) class ParseDestinationDescriptionTests(SyncTestCase): From eee1f0975d5bd32acbb5d1c481623235558ae47c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Dec 2021 15:16:16 -0500 Subject: [PATCH 262/916] note about how to clean this up later --- src/allmydata/util/_eliot_updates.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py index 8e3beca45..81db566a4 100644 --- a/src/allmydata/util/_eliot_updates.py +++ b/src/allmydata/util/_eliot_updates.py @@ -6,6 +6,15 @@ only changed enough to add Python 2 compatibility. Every API in this module (except ``eliot_json_encoder``) should be obsolete as soon as we depend on Eliot 1.14 or newer. +When that happens: + +* replace ``capture_logging`` + with ``partial(eliot.testing.capture_logging, encoder_=eliot_json_encoder)`` +* replace ``validateLogging`` + with ``partial(eliot.testing.validateLogging, encoder_=eliot_json_encoder)`` +* replace ``MemoryLogger`` + with ``partial(eliot.MemoryLogger, encoder=eliot_json_encoder)`` + Ported to Python 3. """ From e0092ededaa64a800058658de2d9ab8472acb3bf Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 20:52:22 -0700 Subject: [PATCH 263/916] fine, just skip tests on windows --- src/allmydata/test/test_storage_web.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 961bbef98..a49b71325 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -19,6 +19,7 @@ import time import os.path import re import json +from unittest import skipIf from six.moves import StringIO from twisted.trial import unittest @@ -1153,6 +1154,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d + @skipIf(platform.isWindows()) def test_deserialize_pickle(self): """ The crawler can read existing state from the old pickle format @@ -1163,12 +1165,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.state") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: - for line in remote.readlines(): - if platform.isWindows(): - local.write(line.replace("\n", "\r\n")) - else: - local.write(line) + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: + test_pickle.write(original_pickle.read()) # convert from pickle format to JSON top = Options() @@ -1358,6 +1356,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): second_serial.load(), ) + @skipIf(platform.isWindows()) def test_deserialize_history_pickle(self): """ The crawler can read existing history state from the old pickle @@ -1369,12 +1368,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.history") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: - for line in remote.readlines(): - if platform.isWindows(): - local.write(line.replace("\n", "\r\n")) - else: - local.write(line) + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: + test_pickle.write(original_pickle.read()) # convert from pickle format to JSON top = Options() From 40e7be6d8d7581f4c9fa71c0817207e11ac1a7e6 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 23:46:10 -0700 Subject: [PATCH 264/916] needs reason --- src/allmydata/test/test_storage_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index a49b71325..9292c0b20 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1154,7 +1154,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d - @skipIf(platform.isWindows()) + @skipIf(platform.isWindows(), "pickle test-data can't be loaded on windows") def test_deserialize_pickle(self): """ The crawler can read existing state from the old pickle format @@ -1356,7 +1356,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): second_serial.load(), ) - @skipIf(platform.isWindows()) + @skipIf(platform.isWindows(), "pickle test-data can't be loaded on windows") def test_deserialize_history_pickle(self): """ The crawler can read existing history state from the old pickle From 4bc0df7cc14f53901470a3e0d0f78a6d975c4781 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 2 Dec 2021 00:05:21 -0700 Subject: [PATCH 265/916] file, not path --- src/allmydata/test/test_storage_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 9292c0b20..18ea0220c 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1166,7 +1166,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage.makedirs() test_pickle = storage.child("lease_checker.state") with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - test_pickle.write(original_pickle.read()) + local.write(remote.read()) # convert from pickle format to JSON top = Options() @@ -1369,7 +1369,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage.makedirs() test_pickle = storage.child("lease_checker.history") with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - test_pickle.write(original_pickle.read()) + local.write(remote.read()) # convert from pickle format to JSON top = Options() From 6b8a42b0439bd81bbb8359c256538daf53622733 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 09:34:29 -0500 Subject: [PATCH 266/916] Make the test more robust. --- newsfragments/3850.minor | 0 src/allmydata/test/test_storage_http.py | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 newsfragments/3850.minor diff --git a/newsfragments/3850.minor b/newsfragments/3850.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 442e154a0..e30eb24c7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -63,7 +63,15 @@ class HTTPTests(TestCase): def test_version(self): """ The client can return the version. + + We ignore available disk space since that might change across calls. """ version = yield self.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) expected_version = self.storage_server.remote_get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) self.assertEqual(version, expected_version) From 541b28f4693c900f83fc767c5e012918e98d3b9a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 09:36:56 -0500 Subject: [PATCH 267/916] News file. --- newsfragments/3849.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3849.minor diff --git a/newsfragments/3849.minor b/newsfragments/3849.minor new file mode 100644 index 000000000..e69de29bb From f7cb4d5c92e33b2b25286e4adb8c3ba4d32755bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:02:46 -0500 Subject: [PATCH 268/916] Hook up the new FoolscapStorageServer, and fix enough bugs, such that almost all end-to-end and integration tests pass. --- src/allmydata/client.py | 4 ++-- src/allmydata/storage/immutable.py | 8 ++++---- src/allmydata/storage/server.py | 24 +++++++++++++++++++----- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a2f88ebd6..645e157b6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -36,7 +36,7 @@ from twisted.python.filepath import FilePath import allmydata from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix -from allmydata.storage.server import StorageServer +from allmydata.storage.server import StorageServer, FoolscapStorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper @@ -834,7 +834,7 @@ class _Client(node.Node, pollmixin.PollMixin): if anonymous_storage_enabled(self.config): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) - furl = self.tub.registerReference(ss, furlFile=furl_file) + furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 08b83cd87..173a43e8e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -382,7 +382,7 @@ class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 return data def remote_advise_corrupt_share(self, reason): - return self.ss.remote_advise_corrupt_share(b"immutable", - self.storage_index, - self.shnum, - reason) + return self.ss.advise_corrupt_share(b"immutable", + self.storage_index, + self.shnum, + reason) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9d3ac4012..5dd8cd0bc 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -127,6 +127,9 @@ class StorageServer(service.MultiService): # Map in-progress filesystem path -> BucketWriter: self._bucket_writers = {} # type: Dict[str,BucketWriter] + # These callables will be called with BucketWriters that closed: + self._call_on_bucket_writer_close = [] + def stopService(self): # Cancel any in-progress uploads: for bw in list(self._bucket_writers.values()): @@ -405,9 +408,14 @@ class StorageServer(service.MultiService): if self.stats_provider: self.stats_provider.count('storage_server.bytes_added', consumed_size) del self._bucket_writers[bw.incominghome] - if bw in self._bucket_writer_disconnect_markers: - canary, disconnect_marker = self._bucket_writer_disconnect_markers.pop(bw) - canary.dontNotifyOnDisconnect(disconnect_marker) + for handler in self._call_on_bucket_writer_close: + handler(bw) + + def register_bucket_writer_close_handler(self, handler): + """ + The handler will be called with any ``BucketWriter`` that closes. + """ + self._call_on_bucket_writer_close.append(handler) def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold @@ -755,9 +763,15 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 # Canaries and disconnect markers for BucketWriters created via Foolscap: self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]] + self._server.register_bucket_writer_close_handler(self._bucket_writer_closed) + + def _bucket_writer_closed(self, bw): + if bw in self._bucket_writer_disconnect_markers: + canary, disconnect_marker = self._bucket_writer_disconnect_markers.pop(bw) + canary.dontNotifyOnDisconnect(disconnect_marker) def remote_get_version(self): - return self.get_version() + return self._server.get_version() def remote_allocate_buckets(self, storage_index, renew_secret, cancel_secret, @@ -797,7 +811,7 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 ) def remote_slot_readv(self, storage_index, shares, readv): - return self._server.slot_readv(self, storage_index, shares, readv) + return self._server.slot_readv(storage_index, shares, readv) def remote_advise_corrupt_share(self, share_type, storage_index, shnum, reason): From 476c41e49ec973db18b13d8f3b4e96a70e7c0933 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:29:52 -0500 Subject: [PATCH 269/916] Split out Foolscap code from BucketReader/Writer. --- src/allmydata/storage/immutable.py | 60 +++++++++++++++++++++++------- src/allmydata/storage/server.py | 18 ++++++++- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 173a43e8e..5fea7e2b6 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -230,8 +230,10 @@ class ShareFile(object): return space_freed -@implementer(RIBucketWriter) -class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 +class BucketWriter(object): + """ + Keep track of the process of writing to a ShareFile. + """ def __init__(self, ss, incominghome, finalhome, max_size, lease_info, clock): self.ss = ss @@ -251,7 +253,7 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def allocated_size(self): return self._max_size - def remote_write(self, offset, data): + def write(self, offset, data): # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() @@ -275,9 +277,6 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") - def remote_close(self): - self.close() - def close(self): precondition(not self.closed) self._timeout.cancel() @@ -329,13 +328,10 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 facility="tahoe.storage", level=log.UNUSUAL) self.abort() - def remote_abort(self): + def abort(self): log.msg("storage: aborting sharefile %s" % self.incominghome, facility="tahoe.storage", level=log.UNUSUAL) - self.abort() self.ss.count("abort") - - def abort(self): if self.closed: return @@ -358,8 +354,28 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._timeout.cancel() -@implementer(RIBucketReader) -class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 +@implementer(RIBucketWriter) +class FoolscapBucketWriter(Referenceable): # type: ignore # warner/foolscap#78 + """ + Foolscap-specific BucketWriter. + """ + def __init__(self, bucket_writer): + self._bucket_writer = bucket_writer + + def remote_write(self, offset, data): + return self._bucket_writer.write(offset, data) + + def remote_close(self): + return self._bucket_writer.close() + + def remote_abort(self): + return self._bucket_writer.abort() + + +class BucketReader(object): + """ + Manage the process for reading from a ``ShareFile``. + """ def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss @@ -374,15 +390,31 @@ class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 ), self.shnum) - def remote_read(self, offset, length): + def read(self, offset, length): start = time.time() data = self._share_file.read_share_data(offset, length) self.ss.add_latency("read", time.time() - start) self.ss.count("read") return data - def remote_advise_corrupt_share(self, reason): + def advise_corrupt_share(self, reason): return self.ss.advise_corrupt_share(b"immutable", self.storage_index, self.shnum, reason) + + +@implementer(RIBucketReader) +class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 + """ + Foolscap wrapper for ``BucketReader`` + """ + + def __init__(self, bucket_reader): + self._bucket_reader = bucket_reader + + def remote_read(self, offset, length): + return self._bucket_reader.read(offset, length) + + def remote_advise_corrupt_share(self, reason): + return self._bucket_reader.advise_corrupt_share(reason) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 5dd8cd0bc..8c790b66f 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -33,7 +33,10 @@ from allmydata.storage.lease import LeaseInfo from allmydata.storage.mutable import MutableShareFile, EmptyShare, \ create_mutable_sharefile from allmydata.mutable.layout import MAX_MUTABLE_SHARE_SIZE -from allmydata.storage.immutable import ShareFile, BucketWriter, BucketReader +from allmydata.storage.immutable import ( + ShareFile, BucketWriter, BucketReader, FoolscapBucketWriter, + FoolscapBucketReader, +) from allmydata.storage.crawler import BucketCountingCrawler from allmydata.storage.expirer import LeaseCheckingCrawler @@ -782,10 +785,18 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 storage_index, renew_secret, cancel_secret, sharenums, allocated_size, owner_num=owner_num, renew_leases=True, ) + # Abort BucketWriters if disconnection happens. for bw in bucketwriters.values(): disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) + + # Wrap BucketWriters with Foolscap adapter: + bucketwriters = { + k: FoolscapBucketWriter(bw) + for (k, bw) in bucketwriters.items() + } + return alreadygot, bucketwriters def remote_add_lease(self, storage_index, renew_secret, cancel_secret, @@ -796,7 +807,10 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 return self._server.renew_lease(storage_index, renew_secret) def remote_get_buckets(self, storage_index): - return self._server.get_buckets(storage_index) + return { + k: FoolscapBucketReader(bucket) + for (k, bucket) in self._server.get_buckets(storage_index).items() + } def remote_slot_testv_and_readv_and_writev(self, storage_index, secrets, From 8c3d61a94e6da2e590831afc86afa85e0b0d6e36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:49:23 -0500 Subject: [PATCH 270/916] Bit more backwards compatible. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8c790b66f..bb116ce8e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -626,7 +626,7 @@ class StorageServer(service.MultiService): secrets, test_and_write_vectors, read_vector, - renew_leases, + renew_leases=True, ): """ Read data from shares and conditionally write some data to them. From 439e5f2998ac178f7773190842dba0a9855da893 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:49:30 -0500 Subject: [PATCH 271/916] Insofar as possible, switch to testing without the Foolscap API. --- src/allmydata/test/test_storage.py | 303 +++++++++++++------------ src/allmydata/test/test_storage_web.py | 24 +- 2 files changed, 165 insertions(+), 162 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bc87e168d..bfc1a7cd4 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -31,10 +31,15 @@ from hypothesis import given, strategies import itertools from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32 -from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME +from allmydata.storage.server import ( + StorageServer, DEFAULT_RENEWAL_TIME, FoolscapStorageServer, +) from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile -from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile +from allmydata.storage.immutable import ( + BucketWriter, BucketReader, ShareFile, FoolscapBucketWriter, + FoolscapBucketReader, +) from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ si_b2a, si_a2b @@ -129,25 +134,25 @@ class Bucket(unittest.TestCase): def test_create(self): incoming, final = self.make_workdir("test_create") bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) - bw.remote_write(0, b"a"*25) - bw.remote_write(25, b"b"*25) - bw.remote_write(50, b"c"*25) - bw.remote_write(75, b"d"*7) - bw.remote_close() + bw.write(0, b"a"*25) + bw.write(25, b"b"*25) + bw.write(50, b"c"*25) + bw.write(75, b"d"*7) + bw.close() def test_readwrite(self): incoming, final = self.make_workdir("test_readwrite") bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) - bw.remote_write(0, b"a"*25) - bw.remote_write(25, b"b"*25) - bw.remote_write(50, b"c"*7) # last block may be short - bw.remote_close() + bw.write(0, b"a"*25) + bw.write(25, b"b"*25) + bw.write(50, b"c"*7) # last block may be short + bw.close() # now read from it br = BucketReader(self, bw.finalhome) - self.failUnlessEqual(br.remote_read(0, 25), b"a"*25) - self.failUnlessEqual(br.remote_read(25, 25), b"b"*25) - self.failUnlessEqual(br.remote_read(50, 7), b"c"*7) + self.failUnlessEqual(br.read(0, 25), b"a"*25) + self.failUnlessEqual(br.read(25, 25), b"b"*25) + self.failUnlessEqual(br.read(50, 7), b"c"*7) def test_write_past_size_errors(self): """Writing beyond the size of the bucket throws an exception.""" @@ -157,7 +162,7 @@ class Bucket(unittest.TestCase): ) bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) with self.assertRaises(DataTooLargeError): - bw.remote_write(offset, b"a" * length) + bw.write(offset, b"a" * length) @given( maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), @@ -177,25 +182,25 @@ class Bucket(unittest.TestCase): self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - bw.remote_write(10, expected_data[10:20]) - bw.remote_write(30, expected_data[30:40]) - bw.remote_write(50, expected_data[50:60]) + bw.write(10, expected_data[10:20]) + bw.write(30, expected_data[30:40]) + bw.write(50, expected_data[50:60]) # Then, an overlapping write but with matching data: - bw.remote_write( + bw.write( maybe_overlapping_offset, expected_data[ maybe_overlapping_offset:maybe_overlapping_offset + maybe_overlapping_length ] ) # Now fill in the holes: - bw.remote_write(0, expected_data[0:10]) - bw.remote_write(20, expected_data[20:30]) - bw.remote_write(40, expected_data[40:50]) - bw.remote_write(60, expected_data[60:]) - bw.remote_close() + bw.write(0, expected_data[0:10]) + bw.write(20, expected_data[20:30]) + bw.write(40, expected_data[40:50]) + bw.write(60, expected_data[60:]) + bw.close() br = BucketReader(self, bw.finalhome) - self.assertEqual(br.remote_read(0, length), expected_data) + self.assertEqual(br.read(0, length), expected_data) @given( @@ -215,21 +220,21 @@ class Bucket(unittest.TestCase): self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - bw.remote_write(10, b"1" * 10) - bw.remote_write(30, b"1" * 10) - bw.remote_write(50, b"1" * 10) + bw.write(10, b"1" * 10) + bw.write(30, b"1" * 10) + bw.write(50, b"1" * 10) # Then, write something that might overlap with some of them, but # conflicts. Then fill in holes left by first three writes. Conflict is # inevitable. with self.assertRaises(ConflictingWriteError): - bw.remote_write( + bw.write( maybe_overlapping_offset, b'X' * min(maybe_overlapping_length, length - maybe_overlapping_offset), ) - bw.remote_write(0, b"1" * 10) - bw.remote_write(20, b"1" * 10) - bw.remote_write(40, b"1" * 10) - bw.remote_write(60, b"1" * 40) + bw.write(0, b"1" * 10) + bw.write(20, b"1" * 10) + bw.write(40, b"1" * 10) + bw.write(60, b"1" * 40) def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share @@ -274,15 +279,15 @@ class Bucket(unittest.TestCase): # Now read from it. br = BucketReader(mockstorageserver, final) - self.failUnlessEqual(br.remote_read(0, len(share_data)), share_data) + self.failUnlessEqual(br.read(0, len(share_data)), share_data) # Read past the end of share data to get the cancel secret. read_length = len(share_data) + len(ownernumber) + len(renewsecret) + len(cancelsecret) - result_of_read = br.remote_read(0, read_length) + result_of_read = br.read(0, read_length) self.failUnlessEqual(result_of_read, share_data) - result_of_read = br.remote_read(0, len(share_data)+1) + result_of_read = br.read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) def _assert_timeout_only_after_30_minutes(self, clock, bw): @@ -320,7 +325,7 @@ class Bucket(unittest.TestCase): clock.advance(29 * 60) # .. but we receive a write! So that should delay the timeout again to # another 30 minutes. - bw.remote_write(0, b"hello") + bw.write(0, b"hello") self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_closing_cancels_timeout(self): @@ -374,7 +379,7 @@ class BucketProxy(unittest.TestCase): fileutil.make_dirs(basedir) fileutil.make_dirs(os.path.join(basedir, "tmp")) bw = BucketWriter(self, incoming, final, size, self.make_lease(), Clock()) - rb = RemoteBucket(bw) + rb = RemoteBucket(FoolscapBucketWriter(bw)) return bw, rb, final def make_lease(self): @@ -446,7 +451,7 @@ class BucketProxy(unittest.TestCase): # now read everything back def _start_reading(res): br = BucketReader(self, sharefname) - rb = RemoteBucket(br) + rb = RemoteBucket(FoolscapBucketReader(br)) server = NoNetworkServer(b"abc", None) rbp = rbp_class(rb, server, storage_index=b"") self.failUnlessIn("to peer", repr(rbp)) @@ -514,20 +519,20 @@ class Server(unittest.TestCase): def test_declares_fixed_1528(self): ss = self.create("test_declares_fixed_1528") - ver = ss.remote_get_version() + ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnless(sv1.get(b'prevents-read-past-end-of-share-data'), sv1) def test_declares_maximum_share_sizes(self): ss = self.create("test_declares_maximum_share_sizes") - ver = ss.remote_get_version() + ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnlessIn(b'maximum-immutable-share-size', sv1) self.failUnlessIn(b'maximum-mutable-share-size', sv1) def test_declares_available_space(self): ss = self.create("test_declares_available_space") - ver = ss.remote_get_version() + ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnlessIn(b'available-space', sv1) @@ -538,7 +543,9 @@ class Server(unittest.TestCase): """ renew_secret = hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)) cancel_secret = hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret)) - return ss._allocate_buckets( + if isinstance(ss, FoolscapStorageServer): + ss = ss._server + return ss.allocate_buckets( storage_index, renew_secret, cancel_secret, sharenums, size, @@ -562,12 +569,12 @@ class Server(unittest.TestCase): shnum, bucket = list(writers.items())[0] # This test is going to hammer your filesystem if it doesn't make a sparse file for this. :-( - bucket.remote_write(2**32, b"ab") - bucket.remote_close() + bucket.write(2**32, b"ab") + bucket.close() - readers = ss.remote_get_buckets(b"allocate") + readers = ss.get_buckets(b"allocate") reader = readers[shnum] - self.failUnlessEqual(reader.remote_read(2**32, 2), b"ab") + self.failUnlessEqual(reader.read(2**32, 2), b"ab") def test_dont_overfill_dirs(self): """ @@ -578,8 +585,8 @@ class Server(unittest.TestCase): ss = self.create("test_dont_overfill_dirs") already, writers = self.allocate(ss, b"storageindex", [0], 10) for i, wb in writers.items(): - wb.remote_write(0, b"%10d" % i) - wb.remote_close() + wb.write(0, b"%10d" % i) + wb.close() storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), "shares") children_of_storedir = set(os.listdir(storedir)) @@ -588,8 +595,8 @@ class Server(unittest.TestCase): # chars the same as the first storageindex. already, writers = self.allocate(ss, b"storageindey", [0], 10) for i, wb in writers.items(): - wb.remote_write(0, b"%10d" % i) - wb.remote_close() + wb.write(0, b"%10d" % i) + wb.close() storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), "shares") new_children_of_storedir = set(os.listdir(storedir)) @@ -599,8 +606,8 @@ class Server(unittest.TestCase): ss = self.create("test_remove_incoming") already, writers = self.allocate(ss, b"vid", list(range(3)), 10) for i,wb in writers.items(): - wb.remote_write(0, b"%10d" % i) - wb.remote_close() + wb.write(0, b"%10d" % i) + wb.close() incoming_share_dir = wb.incominghome incoming_bucket_dir = os.path.dirname(incoming_share_dir) incoming_prefix_dir = os.path.dirname(incoming_bucket_dir) @@ -619,32 +626,32 @@ class Server(unittest.TestCase): # Now abort the writers. for writer in writers.values(): - writer.remote_abort() + writer.abort() self.failUnlessEqual(ss.allocated_size(), 0) def test_allocate(self): ss = self.create("test_allocate") - self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {}) + self.failUnlessEqual(ss.get_buckets(b"allocate"), {}) already,writers = self.allocate(ss, b"allocate", [0,1,2], 75) self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) # while the buckets are open, they should not count as readable - self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {}) + self.failUnlessEqual(ss.get_buckets(b"allocate"), {}) # close the buckets for i,wb in writers.items(): - wb.remote_write(0, b"%25d" % i) - wb.remote_close() + wb.write(0, b"%25d" % i) + wb.close() # aborting a bucket that was already closed is a no-op - wb.remote_abort() + wb.abort() # now they should be readable - b = ss.remote_get_buckets(b"allocate") + b = ss.get_buckets(b"allocate") self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].remote_read(0, 25), b"%25d" % 0) + self.failUnlessEqual(b[0].read(0, 25), b"%25d" % 0) b_str = str(b[0]) self.failUnlessIn("BucketReader", b_str) self.failUnlessIn("mfwgy33dmf2g 0", b_str) @@ -665,15 +672,15 @@ class Server(unittest.TestCase): # aborting the writes should remove the tempfiles for i,wb in writers2.items(): - wb.remote_abort() + wb.abort() already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75) self.failUnlessEqual(already2, set([0,1,2])) self.failUnlessEqual(set(writers2.keys()), set([5])) for i,wb in writers2.items(): - wb.remote_abort() + wb.abort() for i,wb in writers.items(): - wb.remote_abort() + wb.abort() def test_allocate_without_lease_renewal(self): """ @@ -696,8 +703,8 @@ class Server(unittest.TestCase): ss, storage_index, [0], 1, renew_leases=False, ) (writer,) = writers.values() - writer.remote_write(0, b"x") - writer.remote_close() + writer.write(0, b"x") + writer.close() # It should have a lease granted at the current time. shares = dict(ss._get_bucket_shares(storage_index)) @@ -719,8 +726,8 @@ class Server(unittest.TestCase): ss, storage_index, [1], 1, renew_leases=False, ) (writer,) = writers.values() - writer.remote_write(0, b"x") - writer.remote_close() + writer.write(0, b"x") + writer.close() # The first share's lease expiration time is unchanged. shares = dict(ss._get_bucket_shares(storage_index)) @@ -736,8 +743,8 @@ class Server(unittest.TestCase): def test_bad_container_version(self): ss = self.create("test_bad_container_version") a,w = self.allocate(ss, b"si1", [0], 10) - w[0].remote_write(0, b"\xff"*10) - w[0].remote_close() + w[0].write(0, b"\xff"*10) + w[0].close() fn = os.path.join(ss.sharedir, storage_index_to_dir(b"si1"), "0") f = open(fn, "rb+") @@ -745,15 +752,15 @@ class Server(unittest.TestCase): f.write(struct.pack(">L", 0)) # this is invalid: minimum used is v1 f.close() - ss.remote_get_buckets(b"allocate") + ss.get_buckets(b"allocate") e = self.failUnlessRaises(UnknownImmutableContainerVersionError, - ss.remote_get_buckets, b"si1") + ss.get_buckets, b"si1") self.failUnlessIn(" had version 0 but we wanted 1", str(e)) def test_disconnect(self): # simulate a disconnection - ss = self.create("test_disconnect") + ss = FoolscapStorageServer(self.create("test_disconnect")) renew_secret = b"r" * 32 cancel_secret = b"c" * 32 canary = FakeCanary() @@ -789,7 +796,7 @@ class Server(unittest.TestCase): } self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) - ss = self.create("test_reserved_space", reserved_space=reserved) + ss = FoolscapStorageServer(self.create("test_reserved_space", reserved_space=reserved)) # 15k available, 10k reserved, leaves 5k for shares # a newly created and filled share incurs this much overhead, beyond @@ -810,28 +817,28 @@ class Server(unittest.TestCase): self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed - self.failUnlessEqual(len(ss._bucket_writers), 3) + self.failUnlessEqual(len(ss._server._bucket_writers), 3) # allocating 1001-byte shares only leaves room for one canary2 = FakeCanary() already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary2) self.failUnlessEqual(len(writers2), 1) - self.failUnlessEqual(len(ss._bucket_writers), 4) + self.failUnlessEqual(len(ss._server._bucket_writers), 4) # we abandon the first set, so their provisional allocation should be # returned canary.disconnected() - self.failUnlessEqual(len(ss._bucket_writers), 1) + self.failUnlessEqual(len(ss._server._bucket_writers), 1) # now we have a provisional allocation of 1001 bytes # and we close the second set, so their provisional allocation should # become real, long-term allocation, and grows to include the # overhead. for bw in writers2.values(): - bw.remote_write(0, b"a"*25) - bw.remote_close() - self.failUnlessEqual(len(ss._bucket_writers), 0) + bw.write(0, b"a"*25) + bw.close() + self.failUnlessEqual(len(ss._server._bucket_writers), 0) # this also changes the amount reported as available by call_get_disk_stats allocated = 1001 + OVERHEAD + LEASE_SIZE @@ -848,12 +855,12 @@ class Server(unittest.TestCase): canary=canary3, ) self.failUnlessEqual(len(writers3), 39) - self.failUnlessEqual(len(ss._bucket_writers), 39) + self.failUnlessEqual(len(ss._server._bucket_writers), 39) canary3.disconnected() - self.failUnlessEqual(len(ss._bucket_writers), 0) - ss.disownServiceParent() + self.failUnlessEqual(len(ss._server._bucket_writers), 0) + ss._server.disownServiceParent() del ss def test_seek(self): @@ -882,24 +889,22 @@ class Server(unittest.TestCase): Given a StorageServer, create a bucket with 5 shares and return renewal and cancellation secrets. """ - canary = FakeCanary() sharenums = list(range(5)) size = 100 # Creating a bucket also creates a lease: rs, cs = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) - already, writers = ss.remote_allocate_buckets(storage_index, rs, cs, - sharenums, size, canary) + already, writers = ss.allocate_buckets(storage_index, rs, cs, + sharenums, size) self.failUnlessEqual(len(already), expected_already) self.failUnlessEqual(len(writers), expected_writers) for wb in writers.values(): - wb.remote_close() + wb.close() return rs, cs def test_leases(self): ss = self.create("test_leases") - canary = FakeCanary() sharenums = list(range(5)) size = 100 @@ -919,54 +924,54 @@ class Server(unittest.TestCase): # and a third lease, using add-lease rs2a,cs2a = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) - ss.remote_add_lease(b"si1", rs2a, cs2a) + ss.add_lease(b"si1", rs2a, cs2a) (lease1, lease2, lease3) = ss.get_leases(b"si1") self.assertTrue(lease1.is_renew_secret(rs1)) self.assertTrue(lease2.is_renew_secret(rs2)) self.assertTrue(lease3.is_renew_secret(rs2a)) # add-lease on a missing storage index is silently ignored - self.assertIsNone(ss.remote_add_lease(b"si18", b"", b"")) + self.assertIsNone(ss.add_lease(b"si18", b"", b"")) # check that si0 is readable - readers = ss.remote_get_buckets(b"si0") + readers = ss.get_buckets(b"si0") self.failUnlessEqual(len(readers), 5) # renew the first lease. Only the proper renew_secret should work - ss.remote_renew_lease(b"si0", rs0) - self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", cs0) - self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", rs1) + ss.renew_lease(b"si0", rs0) + self.failUnlessRaises(IndexError, ss.renew_lease, b"si0", cs0) + self.failUnlessRaises(IndexError, ss.renew_lease, b"si0", rs1) # check that si0 is still readable - readers = ss.remote_get_buckets(b"si0") + readers = ss.get_buckets(b"si0") self.failUnlessEqual(len(readers), 5) # There is no such method as remote_cancel_lease for now -- see # ticket #1528. - self.failIf(hasattr(ss, 'remote_cancel_lease'), \ - "ss should not have a 'remote_cancel_lease' method/attribute") + self.failIf(hasattr(FoolscapStorageServer(ss), 'remote_cancel_lease'), \ + "ss should not have a 'remote_cancel_lease' method/attribute") # test overlapping uploads rs3,cs3 = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) rs4,cs4 = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) - already,writers = ss.remote_allocate_buckets(b"si3", rs3, cs3, - sharenums, size, canary) + already,writers = ss.allocate_buckets(b"si3", rs3, cs3, + sharenums, size) self.failUnlessEqual(len(already), 0) self.failUnlessEqual(len(writers), 5) - already2,writers2 = ss.remote_allocate_buckets(b"si3", rs4, cs4, - sharenums, size, canary) + already2,writers2 = ss.allocate_buckets(b"si3", rs4, cs4, + sharenums, size) self.failUnlessEqual(len(already2), 0) self.failUnlessEqual(len(writers2), 0) for wb in writers.values(): - wb.remote_close() + wb.close() leases = list(ss.get_leases(b"si3")) self.failUnlessEqual(len(leases), 1) - already3,writers3 = ss.remote_allocate_buckets(b"si3", rs4, cs4, - sharenums, size, canary) + already3,writers3 = ss.allocate_buckets(b"si3", rs4, cs4, + sharenums, size) self.failUnlessEqual(len(already3), 5) self.failUnlessEqual(len(writers3), 0) @@ -991,7 +996,7 @@ class Server(unittest.TestCase): clock.advance(123456) # Adding a lease with matching renewal secret just renews it: - ss.remote_add_lease(b"si0", renewal_secret, cancel_secret) + ss.add_lease(b"si0", renewal_secret, cancel_secret) [lease] = ss.get_leases(b"si0") self.assertEqual(lease.get_expiration_time(), 123 + 123456 + DEFAULT_RENEWAL_TIME) @@ -1027,14 +1032,14 @@ class Server(unittest.TestCase): self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) for i,wb in writers.items(): - wb.remote_write(0, b"%25d" % i) - wb.remote_close() + wb.write(0, b"%25d" % i) + wb.close() # since we discard the data, the shares should be present but sparse. # Since we write with some seeks, the data we read back will be all # zeros. - b = ss.remote_get_buckets(b"vid") + b = ss.get_buckets(b"vid") self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].remote_read(0, 25), b"\x00" * 25) + self.failUnlessEqual(b[0].read(0, 25), b"\x00" * 25) def test_advise_corruption(self): workdir = self.workdir("test_advise_corruption") @@ -1042,8 +1047,8 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) si0_s = base32.b2a(b"si0") - ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, - b"This share smells funny.\n") + ss.advise_corrupt_share(b"immutable", b"si0", 0, + b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") reports = os.listdir(reportdir) self.failUnlessEqual(len(reports), 1) @@ -1062,12 +1067,12 @@ class Server(unittest.TestCase): already,writers = self.allocate(ss, b"si1", [1], 75) self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([1])) - writers[1].remote_write(0, b"data") - writers[1].remote_close() + writers[1].write(0, b"data") + writers[1].close() - b = ss.remote_get_buckets(b"si1") + b = ss.get_buckets(b"si1") self.failUnlessEqual(set(b.keys()), set([1])) - b[1].remote_advise_corrupt_share(b"This share tastes like dust.\n") + b[1].advise_corrupt_share(b"This share tastes like dust.\n") reports = os.listdir(reportdir) self.failUnlessEqual(len(reports), 2) @@ -1125,7 +1130,7 @@ class MutableServer(unittest.TestCase): write_enabler = self.write_enabler(we_tag) renew_secret = self.renew_secret(lease_tag) cancel_secret = self.cancel_secret(lease_tag) - rstaraw = ss.remote_slot_testv_and_readv_and_writev + rstaraw = ss.slot_testv_and_readv_and_writev testandwritev = dict( [ (shnum, ([], [], None) ) for shnum in sharenums ] ) readv = [] @@ -1146,7 +1151,7 @@ class MutableServer(unittest.TestCase): f.seek(0) f.write(b"BAD MAGIC") f.close() - read = ss.remote_slot_readv + read = ss.slot_readv e = self.failUnlessRaises(UnknownMutableContainerVersionError, read, b"si1", [0], [(0,10)]) self.failUnlessIn(" had magic ", str(e)) @@ -1156,8 +1161,8 @@ class MutableServer(unittest.TestCase): ss = self.create("test_container_size") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0,1,2]), 100) - read = ss.remote_slot_readv - rstaraw = ss.remote_slot_testv_and_readv_and_writev + read = ss.slot_readv + rstaraw = ss.slot_testv_and_readv_and_writev secrets = ( self.write_enabler(b"we1"), self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) @@ -1237,7 +1242,7 @@ class MutableServer(unittest.TestCase): # Also see if the server explicitly declares that it supports this # feature. - ver = ss.remote_get_version() + ver = ss.get_version() storage_v1_ver = ver[b"http://allmydata.org/tahoe/protocols/storage/v1"] self.failUnless(storage_v1_ver.get(b"fills-holes-with-zero-bytes")) @@ -1255,7 +1260,7 @@ class MutableServer(unittest.TestCase): self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0,1,2]), 100) - read = ss.remote_slot_readv + read = ss.slot_readv self.failUnlessEqual(read(b"si1", [0], [(0, 10)]), {0: [b""]}) self.failUnlessEqual(read(b"si1", [], [(0, 10)]), @@ -1268,7 +1273,7 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev + write = ss.slot_testv_and_readv_and_writev answer = write(b"si1", secrets, {0: ([], [(0,data)], None)}, []) @@ -1278,7 +1283,7 @@ class MutableServer(unittest.TestCase): {0: [b"00000000001111111111"]}) self.failUnlessEqual(read(b"si1", [0], [(95,10)]), {0: [b"99999"]}) - #self.failUnlessEqual(s0.remote_get_length(), 100) + #self.failUnlessEqual(s0.get_length(), 100) bad_secrets = (b"bad write enabler", secrets[1], secrets[2]) f = self.failUnlessRaises(BadWriteEnablerError, @@ -1312,8 +1317,8 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv + write = ss.slot_testv_and_readv_and_writev + read = ss.slot_readv def reset(): write(b"si1", secrets, @@ -1357,8 +1362,8 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv + write = ss.slot_testv_and_readv_and_writev + read = ss.slot_readv data = [(b"%d" % i) * 100 for i in range(3)] rc = write(b"si1", secrets, {0: ([], [(0,data[0])], None), @@ -1389,8 +1394,8 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1-%d" % n), self.cancel_secret(b"we1-%d" % n) ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv + write = ss.slot_testv_and_readv_and_writev + read = ss.slot_readv rc = write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, []) self.failUnlessEqual(rc, (True, {})) @@ -1406,7 +1411,7 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(len(list(s0.get_leases())), 1) # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None) + self.failUnlessEqual(ss.add_lease(b"si18", b"", b""), None) # re-allocate the slots and use the same secrets, that should update # the lease @@ -1414,7 +1419,7 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(len(list(s0.get_leases())), 1) # renew it directly - ss.remote_renew_lease(b"si1", secrets(0)[1]) + ss.renew_lease(b"si1", secrets(0)[1]) self.failUnlessEqual(len(list(s0.get_leases())), 1) # now allocate them with a bunch of different secrets, to trigger the @@ -1422,7 +1427,7 @@ class MutableServer(unittest.TestCase): write(b"si1", secrets(1), {0: ([], [(0,data)], None)}, []) self.failUnlessEqual(len(list(s0.get_leases())), 2) secrets2 = secrets(2) - ss.remote_add_lease(b"si1", secrets2[1], secrets2[2]) + ss.add_lease(b"si1", secrets2[1], secrets2[2]) self.failUnlessEqual(len(list(s0.get_leases())), 3) write(b"si1", secrets(3), {0: ([], [(0,data)], None)}, []) write(b"si1", secrets(4), {0: ([], [(0,data)], None)}, []) @@ -1440,11 +1445,11 @@ class MutableServer(unittest.TestCase): # read back the leases, make sure they're still intact. self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) - ss.remote_renew_lease(b"si1", secrets(0)[1]) - ss.remote_renew_lease(b"si1", secrets(1)[1]) - ss.remote_renew_lease(b"si1", secrets(2)[1]) - ss.remote_renew_lease(b"si1", secrets(3)[1]) - ss.remote_renew_lease(b"si1", secrets(4)[1]) + ss.renew_lease(b"si1", secrets(0)[1]) + ss.renew_lease(b"si1", secrets(1)[1]) + ss.renew_lease(b"si1", secrets(2)[1]) + ss.renew_lease(b"si1", secrets(3)[1]) + ss.renew_lease(b"si1", secrets(4)[1]) self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) # get a new copy of the leases, with the current timestamps. Reading # data and failing to renew/cancel leases should leave the timestamps @@ -1455,7 +1460,7 @@ class MutableServer(unittest.TestCase): # examine the exception thus raised, make sure the old nodeid is # present, to provide for share migration e = self.failUnlessRaises(IndexError, - ss.remote_renew_lease, b"si1", + ss.renew_lease, b"si1", secrets(20)[1]) e_s = str(e) self.failUnlessIn("Unable to renew non-existent lease", e_s) @@ -1490,7 +1495,7 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1-%d" % n), self.cancel_secret(b"we1-%d" % n) ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev + write = ss.slot_testv_and_readv_and_writev write_enabler, renew_secret, cancel_secret = secrets(0) rc = write(b"si1", (write_enabler, renew_secret, cancel_secret), {0: ([], [(0,data)], None)}, []) @@ -1506,7 +1511,7 @@ class MutableServer(unittest.TestCase): clock.advance(835) # Adding a lease renews it: - ss.remote_add_lease(b"si1", renew_secret, cancel_secret) + ss.add_lease(b"si1", renew_secret, cancel_secret) [lease] = s0.get_leases() self.assertEqual(lease.get_expiration_time(), 235 + 835 + DEFAULT_RENEWAL_TIME) @@ -1515,8 +1520,8 @@ class MutableServer(unittest.TestCase): ss = self.create("test_remove") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0,1,2]), 100) - readv = ss.remote_slot_readv - writev = ss.remote_slot_testv_and_readv_and_writev + readv = ss.slot_readv + writev = ss.slot_testv_and_readv_and_writev secrets = ( self.write_enabler(b"we1"), self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) @@ -1620,7 +1625,7 @@ class MutableServer(unittest.TestCase): # We don't even need to create any shares to exercise this # functionality. Just go straight to sending a truncate-to-zero # write. - testv_is_good, read_data = ss.remote_slot_testv_and_readv_and_writev( + testv_is_good, read_data = ss.slot_testv_and_readv_and_writev( storage_index=storage_index, secrets=secrets, test_and_write_vectors={ @@ -1638,7 +1643,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() self.ss = self.create("MDMFProxies storage test server") - self.rref = RemoteBucket(self.ss) + self.rref = RemoteBucket(FoolscapStorageServer(self.ss)) self.storage_server = _StorageServer(lambda: self.rref) self.secrets = (self.write_enabler(b"we_secret"), self.renew_secret(b"renew_secret"), @@ -1805,7 +1810,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): If tail_segment=True, then I will write a share that has a smaller tail segment than other segments. """ - write = self.ss.remote_slot_testv_and_readv_and_writev + write = self.ss.slot_testv_and_readv_and_writev data = self.build_test_mdmf_share(tail_segment, empty) # Finally, we write the whole thing to the storage server in one # pass. @@ -1873,7 +1878,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): empty=False): # Some tests need SDMF shares to verify that we can still # read them. This method writes one, which resembles but is not - write = self.ss.remote_slot_testv_and_readv_and_writev + write = self.ss.slot_testv_and_readv_and_writev share = self.build_test_sdmf_share(empty) testvs = [(0, 1, b"eq", b"")] tws = {} @@ -2205,7 +2210,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # blocks. mw = self._make_new_mw(b"si1", 0) # Test writing some blocks. - read = self.ss.remote_slot_readv + read = self.ss.slot_readv expected_private_key_offset = struct.calcsize(MDMFHEADER) expected_sharedata_offset = struct.calcsize(MDMFHEADER) + \ PRIVATE_KEY_SIZE + \ @@ -2996,7 +3001,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d = sdmfr.finish_publishing() def _then(ignored): self.failUnlessEqual(self.rref.write_count, 1) - read = self.ss.remote_slot_readv + read = self.ss.slot_readv self.failUnlessEqual(read(b"si1", [0], [(0, len(data))]), {0: [data]}) d.addCallback(_then) @@ -3053,7 +3058,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): sdmfw.finish_publishing()) def _then_again(results): self.failUnless(results[0]) - read = self.ss.remote_slot_readv + read = self.ss.slot_readv self.failUnlessEqual(read(b"si1", [0], [(1, 8)]), {0: [struct.pack(">Q", 1)]}) self.failUnlessEqual(read(b"si1", [0], [(9, len(data) - 9)]), diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 38e380223..5d72fbd82 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -38,7 +38,6 @@ from allmydata.web.storage import ( StorageStatusElement, remove_prefix ) -from .common_util import FakeCanary from .common_web import ( render, @@ -289,28 +288,27 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): mutable_si_3, rs3, cs3, we3 = make_mutable(b"\x03" * 16) rs3a, cs3a = make_extra_lease(mutable_si_3, 1) sharenums = [0] - canary = FakeCanary() # note: 'tahoe debug dump-share' will not handle this file, since the # inner contents are not a valid CHK share data = b"\xff" * 1000 - a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() + a,w = ss.allocate_buckets(immutable_si_0, rs0, cs0, sharenums, + 1000) + w[0].write(0, data) + w[0].close() - a,w = ss.remote_allocate_buckets(immutable_si_1, rs1, cs1, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() - ss.remote_add_lease(immutable_si_1, rs1a, cs1a) + a,w = ss.allocate_buckets(immutable_si_1, rs1, cs1, sharenums, + 1000) + w[0].write(0, data) + w[0].close() + ss.add_lease(immutable_si_1, rs1a, cs1a) - writev = ss.remote_slot_testv_and_readv_and_writev + writev = ss.slot_testv_and_readv_and_writev writev(mutable_si_2, (we2, rs2, cs2), {0: ([], [(0,data)], len(data))}, []) writev(mutable_si_3, (we3, rs3, cs3), {0: ([], [(0,data)], len(data))}, []) - ss.remote_add_lease(mutable_si_3, rs3a, cs3a) + ss.add_lease(mutable_si_3, rs3a, cs3a) self.sis = [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] self.renew_secrets = [rs0, rs1, rs1a, rs2, rs3, rs3a] From 53ff16f1a43717dd86e9582c379beb4d92ea17e9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 2 Dec 2021 12:56:52 -0700 Subject: [PATCH 272/916] rst for news --- newsfragments/3825.security | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/newsfragments/3825.security b/newsfragments/3825.security index df83821de..3d112dd49 100644 --- a/newsfragments/3825.security +++ b/newsfragments/3825.security @@ -1,6 +1,8 @@ The lease-checker now uses JSON instead of pickle to serialize its state. tahoe will now refuse to run until you either delete all pickle files or -migrate them using the new command: +migrate them using the new command:: tahoe admin migrate-crawler + +This will migrate all crawler-related pickle files. From 314b20291442bd485b9919081611efb9c145c277 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Dec 2021 12:58:12 -0500 Subject: [PATCH 273/916] Ignore another field which can change. --- src/allmydata/test/test_storage_http.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e30eb24c7..23a3e3ea6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -64,14 +64,21 @@ class HTTPTests(TestCase): """ The client can return the version. - We ignore available disk space since that might change across calls. + We ignore available disk space and max immutable share size, since that + might change across calls. """ version = yield self.client.get_version() version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) expected_version = self.storage_server.remote_get_version() expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) self.assertEqual(version, expected_version) From 90f8480cf0c2fb97fd5e420c6e9ae029dad203b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Dec 2021 13:09:27 -0500 Subject: [PATCH 274/916] Make more of the unittests pass again with the StorageServer factoring. --- src/allmydata/storage/http_server.py | 2 +- src/allmydata/storage/server.py | 1 + src/allmydata/test/no_network.py | 6 ++++-- src/allmydata/test/test_checker.py | 6 +++--- src/allmydata/test/test_client.py | 2 +- src/allmydata/test/test_crawler.py | 14 +++++++------- src/allmydata/test/test_helper.py | 7 ++++--- src/allmydata/test/test_hung_server.py | 4 ++-- src/allmydata/test/test_storage_http.py | 2 +- 9 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 327892ecd..6297ef484 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -91,4 +91,4 @@ class HTTPServer(object): @_authorized_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): - return self._cbor(request, self._storage_server.remote_get_version()) + return self._cbor(request, self._storage_server.get_version()) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index bb116ce8e..0df9f23d8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -64,6 +64,7 @@ class StorageServer(service.MultiService): """ Implement the business logic for the storage server. """ + name = "storage" LeaseCheckerClass = LeaseCheckingCrawler def __init__(self, storedir, nodeid, reserved_space=0, diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index b9fa99005..97cb371e6 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -50,7 +50,9 @@ from allmydata.util.assertutil import _assert from allmydata import uri as tahoe_uri from allmydata.client import _Client -from allmydata.storage.server import StorageServer, storage_index_to_dir +from allmydata.storage.server import ( + StorageServer, storage_index_to_dir, FoolscapStorageServer, +) from allmydata.util import fileutil, idlib, hashutil from allmydata.util.hashutil import permute_server_hash from allmydata.util.fileutil import abspath_expanduser_unicode @@ -417,7 +419,7 @@ class NoNetworkGrid(service.MultiService): ss.setServiceParent(middleman) serverid = ss.my_nodeid self.servers_by_number[i] = ss - wrapper = wrap_storage_server(ss) + wrapper = wrap_storage_server(FoolscapStorageServer(ss)) self.wrappers_by_id[serverid] = wrapper self.proxies_by_id[serverid] = NoNetworkServer(serverid, wrapper) self.rebuild_serverlist() diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index f56ecd089..3d64d4976 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -773,13 +773,13 @@ class AddLease(GridTestMixin, unittest.TestCase): d.addCallback(_check_cr, "mutable-normal") really_did_break = [] - # now break the server's remote_add_lease call + # now break the server's add_lease call def _break_add_lease(ign): def broken_add_lease(*args, **kwargs): really_did_break.append(1) raise KeyError("intentional failure, should be ignored") - assert self.g.servers_by_number[0].remote_add_lease - self.g.servers_by_number[0].remote_add_lease = broken_add_lease + assert self.g.servers_by_number[0].add_lease + self.g.servers_by_number[0].add_lease = broken_add_lease d.addCallback(_break_add_lease) # and confirm that the files still look healthy diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index a2572e735..c65a2fa2c 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -601,7 +601,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): "enabled = true\n") c = yield client.create_client(basedir) ss = c.getServiceNamed("storage") - verdict = ss.remote_get_version() + verdict = ss.get_version() self.failUnlessReallyEqual(verdict[b"application-version"], allmydata.__full_version__.encode("ascii")) self.failIfEqual(str(allmydata.__version__), "unknown") diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index a9be90c43..80d732986 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -27,7 +27,7 @@ from allmydata.util import fileutil, hashutil, pollmixin from allmydata.storage.server import StorageServer, si_b2a from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded -from allmydata.test.common_util import StallMixin, FakeCanary +from allmydata.test.common_util import StallMixin class BucketEnumeratingCrawler(ShareCrawler): cpu_slice = 500 # make sure it can complete in a single slice @@ -124,12 +124,12 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def write(self, i, ss, serverid, tail=0): si = self.si(i) si = si[:-1] + bytes(bytearray((tail,))) - had,made = ss.remote_allocate_buckets(si, - self.rs(i, serverid), - self.cs(i, serverid), - set([0]), 99, FakeCanary()) - made[0].remote_write(0, b"data") - made[0].remote_close() + had,made = ss.allocate_buckets(si, + self.rs(i, serverid), + self.cs(i, serverid), + set([0]), 99) + made[0].write(0, b"data") + made[0].close() return si_b2a(si) def test_immediate(self): diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 3faffbe0d..933a2b591 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -39,6 +39,7 @@ from allmydata.crypto import aes from allmydata.storage.server import ( si_b2a, StorageServer, + FoolscapStorageServer, ) from allmydata.storage_client import StorageFarmBroker from allmydata.immutable.layout import ( @@ -427,7 +428,7 @@ class CHKCheckerAndUEBFetcherTests(SyncTestCase): """ storage_index = b"a" * 16 serverid = b"b" * 20 - storage = StorageServer(self.mktemp(), serverid) + storage = FoolscapStorageServer(StorageServer(self.mktemp(), serverid)) rref_without_ueb = LocalWrapper(storage, fireNow) yield write_bad_share(rref_without_ueb, storage_index) server_without_ueb = NoNetworkServer(serverid, rref_without_ueb) @@ -451,7 +452,7 @@ class CHKCheckerAndUEBFetcherTests(SyncTestCase): """ storage_index = b"a" * 16 serverid = b"b" * 20 - storage = StorageServer(self.mktemp(), serverid) + storage = FoolscapStorageServer(StorageServer(self.mktemp(), serverid)) rref_with_ueb = LocalWrapper(storage, fireNow) ueb = { "needed_shares": 2, @@ -487,7 +488,7 @@ class CHKCheckerAndUEBFetcherTests(SyncTestCase): in [b"b", b"c"] ) storages = list( - StorageServer(self.mktemp(), serverid) + FoolscapStorageServer(StorageServer(self.mktemp(), serverid)) for serverid in serverids ) diff --git a/src/allmydata/test/test_hung_server.py b/src/allmydata/test/test_hung_server.py index 490315500..162b1d79c 100644 --- a/src/allmydata/test/test_hung_server.py +++ b/src/allmydata/test/test_hung_server.py @@ -73,7 +73,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin, def _copy_share(self, share, to_server): (sharenum, sharefile) = share (id, ss) = to_server - shares_dir = os.path.join(ss.original.storedir, "shares") + shares_dir = os.path.join(ss.original._server.storedir, "shares") si = uri.from_string(self.uri).get_storage_index() si_dir = os.path.join(shares_dir, storage_index_to_dir(si)) if not os.path.exists(si_dir): @@ -82,7 +82,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin, shutil.copy(sharefile, new_sharefile) self.shares = self.find_uri_shares(self.uri) # Make sure that the storage server has the share. - self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile) + self.failUnless((sharenum, ss.original._server.my_nodeid, new_sharefile) in self.shares) def _corrupt_share(self, share, corruptor_func): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 442e154a0..6504ac97f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -65,5 +65,5 @@ class HTTPTests(TestCase): The client can return the version. """ version = yield self.client.get_version() - expected_version = self.storage_server.remote_get_version() + expected_version = self.storage_server.get_version() self.assertEqual(version, expected_version) From 5bb6fbc51f4d1d1d871410aba2cc91a09a2bb3ab Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 4 Dec 2021 10:14:31 -0700 Subject: [PATCH 275/916] merge errors --- src/allmydata/storage/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index acca83d6a..ac8c41c07 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -337,7 +337,7 @@ class StorageServer(service.MultiService, Referenceable): alreadygot[shnum] = ShareFile(fn) if renew_leases: sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + sf.add_or_renew_lease(remaining_space, lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -411,7 +411,7 @@ class StorageServer(service.MultiService, Referenceable): renew_secret, cancel_secret, new_expire_time, self.my_nodeid) for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(lease_info) + sf.add_or_renew_lease(self.get_available_space(), lease_info) self.add_latency("add-lease", self._clock.seconds() - start) return None From 50cdd9bd9659ad9886de0ca021b34ef3028f411d Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 4 Dec 2021 17:20:10 -0700 Subject: [PATCH 276/916] unused --- src/allmydata/storage/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ac8c41c07..9a9b3e624 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -15,7 +15,6 @@ else: from typing import Dict import os, re -import six from foolscap.api import Referenceable from foolscap.ipb import IRemoteReference From 402d11ecd61cd821b0d6afe8f492253106747759 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:39:31 -0700 Subject: [PATCH 277/916] update NEWS.txt for release --- NEWS.rst | 75 ++++++++++++++++++++++++++++++++ newsfragments/3525.minor | 0 newsfragments/3527.minor | 0 newsfragments/3735.feature | 1 - newsfragments/3754.minor | 0 newsfragments/3758.minor | 0 newsfragments/3784.minor | 0 newsfragments/3786.feature | 1 - newsfragments/3792.minor | 0 newsfragments/3793.minor | 0 newsfragments/3795.minor | 0 newsfragments/3797.minor | 0 newsfragments/3798.minor | 0 newsfragments/3799.minor | 0 newsfragments/3800.minor | 0 newsfragments/3801.bugfix | 1 - newsfragments/3805.minor | 0 newsfragments/3806.minor | 0 newsfragments/3807.feature | 1 - newsfragments/3808.installation | 1 - newsfragments/3810.minor | 0 newsfragments/3812.minor | 0 newsfragments/3814.removed | 1 - newsfragments/3815.documentation | 1 - newsfragments/3819.security | 1 - newsfragments/3820.minor | 0 newsfragments/3821.security | 2 - newsfragments/3822.security | 2 - newsfragments/3823.security | 4 -- newsfragments/3824.security | 1 - newsfragments/3825.security | 8 ---- newsfragments/3827.security | 4 -- newsfragments/3829.minor | 0 newsfragments/3830.minor | 0 newsfragments/3831.minor | 0 newsfragments/3832.minor | 0 newsfragments/3833.minor | 0 newsfragments/3834.minor | 0 newsfragments/3835.minor | 0 newsfragments/3836.minor | 0 newsfragments/3837.other | 1 - newsfragments/3838.minor | 0 newsfragments/3839.security | 1 - newsfragments/3841.security | 1 - newsfragments/3842.minor | 0 newsfragments/3843.minor | 0 newsfragments/3847.minor | 0 47 files changed, 75 insertions(+), 32 deletions(-) delete mode 100644 newsfragments/3525.minor delete mode 100644 newsfragments/3527.minor delete mode 100644 newsfragments/3735.feature delete mode 100644 newsfragments/3754.minor delete mode 100644 newsfragments/3758.minor delete mode 100644 newsfragments/3784.minor delete mode 100644 newsfragments/3786.feature delete mode 100644 newsfragments/3792.minor delete mode 100644 newsfragments/3793.minor delete mode 100644 newsfragments/3795.minor delete mode 100644 newsfragments/3797.minor delete mode 100644 newsfragments/3798.minor delete mode 100644 newsfragments/3799.minor delete mode 100644 newsfragments/3800.minor delete mode 100644 newsfragments/3801.bugfix delete mode 100644 newsfragments/3805.minor delete mode 100644 newsfragments/3806.minor delete mode 100644 newsfragments/3807.feature delete mode 100644 newsfragments/3808.installation delete mode 100644 newsfragments/3810.minor delete mode 100644 newsfragments/3812.minor delete mode 100644 newsfragments/3814.removed delete mode 100644 newsfragments/3815.documentation delete mode 100644 newsfragments/3819.security delete mode 100644 newsfragments/3820.minor delete mode 100644 newsfragments/3821.security delete mode 100644 newsfragments/3822.security delete mode 100644 newsfragments/3823.security delete mode 100644 newsfragments/3824.security delete mode 100644 newsfragments/3825.security delete mode 100644 newsfragments/3827.security delete mode 100644 newsfragments/3829.minor delete mode 100644 newsfragments/3830.minor delete mode 100644 newsfragments/3831.minor delete mode 100644 newsfragments/3832.minor delete mode 100644 newsfragments/3833.minor delete mode 100644 newsfragments/3834.minor delete mode 100644 newsfragments/3835.minor delete mode 100644 newsfragments/3836.minor delete mode 100644 newsfragments/3837.other delete mode 100644 newsfragments/3838.minor delete mode 100644 newsfragments/3839.security delete mode 100644 newsfragments/3841.security delete mode 100644 newsfragments/3842.minor delete mode 100644 newsfragments/3843.minor delete mode 100644 newsfragments/3847.minor diff --git a/NEWS.rst b/NEWS.rst index e4fef833a..697c44c30 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,81 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.16.0.post463 (2021-12-05)Release 1.16.0.post463 (2021-12-05) +''''''''''''''''''''''''''''''''''' + +Security-related Changes +------------------------ + +- The introducer server no longer writes the sensitive introducer fURL value to its log at startup time. Instead it writes the well-known path of the file from which this value can be read. (`#3819 `_) +- The storage protocol operation ``add_lease`` now safely rejects an attempt to add a 4,294,967,296th lease to an immutable share. + Previously this failed with an error after recording the new lease in the share file, resulting in the share file losing track of a one previous lease. (`#3821 `_) +- The storage protocol operation ``readv`` now safely rejects attempts to read negative lengths. + Previously these read requests were satisfied with the complete contents of the share file (including trailing metadata) starting from the specified offset. (`#3822 `_) +- The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information and recording corruption advisories. + Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. + Now this operation will fail with an exception and the lease will not be created. + Similarly, if there is no space available, corruption advisories will be logged but not written to disk. (`#3823 `_) +- The storage server implementation no longer records corruption advisories about storage indexes for which it holds no shares. (`#3824 `_) +- The lease-checker now uses JSON instead of pickle to serialize its state. + + tahoe will now refuse to run until you either delete all pickle files or + migrate them using the new command:: + + tahoe admin migrate-crawler + + This will migrate all crawler-related pickle files. (`#3825 `_) +- The SFTP server no longer accepts password-based credentials for authentication. + Public/private key-based credentials are now the only supported authentication type. + This removes plaintext password storage from the SFTP credentials file. + It also removes a possible timing side-channel vulnerability which might have allowed attackers to discover an account's plaintext password. (`#3827 `_) +- The storage server now keeps hashes of lease renew and cancel secrets for immutable share files instead of keeping the original secrets. (`#3839 `_) +- The storage server now keeps hashes of lease renew and cancel secrets for mutable share files instead of keeping the original secrets. (`#3841 `_) + + +Features +-------- + +- Tahoe-LAFS releases now have just a .tar.gz source release and a (universal) wheel (`#3735 `_) +- tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) at `/statistics?t=openmetrics`. (`#3786 `_) +- If uploading an immutable hasn't had a write for 30 minutes, the storage server will abort the upload. (`#3807 `_) + + +Bug Fixes +--------- + +- When uploading an immutable, overlapping writes that include conflicting data are rejected. In practice, this likely didn't happen in real-world usage. (`#3801 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS now supports running on NixOS 21.05 with Python 3. (`#3808 `_) + + +Documentation Changes +--------------------- + +- The news file for future releases will include a section for changes with a security impact. (`#3815 `_) + + +Removed Features +---------------- + +- The little-used "control port" has been removed from all node types. (`#3814 `_) + + +Other Changes +------------- + +- Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies. (`#3837 `_) + + +Misc/Other +---------- + +- `#3525 `_, `#3527 `_, `#3754 `_, `#3758 `_, `#3784 `_, `#3792 `_, `#3793 `_, `#3795 `_, `#3797 `_, `#3798 `_, `#3799 `_, `#3800 `_, `#3805 `_, `#3806 `_, `#3810 `_, `#3812 `_, `#3820 `_, `#3829 `_, `#3830 `_, `#3831 `_, `#3832 `_, `#3833 `_, `#3834 `_, `#3835 `_, `#3836 `_, `#3838 `_, `#3842 `_, `#3843 `_, `#3847 `_ + Release 1.16.0 (2021-09-17) ''''''''''''''''''''''''''' diff --git a/newsfragments/3525.minor b/newsfragments/3525.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3527.minor b/newsfragments/3527.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3735.feature b/newsfragments/3735.feature deleted file mode 100644 index 5a86d5547..000000000 --- a/newsfragments/3735.feature +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS releases now have just a .tar.gz source release and a (universal) wheel diff --git a/newsfragments/3754.minor b/newsfragments/3754.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3758.minor b/newsfragments/3758.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3784.minor b/newsfragments/3784.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3786.feature b/newsfragments/3786.feature deleted file mode 100644 index ecbfc0372..000000000 --- a/newsfragments/3786.feature +++ /dev/null @@ -1 +0,0 @@ -tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) at `/statistics?t=openmetrics`. diff --git a/newsfragments/3792.minor b/newsfragments/3792.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3793.minor b/newsfragments/3793.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3795.minor b/newsfragments/3795.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3797.minor b/newsfragments/3797.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3798.minor b/newsfragments/3798.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3799.minor b/newsfragments/3799.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3800.minor b/newsfragments/3800.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3801.bugfix b/newsfragments/3801.bugfix deleted file mode 100644 index 504b3999d..000000000 --- a/newsfragments/3801.bugfix +++ /dev/null @@ -1 +0,0 @@ -When uploading an immutable, overlapping writes that include conflicting data are rejected. In practice, this likely didn't happen in real-world usage. \ No newline at end of file diff --git a/newsfragments/3805.minor b/newsfragments/3805.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3806.minor b/newsfragments/3806.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3807.feature b/newsfragments/3807.feature deleted file mode 100644 index f82363ffd..000000000 --- a/newsfragments/3807.feature +++ /dev/null @@ -1 +0,0 @@ -If uploading an immutable hasn't had a write for 30 minutes, the storage server will abort the upload. \ No newline at end of file diff --git a/newsfragments/3808.installation b/newsfragments/3808.installation deleted file mode 100644 index 157f08a0c..000000000 --- a/newsfragments/3808.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now supports running on NixOS 21.05 with Python 3. diff --git a/newsfragments/3810.minor b/newsfragments/3810.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3812.minor b/newsfragments/3812.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3814.removed b/newsfragments/3814.removed deleted file mode 100644 index 939d20ffc..000000000 --- a/newsfragments/3814.removed +++ /dev/null @@ -1 +0,0 @@ -The little-used "control port" has been removed from all node types. diff --git a/newsfragments/3815.documentation b/newsfragments/3815.documentation deleted file mode 100644 index 7abc70bd1..000000000 --- a/newsfragments/3815.documentation +++ /dev/null @@ -1 +0,0 @@ -The news file for future releases will include a section for changes with a security impact. \ No newline at end of file diff --git a/newsfragments/3819.security b/newsfragments/3819.security deleted file mode 100644 index 975fd0035..000000000 --- a/newsfragments/3819.security +++ /dev/null @@ -1 +0,0 @@ -The introducer server no longer writes the sensitive introducer fURL value to its log at startup time. Instead it writes the well-known path of the file from which this value can be read. diff --git a/newsfragments/3820.minor b/newsfragments/3820.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3821.security b/newsfragments/3821.security deleted file mode 100644 index 75d9904a2..000000000 --- a/newsfragments/3821.security +++ /dev/null @@ -1,2 +0,0 @@ -The storage protocol operation ``add_lease`` now safely rejects an attempt to add a 4,294,967,296th lease to an immutable share. -Previously this failed with an error after recording the new lease in the share file, resulting in the share file losing track of a one previous lease. diff --git a/newsfragments/3822.security b/newsfragments/3822.security deleted file mode 100644 index 5d6c07ab5..000000000 --- a/newsfragments/3822.security +++ /dev/null @@ -1,2 +0,0 @@ -The storage protocol operation ``readv`` now safely rejects attempts to read negative lengths. -Previously these read requests were satisfied with the complete contents of the share file (including trailing metadata) starting from the specified offset. diff --git a/newsfragments/3823.security b/newsfragments/3823.security deleted file mode 100644 index ba2bbd741..000000000 --- a/newsfragments/3823.security +++ /dev/null @@ -1,4 +0,0 @@ -The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information and recording corruption advisories. -Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. -Now this operation will fail with an exception and the lease will not be created. -Similarly, if there is no space available, corruption advisories will be logged but not written to disk. diff --git a/newsfragments/3824.security b/newsfragments/3824.security deleted file mode 100644 index b29b2acc8..000000000 --- a/newsfragments/3824.security +++ /dev/null @@ -1 +0,0 @@ -The storage server implementation no longer records corruption advisories about storage indexes for which it holds no shares. diff --git a/newsfragments/3825.security b/newsfragments/3825.security deleted file mode 100644 index 3d112dd49..000000000 --- a/newsfragments/3825.security +++ /dev/null @@ -1,8 +0,0 @@ -The lease-checker now uses JSON instead of pickle to serialize its state. - -tahoe will now refuse to run until you either delete all pickle files or -migrate them using the new command:: - - tahoe admin migrate-crawler - -This will migrate all crawler-related pickle files. diff --git a/newsfragments/3827.security b/newsfragments/3827.security deleted file mode 100644 index 4fee19c76..000000000 --- a/newsfragments/3827.security +++ /dev/null @@ -1,4 +0,0 @@ -The SFTP server no longer accepts password-based credentials for authentication. -Public/private key-based credentials are now the only supported authentication type. -This removes plaintext password storage from the SFTP credentials file. -It also removes a possible timing side-channel vulnerability which might have allowed attackers to discover an account's plaintext password. diff --git a/newsfragments/3829.minor b/newsfragments/3829.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3830.minor b/newsfragments/3830.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3831.minor b/newsfragments/3831.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3832.minor b/newsfragments/3832.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3833.minor b/newsfragments/3833.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3834.minor b/newsfragments/3834.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3835.minor b/newsfragments/3835.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3836.minor b/newsfragments/3836.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3837.other b/newsfragments/3837.other deleted file mode 100644 index a9e4e6986..000000000 --- a/newsfragments/3837.other +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies. diff --git a/newsfragments/3838.minor b/newsfragments/3838.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3839.security b/newsfragments/3839.security deleted file mode 100644 index 1ae054542..000000000 --- a/newsfragments/3839.security +++ /dev/null @@ -1 +0,0 @@ -The storage server now keeps hashes of lease renew and cancel secrets for immutable share files instead of keeping the original secrets. diff --git a/newsfragments/3841.security b/newsfragments/3841.security deleted file mode 100644 index 867322e0a..000000000 --- a/newsfragments/3841.security +++ /dev/null @@ -1 +0,0 @@ -The storage server now keeps hashes of lease renew and cancel secrets for mutable share files instead of keeping the original secrets. \ No newline at end of file diff --git a/newsfragments/3842.minor b/newsfragments/3842.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3843.minor b/newsfragments/3843.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3847.minor b/newsfragments/3847.minor deleted file mode 100644 index e69de29bb..000000000 From b8d00ab04a1ae6309d3dd5cf93b937d759f3c9d6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:50:22 -0700 Subject: [PATCH 278/916] update release notes --- NEWS.rst | 6 ++++-- relnotes.txt | 39 ++++++++++++++++----------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 697c44c30..15cb9459d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,10 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.16.0.post463 (2021-12-05)Release 1.16.0.post463 (2021-12-05) -''''''''''''''''''''''''''''''''''' + + +Release 1.17.0 (2021-12-06) +''''''''''''''''''''''''''' Security-related Changes ------------------------ diff --git a/relnotes.txt b/relnotes.txt index 2748bc4fa..dff4f192e 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.16.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.0 -The Tahoe-LAFS team is pleased to announce version 1.16.0 of +The Tahoe-LAFS team is pleased to announce version 1.17.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,24 +15,17 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.15.1, released on -March 23rd, 2021. +The previous stable release of Tahoe-LAFS was v1.16.0, released on +October 19, 2021. -The major change in this release is the completion of the Python 3 -port -- while maintaining support for Python 2. A future release will -remove Python 2 support. +This release fixes several security issues raised as part of an audit +by Cure53. We developed fixes for these issues in a private +repository. Shortly after this release, public tickets will be updated +with further information (along with, of course, all the code). -The previously deprecated subcommands "start", "stop", "restart" and -"daemonize" have been removed. You must now use "tahoe run" (possibly -along with your favourite daemonization software). +There is also OpenMetrics support now and several bug fixes. -Several features are now removed: the Account Server, stats-gatherer -and FTP support. - -There are several dependency changes that will be interesting for -distribution maintainers. - -In all, 240 issues have been fixed since the last release. +In all, 46 issues have been fixed since the last release. Please see ``NEWS.rst`` for a more complete list of changes. @@ -151,19 +144,19 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -fenn-cs + meejah +meejah on behalf of the Tahoe-LAFS team -October 19, 2021 +December 6, 2021 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.16.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 95fdaf286e2f195672d4f2cd3371ed41ee49aae1 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:51:13 -0700 Subject: [PATCH 279/916] update nix version --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 59864d36d..04d6c4163 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.16.0). + # Most of the time this is not exactly the release version (eg 1.17.0). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.16.0.post1"; + version = "1.17.0.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From a8bdb8dcbb66bfa75389816817e25b6f9ec5d74b Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:53:50 -0700 Subject: [PATCH 280/916] add Florian --- CREDITS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CREDITS b/CREDITS index 8a6e876ec..89e1468aa 100644 --- a/CREDITS +++ b/CREDITS @@ -260,3 +260,7 @@ D: Community-manager and documentation improvements N: Yash Nayani E: yashaswi.nram@gmail.com D: Installation Guide improvements + +N: Florian Sesser +E: florian@private.storage +D: OpenMetrics support \ No newline at end of file From 5f6579d44622b0774aec8ba06b67f2d4f2ee7af7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 6 Dec 2021 12:47:37 -0500 Subject: [PATCH 281/916] hew closer to security/master version of these lines --- src/allmydata/storage/server.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9a9b3e624..80b337d36 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -334,9 +334,8 @@ class StorageServer(service.MultiService, Referenceable): # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot[shnum] = ShareFile(fn) - if renew_leases: - sf = ShareFile(fn) - sf.add_or_renew_lease(remaining_space, lease_info) + if renew_leases: + self._add_or_renew_leases(alreadygot.values(), lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -409,8 +408,10 @@ class StorageServer(service.MultiService, Referenceable): lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) - for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(self.get_available_space(), lease_info) + self._add_or_renew_leases( + self._iter_share_files(storage_index), + lease_info, + ) self.add_latency("add-lease", self._clock.seconds() - start) return None From 91dd70ad2926d1759402f9ba9214ba21063337e2 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 6 Dec 2021 22:51:44 +0100 Subject: [PATCH 282/916] fixed typo, made version name inline literal Signed-off-by: fenn-cs --- docs/release-checklist.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 796be75ba..8bba63e79 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -116,8 +116,8 @@ they will need to evaluate which contributors' signatures they trust. - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` .. note:: - - Replace the key-id above with your own, which can simply be your email if's attached your fingerprint. - - Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0` + - Replace the key-id above with your own, which can simply be your email if it's attached to your fingerprint. + - Don't forget to put the correct tag message and name. In this example, the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is ``tahoe-lafs-1.16.0rc0`` - build all code locally From 911eb6cf9a4a5eccb39a40bc128364f4778c8e96 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 7 Dec 2021 11:10:51 +0100 Subject: [PATCH 283/916] add gpg-setup doc to toctree Signed-off-by: fenn-cs --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 16067597a..3da03341a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ Contents: contributing CODE_OF_CONDUCT release-checklist + gpg-setup servers helper From b32374c8bcee4120dd89e37fe2472aabf48abce2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 10:39:58 -0500 Subject: [PATCH 284/916] Secret header parsing. --- src/allmydata/storage/http_server.py | 38 +++++++++++++ src/allmydata/test/test_storage_http.py | 73 ++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6297ef484..47722180d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -13,8 +13,12 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +else: + from typing import Dict, List, Set from functools import wraps +from enum import Enum +from base64 import b64decode from klein import Klein from twisted.web import http @@ -26,6 +30,40 @@ from .server import StorageServer from .http_client import swissnum_auth_header +class Secrets(Enum): + """Different kinds of secrets the client may send.""" + LEASE_RENEW = "lease-renew-secret" + LEASE_CANCEL = "lease-cancel-secret" + UPLOAD = "upload-secret" + + +class ClientSecretsException(Exception): + """The client did not send the appropriate secrets.""" + + +def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] + """ + Given list of values of ``X-Tahoe-Authorization`` headers, and required + secrets, return dictionary mapping secrets to decoded values. + + If too few secrets were given, or too many, a ``ClientSecretsException`` is + raised. + """ + key_to_enum = {e.value: e for e in Secrets} + result = {} + try: + for header_value in header_values: + key, value = header_value.strip().split(" ", 1) + result[key_to_enum[key]] = b64decode(value) + except (ValueError, KeyError) as e: + raise ClientSecretsException("Bad header value(s): {}".format(header_values)) + if result.keys() != required_secrets: + raise ClientSecretsException( + "Expected {} secrets, got {}".format(required_secrets, result.keys()) + ) + return result + + def _authorization_decorator(f): """ Check the ``Authorization`` header, and (TODO: in later revision of code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e413a0624..2a84d477f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from unittest import SkipTest +from base64 import b64encode from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks @@ -23,10 +24,80 @@ from treq.testing import StubTreq from hyperlink import DecodedURL from ..storage.server import StorageServer -from ..storage.http_server import HTTPServer +from ..storage.http_server import HTTPServer, _extract_secrets, Secrets, ClientSecretsException from ..storage.http_client import StorageClient, ClientException +class ExtractSecretsTests(TestCase): + """ + Tests for ``_extract_secrets``. + """ + def test_extract_secrets(self): + """ + ``_extract_secrets()`` returns a dictionary with the extracted secrets + if the input secrets match the required secrets. + """ + secret1 = b"\xFF\x11ZEBRa" + secret2 = b"\x34\xF2lalalalalala" + lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() + upload_secret = "upload-secret " + str(b64encode(secret2), "ascii").strip() + + # No secrets needed, none given: + self.assertEqual(_extract_secrets([], set()), {}) + + # One secret: + self.assertEqual( + _extract_secrets([lease_secret], + {Secrets.LEASE_RENEW}), + {Secrets.LEASE_RENEW: secret1} + ) + + # Two secrets: + self.assertEqual( + _extract_secrets([upload_secret, lease_secret], + {Secrets.LEASE_RENEW, Secrets.UPLOAD}), + {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2} + ) + + def test_wrong_number_of_secrets(self): + """ + If the wrong number of secrets are passed to ``_extract_secrets``, a + ``ClientSecretsException`` is raised. + """ + secret1 = b"\xFF\x11ZEBRa" + lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() + + # Missing secret: + with self.assertRaises(ClientSecretsException): + _extract_secrets([], {Secrets.LEASE_RENEW}) + + # Wrong secret: + with self.assertRaises(ClientSecretsException): + _extract_secrets([lease_secret], {Secrets.UPLOAD}) + + # Extra secret: + with self.assertRaises(ClientSecretsException): + _extract_secrets([lease_secret], {}) + + def test_bad_secrets(self): + """ + Bad inputs to ``_extract_secrets`` result in + ``ClientSecretsException``. + """ + + # Missing value. + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-renew-secret"], {Secrets.LEASE_RENEW}) + + # Garbage prefix + with self.assertRaises(ClientSecretsException): + _extract_secrets(["FOO eA=="], {}) + + # Not base64. + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) + + class HTTPTests(TestCase): """ Tests of HTTP client talking to the HTTP server. From 1737340df69ff443c1e548d9126066c05e00bf30 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 10:52:02 -0500 Subject: [PATCH 285/916] Document response codes some more. --- docs/proposed/http-storage-node-protocol.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 490d3f3ca..0d8cee466 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -369,6 +369,19 @@ The authentication *type* used is ``Tahoe-LAFS``. The swissnum from the NURL used to locate the storage service is used as the *credentials*. If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response. +There are also, for some endpoints, secrets sent via ``X-Tahoe-Authorization`` headers. +If these are: + +1. Missing. +2. The wrong length. +3. Not the expected kind of secret. +4. They are otherwise unparseable before they are actually semantically used. + +the server will respond with ``400 BAD REQUEST``. +401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug. + +If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. + General ~~~~~~~ From 87fa9ac2a8e507b385c4b0845c50d34cf6d30da6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:17:11 -0500 Subject: [PATCH 286/916] Infrastructure for sending secrets. --- src/allmydata/storage/http_client.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f8a7590aa..774ecbde1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -19,7 +19,7 @@ else: from typing import Union from treq.testing import StubTreq -import base64 +from base64 import b64encode # TODO Make sure to import Python version? from cbor2 import loads @@ -44,7 +44,7 @@ def _decode_cbor(response): def swissnum_auth_header(swissnum): # type: (bytes) -> bytes """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + base64.b64encode(swissnum).strip() + return b"Tahoe-LAFS " + b64encode(swissnum).strip() class StorageClient(object): @@ -68,12 +68,25 @@ class StorageClient(object): ) return headers + def _request(self, method, url, secrets, **kwargs): + """ + Like ``treq.request()``, but additional argument of secrets mapping + ``http_server.Secret`` to the bytes value of the secret. + """ + headers = self._get_headers() + for key, value in secrets.items(): + headers.addRawHeader( + "X-Tahoe-Authorization", + b"{} {}".format(key.value.encode("ascii"), b64encode(value).strip()) + ) + return self._treq.request(method, url, headers=headers, **kwargs) + @inlineCallbacks def get_version(self): """ Return the version metadata for the server. """ url = self._base_url.click("/v1/version") - response = yield self._treq.get(url, headers=self._get_headers()) + response = yield self._request("GET", url, {}) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) From da52a9aedeebe39c908f1616bac6ce887d340066 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:17:32 -0500 Subject: [PATCH 287/916] Test for server-side secret handling. --- src/allmydata/test/test_storage_http.py | 88 +++++++++++++++++++++---- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2a84d477f..648530852 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -21,10 +21,14 @@ from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks from treq.testing import StubTreq +from klein import Klein from hyperlink import DecodedURL from ..storage.server import StorageServer -from ..storage.http_server import HTTPServer, _extract_secrets, Secrets, ClientSecretsException +from ..storage.http_server import ( + HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + _authorized_route, +) from ..storage.http_client import StorageClient, ClientException @@ -98,22 +102,80 @@ class ExtractSecretsTests(TestCase): _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) -class HTTPTests(TestCase): +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"abc"}: + return "OK" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(TestCase): """ - Tests of HTTP client talking to the HTTP server. + Tests for the HTTP routing infrastructure. + """ + def setUp(self): + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + secret = b"abc" + + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {} + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD, b"abc"} + ) + self.assertEqual(response.code, 200) + + +def setup_http_test(self): + """ + Setup HTTP tests; call from ``setUp``. + """ + if PY2: + raise SkipTest("Not going to bother supporting Python 2") + self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) + # TODO what should the swissnum _actually_ be? + self._http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server.get_resource()), + ) + + +class GenericHTTPAPITests(TestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. """ def setUp(self): - if PY2: - raise SkipTest("Not going to bother supporting Python 2") - self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self._http_server = HTTPServer(self.storage_server, b"abcd") - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"abcd", - treq=StubTreq(self._http_server.get_resource()), - ) + setup_http_test(self) @inlineCallbacks def test_bad_authentication(self): From 816dc0c73ff3ed7522c63198c6659c17e39f7837 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:42:06 -0500 Subject: [PATCH 288/916] X-Tahoe-Authorization can be validated and are passed to server methods. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 51 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 774ecbde1..72e1af080 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -77,7 +77,7 @@ class StorageClient(object): for key, value in secrets.items(): headers.addRawHeader( "X-Tahoe-Authorization", - b"{} {}".format(key.value.encode("ascii"), b64encode(value).strip()) + b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()) ) return self._treq.request(method, url, headers=headers, **kwargs) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 47722180d..a26a2c266 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -54,8 +54,10 @@ def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[ try: for header_value in header_values: key, value = header_value.strip().split(" ", 1) + # TODO enforce secret is 32 bytes long for lease secrets. dunno + # about upload secret. result[key_to_enum[key]] = b64decode(value) - except (ValueError, KeyError) as e: + except (ValueError, KeyError): raise ClientSecretsException("Bad header value(s): {}".format(header_values)) if result.keys() != required_secrets: raise ClientSecretsException( @@ -64,38 +66,45 @@ def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[ return result -def _authorization_decorator(f): +def _authorization_decorator(required_secrets): """ Check the ``Authorization`` header, and (TODO: in later revision of code) extract ``X-Tahoe-Authorization`` headers and pass them in. """ + def decorator(f): + @wraps(f) + def route(self, request, *args, **kwargs): + if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( + swissnum_auth_header(self._swissnum), "ascii" + ): + request.setResponseCode(http.UNAUTHORIZED) + return b"" + authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) + try: + secrets = _extract_secrets(authorization, required_secrets) + except ClientSecretsException: + request.setResponseCode(400) + return b"" + return f(self, request, secrets, *args, **kwargs) - @wraps(f) - def route(self, request, *args, **kwargs): - if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( - swissnum_auth_header(self._swissnum), "ascii" - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" - # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) - # For now, just a placeholder: - authorization = None - return f(self, request, authorization, *args, **kwargs) + return route - return route + return decorator -def _authorized_route(app, *route_args, **route_kwargs): +def _authorized_route(app, required_secrets, *route_args, **route_kwargs): """ Like Klein's @route, but with additional support for checking the ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The - latter will (TODO: in later revision of code) get passed in as second - argument to wrapped functions. + latter will get passed in as second argument to wrapped functions, a + dictionary mapping a ``Secret`` value to the uploaded secret. + + :param required_secrets: Set of required ``Secret`` types. """ def decorator(f): @app.route(*route_args, **route_kwargs) - @_authorization_decorator + @_authorization_decorator(required_secrets) def handle_route(*args, **kwargs): return f(*args, **kwargs) @@ -127,6 +136,10 @@ class HTTPServer(object): # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) - @_authorized_route(_app, "/v1/version", methods=["GET"]) + + ##### Generic APIs ##### + + @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) def version(self, request, authorization): + """Return version information.""" return self._cbor(request, self._storage_server.get_version()) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 648530852..b28f4aafb 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -147,7 +147,7 @@ class RoutingTests(TestCase): # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD, b"abc"} + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"abc"} ) self.assertEqual(response.code, 200) From d2ce80dab88df8431a2188ae53bd40f2debe5ee4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:42:44 -0500 Subject: [PATCH 289/916] News file. --- newsfragments/3848.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3848.minor diff --git a/newsfragments/3848.minor b/newsfragments/3848.minor new file mode 100644 index 000000000..e69de29bb From fb0be6b8944dcf9b1254e5d3991ddd8d2ff6ad3c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:46:35 -0500 Subject: [PATCH 290/916] Enforce length of lease secrets. --- src/allmydata/storage/http_server.py | 12 +++++++----- src/allmydata/test/test_storage_http.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a26a2c266..1143acce9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -49,14 +49,16 @@ def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[ If too few secrets were given, or too many, a ``ClientSecretsException`` is raised. """ - key_to_enum = {e.value: e for e in Secrets} + string_key_to_enum = {e.value: e for e in Secrets} result = {} try: for header_value in header_values: - key, value = header_value.strip().split(" ", 1) - # TODO enforce secret is 32 bytes long for lease secrets. dunno - # about upload secret. - result[key_to_enum[key]] = b64decode(value) + string_key, string_value = header_value.strip().split(" ", 1) + key = string_key_to_enum[string_key] + value = b64decode(string_value) + if key in (Secrets.LEASE_CANCEL, Secrets.LEASE_RENEW) and len(value) != 32: + raise ClientSecretsException("Lease secrets must be 32 bytes long") + result[key] = value except (ValueError, KeyError): raise ClientSecretsException("Bad header value(s): {}".format(header_values)) if result.keys() != required_secrets: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b28f4aafb..9d4ef3738 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -41,8 +41,8 @@ class ExtractSecretsTests(TestCase): ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secret1 = b"\xFF\x11ZEBRa" - secret2 = b"\x34\xF2lalalalalala" + secret1 = b"\xFF" * 32 + secret2 = b"\x34" * 32 lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() upload_secret = "upload-secret " + str(b64encode(secret2), "ascii").strip() @@ -101,6 +101,12 @@ class ExtractSecretsTests(TestCase): with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) + # Wrong length lease secrets (must be 32 bytes long). + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-renew-secret eA=="], {Secrets.LEASE_RENEW}) + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-upload-secret eA=="], {Secrets.LEASE_RENEW}) + SWISSNUM_FOR_TEST = b"abcd" From 428a9d0573628fdc91c5efd7fc3a2f94bbdd19bf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:47:40 -0500 Subject: [PATCH 291/916] Lint fix. --- src/allmydata/test/test_storage_http.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 9d4ef3738..b9aa59f3e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -143,8 +143,6 @@ class RoutingTests(TestCase): The requirement for secrets is enforced; if they are not given, a 400 response code is returned. """ - secret = b"abc" - # Without secret, get a 400 error. response = yield self.client._request( "GET", "http://127.0.0.1/upload_secret", {} From 81b95f3335c7178fb64cb131d4423780cf04cd29 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:53:31 -0500 Subject: [PATCH 292/916] Ensure secret was validated. --- src/allmydata/test/test_storage_http.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b9aa59f3e..16420b266 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -119,8 +119,8 @@ class TestApp(object): @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"abc"}: - return "OK" + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" else: return "BAD: {}".format(authorization) @@ -151,9 +151,10 @@ class RoutingTests(TestCase): # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"abc"} + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} ) self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") def setup_http_test(self): From a529ba7d5ea84c2b9f4cef48f6e5ef778cb5fbbc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Dec 2021 09:14:09 -0500 Subject: [PATCH 293/916] More skipping on Python 2. --- src/allmydata/test/test_storage_http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 16420b266..aba33fad3 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -36,6 +36,10 @@ class ExtractSecretsTests(TestCase): """ Tests for ``_extract_secrets``. """ + def setUp(self): + if PY2: + raise SkipTest("Not going to bother supporting Python 2") + def test_extract_secrets(self): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets @@ -130,6 +134,8 @@ class RoutingTests(TestCase): Tests for the HTTP routing infrastructure. """ def setUp(self): + if PY2: + raise SkipTest("Not going to bother supporting Python 2") self._http_server = TestApp() self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), From 291b4e1896f52a07fbe92df7b67f90372ca0f052 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Dec 2021 11:17:27 -0500 Subject: [PATCH 294/916] Use more secure comparison to prevent timing-based side-channel attacks. --- src/allmydata/storage/http_server.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1143acce9..386368364 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -28,10 +28,12 @@ from cbor2 import dumps from .server import StorageServer from .http_client import swissnum_auth_header +from ..util.hashutil import timing_safe_compare class Secrets(Enum): """Different kinds of secrets the client may send.""" + LEASE_RENEW = "lease-renew-secret" LEASE_CANCEL = "lease-cancel-secret" UPLOAD = "upload-secret" @@ -41,7 +43,9 @@ class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" -def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] +def _extract_secrets( + header_values, required_secrets +): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] """ Given list of values of ``X-Tahoe-Authorization`` headers, and required secrets, return dictionary mapping secrets to decoded values. @@ -73,15 +77,21 @@ def _authorization_decorator(required_secrets): Check the ``Authorization`` header, and (TODO: in later revision of code) extract ``X-Tahoe-Authorization`` headers and pass them in. """ + def decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( - swissnum_auth_header(self._swissnum), "ascii" + if not timing_safe_compare( + request.requestHeaders.getRawHeaders("Authorization", [None])[0].encode( + "utf-8" + ), + swissnum_auth_header(self._swissnum), ): request.setResponseCode(http.UNAUTHORIZED) return b"" - authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) + authorization = request.requestHeaders.getRawHeaders( + "X-Tahoe-Authorization", [] + ) try: secrets = _extract_secrets(authorization, required_secrets) except ClientSecretsException: @@ -138,7 +148,6 @@ class HTTPServer(object): # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) - ##### Generic APIs ##### @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) From 1721865b20d0d160891f91365dc477fae26ffcb0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Dec 2021 13:46:19 -0500 Subject: [PATCH 295/916] No longer TODO. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 386368364..83bbbe49d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -74,8 +74,8 @@ def _extract_secrets( def _authorization_decorator(required_secrets): """ - Check the ``Authorization`` header, and (TODO: in later revision of code) - extract ``X-Tahoe-Authorization`` headers and pass them in. + Check the ``Authorization`` header, and extract ``X-Tahoe-Authorization`` + headers and pass them in. """ def decorator(f): From 2bda2a01278d1aba5f3a13c772c9c7b887119bdd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Dec 2021 11:10:53 -0500 Subject: [PATCH 296/916] Switch to using a fixture. --- src/allmydata/test/test_storage_http.py | 77 +++++++++++++++---------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index aba33fad3..1cf650875 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -17,28 +17,34 @@ if PY2: from unittest import SkipTest from base64 import b64encode -from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks +from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL +from .common import AsyncTestCase, SyncTestCase from ..storage.server import StorageServer from ..storage.http_server import ( - HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + HTTPServer, + _extract_secrets, + Secrets, + ClientSecretsException, _authorized_route, ) from ..storage.http_client import StorageClient, ClientException -class ExtractSecretsTests(TestCase): +class ExtractSecretsTests(SyncTestCase): """ Tests for ``_extract_secrets``. """ + def setUp(self): if PY2: raise SkipTest("Not going to bother supporting Python 2") + super(ExtractSecretsTests, self).setUp() def test_extract_secrets(self): """ @@ -55,16 +61,16 @@ class ExtractSecretsTests(TestCase): # One secret: self.assertEqual( - _extract_secrets([lease_secret], - {Secrets.LEASE_RENEW}), - {Secrets.LEASE_RENEW: secret1} + _extract_secrets([lease_secret], {Secrets.LEASE_RENEW}), + {Secrets.LEASE_RENEW: secret1}, ) # Two secrets: self.assertEqual( - _extract_secrets([upload_secret, lease_secret], - {Secrets.LEASE_RENEW, Secrets.UPLOAD}), - {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2} + _extract_secrets( + [upload_secret, lease_secret], {Secrets.LEASE_RENEW, Secrets.UPLOAD} + ), + {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2}, ) def test_wrong_number_of_secrets(self): @@ -129,19 +135,23 @@ class TestApp(object): return "BAD: {}".format(authorization) -class RoutingTests(TestCase): +class RoutingTests(AsyncTestCase): """ Tests for the HTTP routing infrastructure. """ + def setUp(self): if PY2: raise SkipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: self._http_server = TestApp() self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) @inlineCallbacks def test_authorization_enforcement(self): @@ -163,30 +173,35 @@ class RoutingTests(TestCase): self.assertEqual((yield response.content()), b"GOOD SECRET") -def setup_http_test(self): +class HttpTestFixture(Fixture): """ - Setup HTTP tests; call from ``setUp``. + Setup HTTP tests' infrastructure, the storage server and corresponding + client. """ - if PY2: - raise SkipTest("Not going to bother supporting Python 2") - self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self._http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server.get_resource()), - ) + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + # TODO what should the swissnum _actually_ be? + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) -class GenericHTTPAPITests(TestCase): +class GenericHTTPAPITests(AsyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API endpoints and concerns. """ def setUp(self): - setup_http_test(self) + if PY2: + raise SkipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) @inlineCallbacks def test_bad_authentication(self): @@ -197,7 +212,7 @@ class GenericHTTPAPITests(TestCase): client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), b"something wrong", - treq=StubTreq(self._http_server.get_resource()), + treq=StubTreq(self.http.http_server.get_resource()), ) with self.assertRaises(ClientException) as e: yield client.get_version() @@ -211,14 +226,14 @@ class GenericHTTPAPITests(TestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = yield self.client.get_version() + version = yield self.http.client.get_version() version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"maximum-immutable-share-size" ) - expected_version = self.storage_server.get_version() + expected_version = self.http.storage_server.get_version() expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) From b1f4e82adfc1d2ff8482b1edb28e0139e7ef0000 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Dec 2021 11:55:16 -0500 Subject: [PATCH 297/916] Switch to using hypothesis. --- src/allmydata/test/test_storage_http.py | 31 +++++++++---------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 1cf650875..2a3a5bc90 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -19,6 +19,7 @@ from base64 import b64encode from twisted.internet.defer import inlineCallbacks +from hypothesis import given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein @@ -46,32 +47,22 @@ class ExtractSecretsTests(SyncTestCase): raise SkipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() - def test_extract_secrets(self): + @given(secret_types=st.sets(st.sampled_from(Secrets))) + def test_extract_secrets(self, secret_types): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secret1 = b"\xFF" * 32 - secret2 = b"\x34" * 32 - lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() - upload_secret = "upload-secret " + str(b64encode(secret2), "ascii").strip() + secrets = {s: bytes([i] * 32) for (i, s) in enumerate(secret_types)} + headers = [ + "{} {}".format( + secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() + ) + for secret_type in secret_types + ] # No secrets needed, none given: - self.assertEqual(_extract_secrets([], set()), {}) - - # One secret: - self.assertEqual( - _extract_secrets([lease_secret], {Secrets.LEASE_RENEW}), - {Secrets.LEASE_RENEW: secret1}, - ) - - # Two secrets: - self.assertEqual( - _extract_secrets( - [upload_secret, lease_secret], {Secrets.LEASE_RENEW, Secrets.UPLOAD} - ), - {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2}, - ) + self.assertEqual(_extract_secrets(headers, secret_types), secrets) def test_wrong_number_of_secrets(self): """ From 776f19cbb2f96607f583dd53ce3c90cbc1353ea8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Dec 2021 12:34:02 -0500 Subject: [PATCH 298/916] Even more hypothesis, this time for secrets' contents. --- src/allmydata/test/test_storage_http.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2a3a5bc90..3dc6bac96 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -47,13 +47,25 @@ class ExtractSecretsTests(SyncTestCase): raise SkipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() - @given(secret_types=st.sets(st.sampled_from(Secrets))) - def test_extract_secrets(self, secret_types): + @given( + params=st.sets(st.sampled_from(Secrets)).flatmap( + lambda secret_types: st.tuples( + st.just(secret_types), + st.lists( + st.binary(min_size=32, max_size=32), + min_size=len(secret_types), + max_size=len(secret_types), + ), + ) + ) + ) + def test_extract_secrets(self, params): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secrets = {s: bytes([i] * 32) for (i, s) in enumerate(secret_types)} + secret_types, secrets = params + secrets = {t: s for (t, s) in zip(secret_types, secrets)} headers = [ "{} {}".format( secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() From 8b4d166a54ebc13c43743cd4263612234f6c68d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Dec 2021 11:44:45 -0500 Subject: [PATCH 299/916] Use hypothesis for another test. --- src/allmydata/test/test_storage_http.py | 78 +++++++++++++------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 3dc6bac96..80bd2661b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -19,7 +19,7 @@ from base64 import b64encode from twisted.internet.defer import inlineCallbacks -from hypothesis import given, strategies as st +from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein @@ -37,6 +37,35 @@ from ..storage.http_server import ( from ..storage.http_client import StorageClient, ClientException +def _post_process(params): + secret_types, secrets = params + secrets = {t: s for (t, s) in zip(secret_types, secrets)} + headers = [ + "{} {}".format( + secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() + ) + for secret_type in secret_types + ] + return secrets, headers + + +# Creates a tuple of ({Secret enum value: secret_bytes}, [http headers with secrets]). +SECRETS_STRATEGY = ( + st.sets(st.sampled_from(Secrets)) + .flatmap( + lambda secret_types: st.tuples( + st.just(secret_types), + st.lists( + st.binary(min_size=32, max_size=32), + min_size=len(secret_types), + max_size=len(secret_types), + ), + ) + ) + .map(_post_process) +) + + class ExtractSecretsTests(SyncTestCase): """ Tests for ``_extract_secrets``. @@ -47,54 +76,31 @@ class ExtractSecretsTests(SyncTestCase): raise SkipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() - @given( - params=st.sets(st.sampled_from(Secrets)).flatmap( - lambda secret_types: st.tuples( - st.just(secret_types), - st.lists( - st.binary(min_size=32, max_size=32), - min_size=len(secret_types), - max_size=len(secret_types), - ), - ) - ) - ) - def test_extract_secrets(self, params): + @given(secrets_to_send=SECRETS_STRATEGY) + def test_extract_secrets(self, secrets_to_send): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secret_types, secrets = params - secrets = {t: s for (t, s) in zip(secret_types, secrets)} - headers = [ - "{} {}".format( - secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() - ) - for secret_type in secret_types - ] + secrets, headers = secrets_to_send # No secrets needed, none given: - self.assertEqual(_extract_secrets(headers, secret_types), secrets) + self.assertEqual(_extract_secrets(headers, secrets.keys()), secrets) - def test_wrong_number_of_secrets(self): + @given( + secrets_to_send=SECRETS_STRATEGY, + secrets_to_require=st.sets(st.sampled_from(Secrets)), + ) + def test_wrong_number_of_secrets(self, secrets_to_send, secrets_to_require): """ If the wrong number of secrets are passed to ``_extract_secrets``, a ``ClientSecretsException`` is raised. """ - secret1 = b"\xFF\x11ZEBRa" - lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() + secrets_to_send, headers = secrets_to_send + assume(secrets_to_send.keys() != secrets_to_require) - # Missing secret: with self.assertRaises(ClientSecretsException): - _extract_secrets([], {Secrets.LEASE_RENEW}) - - # Wrong secret: - with self.assertRaises(ClientSecretsException): - _extract_secrets([lease_secret], {Secrets.UPLOAD}) - - # Extra secret: - with self.assertRaises(ClientSecretsException): - _extract_secrets([lease_secret], {}) + _extract_secrets(headers, secrets_to_require) def test_bad_secrets(self): """ From 7a0c83e71be89f9a1efc631f7cb75df8b063fad8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Dec 2021 11:52:13 -0500 Subject: [PATCH 300/916] Split up test. --- src/allmydata/test/test_storage_http.py | 30 ++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 80bd2661b..160cf8479 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -102,29 +102,43 @@ class ExtractSecretsTests(SyncTestCase): with self.assertRaises(ClientSecretsException): _extract_secrets(headers, secrets_to_require) - def test_bad_secrets(self): + def test_bad_secret_missing_value(self): """ - Bad inputs to ``_extract_secrets`` result in + Missing value in ``_extract_secrets`` result in ``ClientSecretsException``. """ - - # Missing value. with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret"], {Secrets.LEASE_RENEW}) - # Garbage prefix + def test_bad_secret_unknown_prefix(self): + """ + Missing value in ``_extract_secrets`` result in + ``ClientSecretsException``. + """ with self.assertRaises(ClientSecretsException): _extract_secrets(["FOO eA=="], {}) - # Not base64. + def test_bad_secret_not_base64(self): + """ + A non-base64 value in ``_extract_secrets`` result in + ``ClientSecretsException``. + """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) - # Wrong length lease secrets (must be 32 bytes long). + def test_bad_secret_wrong_length_lease_renew(self): + """ + Lease renewal secrets must be 32-bytes long. + """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret eA=="], {Secrets.LEASE_RENEW}) + + def test_bad_secret_wrong_length_lease_cancel(self): + """ + Lease cancel secrets must be 32-bytes long. + """ with self.assertRaises(ClientSecretsException): - _extract_secrets(["lease-upload-secret eA=="], {Secrets.LEASE_RENEW}) + _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) SWISSNUM_FOR_TEST = b"abcd" From 58a71517c1fdc74b735876541d2dfa0ddfb2e5c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Dec 2021 13:16:43 -0500 Subject: [PATCH 301/916] Correct way to skip with testtools. --- src/allmydata/test/test_storage_http.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 160cf8479..181b6d347 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -14,7 +14,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on -from unittest import SkipTest from base64 import b64encode from twisted.internet.defer import inlineCallbacks @@ -73,7 +72,7 @@ class ExtractSecretsTests(SyncTestCase): def setUp(self): if PY2: - raise SkipTest("Not going to bother supporting Python 2") + self.skipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() @given(secrets_to_send=SECRETS_STRATEGY) @@ -165,7 +164,7 @@ class RoutingTests(AsyncTestCase): def setUp(self): if PY2: - raise SkipTest("Not going to bother supporting Python 2") + self.skipTest("Not going to bother supporting Python 2") super(RoutingTests, self).setUp() # Could be a fixture, but will only be used in this test class so not # going to bother: @@ -222,7 +221,7 @@ class GenericHTTPAPITests(AsyncTestCase): def setUp(self): if PY2: - raise SkipTest("Not going to bother supporting Python 2") + self.skipTest("Not going to bother supporting Python 2") super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) From e9aaaaccc4c9783a9d1eb74ca241d72797ceceec Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:31:09 -0700 Subject: [PATCH 302/916] test for json welcome page --- src/allmydata/test/web/test_root.py | 88 ++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 1d5e45ba4..b0789b1d2 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -11,6 +11,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time +import json from urllib.parse import ( quote, @@ -24,14 +25,23 @@ from twisted.web.template import Tag from twisted.web.test.requesthelper import DummyRequest from twisted.application import service from testtools.twistedsupport import succeeded -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import ( + inlineCallbacks, + succeed, +) from ...storage_client import ( NativeStorageServer, StorageFarmBroker, ) -from ...web.root import RootElement +from ...web.root import ( + RootElement, + Root, +) from ...util.connection_status import ConnectionStatus +from ...crypto.ed25519 import ( + create_signing_keypair, +) from allmydata.web.root import URIHandler from allmydata.client import _Client @@ -47,6 +57,7 @@ from ..common import ( from ..common import ( SyncTestCase, + AsyncTestCase, ) from testtools.matchers import ( @@ -138,3 +149,76 @@ class RenderServiceRow(SyncTestCase): self.assertThat(item.slotData.get("version"), Equals("")) self.assertThat(item.slotData.get("nickname"), Equals("")) + + +class RenderRoot(AsyncTestCase): + + @inlineCallbacks + def test_root_json(self): + """ + """ + ann = { + "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) + + class FakeClient(_Client): + history = [] + stats_provider = object() + nickname = "" + nodeid = b"asdf" + _node_public_key = create_signing_keypair()[1] + introducer_clients = [] + helper = None + + def __init__(self): + service.MultiService.__init__(self) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) + self.storage_broker.test_add_server(b"test-srv", srv) + + root = Root(FakeClient(), now_fn=time.time) + + lines = [] + + req = DummyRequest(b"") + req.fields = {} + req.args = { + "t": ["json"], + } + + # for some reason, DummyRequest is already finished when we + # try to add a notifyFinish handler, so override that + # behavior. + + def nop(): + return succeed(None) + req.notifyFinish = nop + req.write = lines.append + + yield root.render(req) + + raw_js = b"".join(lines).decode("utf8") + self.assertThat( + json.loads(raw_js), + Equals({ + "introducers": { + "statuses": [] + }, + "servers": [ + { + "connection_status": "summary", + "nodeid": "server_id", + "last_received_data": 0, + "version": None, + "available_space": None, + "nickname": "" + } + ] + }) + ) From 94b540215f6c32db026cbcaf588f22a9ebdfa866 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:30 -0700 Subject: [PATCH 303/916] args are bytes --- src/allmydata/test/web/test_root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index b0789b1d2..44b91fa48 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -189,7 +189,7 @@ class RenderRoot(AsyncTestCase): req = DummyRequest(b"") req.fields = {} req.args = { - "t": ["json"], + b"t": [b"json"], } # for some reason, DummyRequest is already finished when we From 5be5714bb378a9ad7180f7878ce75b96120afc5c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:40 -0700 Subject: [PATCH 304/916] fix; get rid of sorting --- src/allmydata/web/root.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 1debc1d10..f1a8569d6 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -297,14 +297,12 @@ class Root(MultiFormatResource): } return json.dumps(result, indent=1) + "\n" - def _describe_known_servers(self, broker): - return sorted(list( + return list( self._describe_server(server) for server in broker.get_known_servers() - ), key=lambda o: sorted(o.items())) - + ) def _describe_server(self, server): status = server.get_connection_status() From 872ce021c85b48321fe389200661cf3f087e959f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:59 -0700 Subject: [PATCH 305/916] news --- newsfragments/3852.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3852.minor diff --git a/newsfragments/3852.minor b/newsfragments/3852.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3852.minor @@ -0,0 +1 @@ + From 4c92f9c8cfd1d77d5549de3de50c552dbb442461 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:10:23 -0500 Subject: [PATCH 306/916] Document additional semantics. --- docs/proposed/http-storage-node-protocol.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 0d8cee466..a8555cd26 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -483,6 +483,13 @@ For example:: The upload secret is an opaque _byte_ string. +Handling repeat calls: + +* If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. + This is necessary to ensure retries work in the face of lost responses from the server. +* If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. + In this case, all relevant in-progress uploads are canceled, and then the command is handled as usual. + Discussion `````````` From cac291eb91129e39336ebc508727912224fda606 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:10:38 -0500 Subject: [PATCH 307/916] News file. --- newsfragments/3855.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3855.minor diff --git a/newsfragments/3855.minor b/newsfragments/3855.minor new file mode 100644 index 000000000..e69de29bb From 5f4db487f787b7c88c01974005f19c1f854e8fa4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:43:19 -0500 Subject: [PATCH 308/916] Sketch of required business logic. --- src/allmydata/storage/http_server.py | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 83bbbe49d..2dfb49b65 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -22,12 +22,14 @@ from base64 import b64decode from klein import Klein from twisted.web import http +import attr # TODO Make sure to use pure Python versions? -from cbor2 import dumps +from cbor2 import dumps, loads from .server import StorageServer from .http_client import swissnum_auth_header +from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -125,6 +127,19 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): return decorator +@attr.s +class StorageIndexUploads(object): + """ + In-progress upload to storage index. + """ + + # Map share number to BucketWriter + shares = attr.ib() # type: Dict[int,BucketWriter] + + # The upload key. + upload_key = attr.ib() # type: bytes + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -137,6 +152,8 @@ class HTTPServer(object): ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum + # Maps storage index to StorageIndexUploads: + self._uploads = {} # type: Dict[bytes,StorageIndexUploads] def get_resource(self): """Return twisted.web ``Resource`` for this object.""" @@ -154,3 +171,33 @@ class HTTPServer(object): def version(self, request, authorization): """Return version information.""" return self._cbor(request, self._storage_server.get_version()) + + ##### Immutable APIs ##### + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, + "/v1/immutable/", + methods=["POST"], + ) + def allocate_buckets(self, request, authorization, storage_index): + """Allocate buckets.""" + info = loads(request.content.read()) + upload_key = authorization[Secrets.UPLOAD] + + if storage_index in self._uploads: + # Pre-existing upload. + in_progress = self._uploads[storage_index] + if in_progress.upload_key == upload_key: + # Same session. + # TODO add BucketWriters only for new shares + pass + else: + # New session. + # TODO cancel all existing BucketWriters, then do + # self._storage_server.allocate_buckets() with given inputs. + pass + else: + # New upload. + # TODO self._storage_server.allocate_buckets() with given inputs. + # TODO add results to self._uploads. From 9c20ac8e7b1aaae6705b7980620410b792a4fee5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Jan 2022 16:06:29 -0500 Subject: [PATCH 309/916] Client API sketch for basic immutable interactions. --- src/allmydata/storage/http_client.py | 82 +++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 72e1af080..a13ab1ce6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,17 +16,19 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union + from typing import Union, Set, List from treq.testing import StubTreq from base64 import b64encode +import attr + # TODO Make sure to import Python version? from cbor2 import loads from twisted.web.http_headers import Headers -from twisted.internet.defer import inlineCallbacks, returnValue, fail +from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq @@ -47,6 +49,80 @@ def swissnum_auth_header(swissnum): # type: (bytes) -> bytes return b"Tahoe-LAFS " + b64encode(swissnum).strip() +@attr.s +class ImmutableCreateResult(object): + """Result of creating a storage index for an immutable.""" + + already_have = attr.ib(type=Set[int]) + allocated = attr.ib(type=Set[int]) + + +class StorageClientImmutables(object): + """ + APIs for interacting with immutables. + """ + + def __init__(self, client): # type: (StorageClient) -> None + self._client = client + + @inlineCallbacks + def create( + self, + storage_index, + share_numbers, + allocated_size, + upload_secret, + lease_renew_secret, + lease_cancel_secret, + ): # type: (bytes, List[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + """ + Create a new storage index for an immutable. + + TODO retry internally on failure, to ensure the operation fully + succeeded. If sufficient number of failures occurred, the result may + fire with an error, but there's no expectation that user code needs to + have a recovery codepath; it will most likely just report an error to + the user. + + Result fires when creating the storage index succeeded, if creating the + storage index failed the result will fire with an exception. + """ + + @inlineCallbacks + def write_share_chunk( + self, storage_index, share_number, upload_secret, offset, data + ): # type: (bytes, int, bytes, int, bytes) -> Deferred[bool] + """ + Upload a chunk of data for a specific share. + + TODO The implementation should retry failed uploads transparently a number + of times, so that if a failure percolates up, the caller can assume the + failure isn't a short-term blip. + + Result fires when the upload succeeded, with a boolean indicating + whether the _complete_ share (i.e. all chunks, not just this one) has + been uploaded. + """ + + @inlineCallbacks + def read_share_chunk( + self, storage_index, share_number, offset, length + ): # type: (bytes, int, int, int) -> Deferred[bytes] + """ + Download a chunk of data from a share. + + TODO Failed downloads should be transparently retried and redownloaded + by the implementation a few times so that if a failure percolates up, + the caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ + + class StorageClient(object): """ HTTP client that talks to the HTTP storage server. @@ -77,7 +153,7 @@ class StorageClient(object): for key, value in secrets.items(): headers.addRawHeader( "X-Tahoe-Authorization", - b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()) + b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()), ) return self._treq.request(method, url, headers=headers, **kwargs) From 90a25d010953b1ddf702d27d21d354c13a703d2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 12:36:46 -0500 Subject: [PATCH 310/916] Reorganize into shared file. --- src/allmydata/storage/http_client.py | 124 ++++++++++++++++----------- src/allmydata/storage/http_common.py | 25 ++++++ src/allmydata/storage/http_server.py | 12 +-- 3 files changed, 99 insertions(+), 62 deletions(-) create mode 100644 src/allmydata/storage/http_common.py diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a13ab1ce6..cdcb94a94 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -24,7 +24,7 @@ from base64 import b64encode import attr # TODO Make sure to import Python version? -from cbor2 import loads +from cbor2 import loads, dumps from twisted.web.http_headers import Headers @@ -32,6 +32,8 @@ from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq +from .http_common import swissnum_auth_header, Secrets + class ClientException(Exception): """An unexpected error.""" @@ -44,11 +46,6 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) -def swissnum_auth_header(swissnum): # type: (bytes) -> bytes - """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + b64encode(swissnum).strip() - - @attr.s class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" @@ -57,12 +54,75 @@ class ImmutableCreateResult(object): allocated = attr.ib(type=Set[int]) +class StorageClient(object): + """ + HTTP client that talks to the HTTP storage server. + """ + + def __init__( + self, url, swissnum, treq=treq + ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None + self._base_url = url + self._swissnum = swissnum + self._treq = treq + + def _url(self, path): + """Get a URL relative to the base URL.""" + return self._base_url.click(path) + + def _get_headers(self): # type: () -> Headers + """Return the basic headers to be used by default.""" + headers = Headers() + headers.addRawHeader( + "Authorization", + swissnum_auth_header(self._swissnum), + ) + return headers + + def _request( + self, + method, + url, + lease_renewal_secret=None, + lease_cancel_secret=None, + upload_secret=None, + **kwargs + ): + """ + Like ``treq.request()``, but with optional secrets that get translated + into corresponding HTTP headers. + """ + headers = self._get_headers() + for secret, value in [ + (Secrets.LEASE_RENEW, lease_renewal_secret), + (Secrets.LEASE_CANCEL, lease_cancel_secret), + (Secrets.UPLOAD, upload_secret), + ]: + if value is None: + continue + headers.addRawHeader( + "X-Tahoe-Authorization", + b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), + ) + return self._treq.request(method, url, headers=headers, **kwargs) + + @inlineCallbacks + def get_version(self): + """ + Return the version metadata for the server. + """ + url = self._url("/v1/version") + response = yield self._request("GET", url, {}) + decoded_response = yield _decode_cbor(response) + returnValue(decoded_response) + + class StorageClientImmutables(object): """ APIs for interacting with immutables. """ - def __init__(self, client): # type: (StorageClient) -> None + def __init__(self, client: StorageClient):# # type: (StorageClient) -> None self._client = client @inlineCallbacks @@ -87,6 +147,11 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ + url = self._client._url("/v1/immutable/" + str(storage_index, "ascii")) + message = dumps( + {"share-numbers": share_numbers, "allocated-size": allocated_size} + ) + self._client._request("POST", ) @inlineCallbacks def write_share_chunk( @@ -121,48 +186,3 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - - -class StorageClient(object): - """ - HTTP client that talks to the HTTP storage server. - """ - - def __init__( - self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None - self._base_url = url - self._swissnum = swissnum - self._treq = treq - - def _get_headers(self): # type: () -> Headers - """Return the basic headers to be used by default.""" - headers = Headers() - headers.addRawHeader( - "Authorization", - swissnum_auth_header(self._swissnum), - ) - return headers - - def _request(self, method, url, secrets, **kwargs): - """ - Like ``treq.request()``, but additional argument of secrets mapping - ``http_server.Secret`` to the bytes value of the secret. - """ - headers = self._get_headers() - for key, value in secrets.items(): - headers.addRawHeader( - "X-Tahoe-Authorization", - b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()), - ) - return self._treq.request(method, url, headers=headers, **kwargs) - - @inlineCallbacks - def get_version(self): - """ - Return the version metadata for the server. - """ - url = self._base_url.click("/v1/version") - response = yield self._request("GET", url, {}) - decoded_response = yield _decode_cbor(response) - returnValue(decoded_response) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py new file mode 100644 index 000000000..af4224bd0 --- /dev/null +++ b/src/allmydata/storage/http_common.py @@ -0,0 +1,25 @@ +""" +Common HTTP infrastructure for the storge server. +""" +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + +from enum import Enum +from base64 import b64encode + + +def swissnum_auth_header(swissnum): # type: (bytes) -> bytes + """Return value for ``Authentication`` header.""" + return b"Tahoe-LAFS " + b64encode(swissnum).strip() + + +class Secrets(Enum): + """Different kinds of secrets the client may send.""" + + LEASE_RENEW = "lease-renew-secret" + LEASE_CANCEL = "lease-cancel-secret" + UPLOAD = "upload-secret" diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2dfb49b65..78752e9c5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -17,7 +17,6 @@ else: from typing import Dict, List, Set from functools import wraps -from enum import Enum from base64 import b64decode from klein import Klein @@ -28,19 +27,11 @@ import attr from cbor2 import dumps, loads from .server import StorageServer -from .http_client import swissnum_auth_header +from .http_common import swissnum_auth_header, Secrets from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare -class Secrets(Enum): - """Different kinds of secrets the client may send.""" - - LEASE_RENEW = "lease-renew-secret" - LEASE_CANCEL = "lease-cancel-secret" - UPLOAD = "upload-secret" - - class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" @@ -201,3 +192,4 @@ class HTTPServer(object): # New upload. # TODO self._storage_server.allocate_buckets() with given inputs. # TODO add results to self._uploads. + pass From 2f94fdf372116f18fde2d764b0331edb995303fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 12:47:44 -0500 Subject: [PATCH 311/916] Extra testing coverage, including reproducer for #3854. --- src/allmydata/test/web/test_webish.py | 48 +++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 12a04a6eb..4a77d21ae 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -90,10 +90,11 @@ class TahoeLAFSRequestTests(SyncTestCase): """ self._fields_test(b"GET", {}, b"", Equals(None)) - def test_form_fields(self): + def test_form_fields_if_filename_set(self): """ When a ``POST`` request is received, form fields are parsed into - ``TahoeLAFSRequest.fields``. + ``TahoeLAFSRequest.fields`` and the body is bytes (presuming ``filename`` + is set). """ form_data, boundary = multipart_formdata([ [param(u"name", u"foo"), @@ -121,6 +122,49 @@ class TahoeLAFSRequestTests(SyncTestCase): ), ) + def test_form_fields_if_name_is_file(self): + """ + When a ``POST`` request is received, form fields are parsed into + ``TahoeLAFSRequest.fields`` and the body is bytes when ``name`` + is set to ``"file"``. + """ + form_data, boundary = multipart_formdata([ + [param(u"name", u"foo"), + body(u"bar"), + ], + [param(u"name", u"file"), + body(u"some file contents"), + ], + ]) + self._fields_test( + b"POST", + {b"content-type": b"multipart/form-data; boundary=" + bytes(boundary, 'ascii')}, + form_data.encode("ascii"), + AfterPreprocessing( + lambda fs: { + k: fs.getvalue(k) + for k + in fs.keys() + }, + Equals({ + "foo": "bar", + "file": b"some file contents", + }), + ), + ) + + def test_form_fields_require_correct_mime_type(self): + """ + The body of a ``POST`` is not parsed into fields if its mime type is + not ``multipart/form-data``. + + Reproducer for https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3854 + """ + data = u'{"lalala": "lolo"}' + data = data.encode("utf-8") + self._fields_test(b"POST", {"content-type": "application/json"}, + data, Equals(None)) + class TahoeLAFSSiteTests(SyncTestCase): """ From 9f5d7c6d22d40183aaab480990d83c122049495d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:09:25 -0500 Subject: [PATCH 312/916] Fix a bug where we did unnecessary parsing. --- src/allmydata/webish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 9b63a220c..559b475cb 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -114,7 +114,8 @@ class TahoeLAFSRequest(Request, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - if self.method == b'POST': + content_type = (self.requestHeaders.getRawHeaders("content-type") or [""])[0] + if self.method == b'POST' and content_type.split(";")[0] == "multipart/form-data": # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. From 310b77aef0765bf26a28ac4ed8d03f10c05dbb49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:10:13 -0500 Subject: [PATCH 313/916] News file. --- newsfragments/3854.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3854.bugfix diff --git a/newsfragments/3854.bugfix b/newsfragments/3854.bugfix new file mode 100644 index 000000000..d12e174f9 --- /dev/null +++ b/newsfragments/3854.bugfix @@ -0,0 +1 @@ +Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. \ No newline at end of file From 2864ff872d4ddb1f4a16f1669e597e6e7ab3565a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:34:56 -0500 Subject: [PATCH 314/916] Another MIME type that needs to be handled by FieldStorage. --- src/allmydata/webish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 559b475cb..519b3e1f0 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -115,7 +115,7 @@ class TahoeLAFSRequest(Request, object): self.args = parse_qs(argstring, 1) content_type = (self.requestHeaders.getRawHeaders("content-type") or [""])[0] - if self.method == b'POST' and content_type.split(";")[0] == "multipart/form-data": + if self.method == b'POST' and content_type.split(";")[0] in ("multipart/form-data", "application/x-www-form-urlencoded"): # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. From 983f90116b7b120d30a01b336f59bf1c0a62b9f2 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 6 Jan 2022 13:15:31 -0700 Subject: [PATCH 315/916] check differently, don't depend on order --- src/allmydata/test/web/test_web.py | 50 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1c9d6b65c..03cd6e560 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -820,29 +820,37 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ d = self.GET("/?t=json") def _check(res): + """ + Check that the results are correct. + We can't depend on the order of servers in the output + """ decoded = json.loads(res) - expected = { - u'introducers': { - u'statuses': [], + self.assertEqual(decoded['introducers'], {u'statuses': []}) + actual_servers = decoded[u"servers"] + self.assertEquals(len(actual_servers), 2) + self.assertIn( + { + u"nodeid": u'other_nodeid', + u'available_space': 123456, + u'connection_status': u'summary', + u'last_received_data': 30, + u'nickname': u'other_nickname \u263b', + u'version': u'1.0', }, - u'servers': sorted([ - {u"nodeid": u'other_nodeid', - u'available_space': 123456, - u'connection_status': u'summary', - u'last_received_data': 30, - u'nickname': u'other_nickname \u263b', - u'version': u'1.0', - }, - {u"nodeid": u'disconnected_nodeid', - u'available_space': 123456, - u'connection_status': u'summary', - u'last_received_data': 35, - u'nickname': u'disconnected_nickname \u263b', - u'version': u'1.0', - }, - ], key=lambda o: sorted(o.items())), - } - self.assertEqual(expected, decoded) + actual_servers + ) + self.assertIn( + { + u"nodeid": u'disconnected_nodeid', + u'available_space': 123456, + u'connection_status': u'summary', + u'last_received_data': 35, + u'nickname': u'disconnected_nickname \u263b', + u'version': u'1.0', + }, + actual_servers + ) + d.addCallback(_check) return d From 0bf713c38ab36651e841ca8c84e23ecf104aea55 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 10:12:21 -0500 Subject: [PATCH 316/916] News fragment. --- newsfragments/3856.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3856.minor diff --git a/newsfragments/3856.minor b/newsfragments/3856.minor new file mode 100644 index 000000000..e69de29bb From 7e3cb44ede60de3bed90470bf7f7803abac607b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 10:13:29 -0500 Subject: [PATCH 317/916] Pin non-broken version of Paramiko. --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e7a955c6..53057b808 100644 --- a/setup.py +++ b/setup.py @@ -409,7 +409,9 @@ setup(name="tahoe-lafs", # also set in __init__.py "html5lib", "junitxml", "tenacity", - "paramiko", + # Pin old version until + # https://github.com/paramiko/paramiko/issues/1961 is fixed. + "paramiko < 2.9", "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: "prometheus-client == 0.11.0", From 11f2097591e8161416237ecb4676d1843478eb5d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:58:58 -0700 Subject: [PATCH 318/916] docstring --- src/allmydata/test/web/test_root.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 44b91fa48..199c8e545 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -156,6 +156,11 @@ class RenderRoot(AsyncTestCase): @inlineCallbacks def test_root_json(self): """ + The 'welcome' / root page renders properly with ?t=json when some + servers show None for available_space while others show a + valid int + + See also https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3852 """ ann = { "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", From a49baf44b68eac81bb1538000c042ed537e57ef0 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:59:13 -0700 Subject: [PATCH 319/916] actually-reproduce 3852 --- src/allmydata/test/web/test_root.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 199c8e545..8c46b809a 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -166,8 +166,13 @@ class RenderRoot(AsyncTestCase): "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) - srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) + srv0 = NativeStorageServer(b"server_id0", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv0.get_connection_status = lambda: ConnectionStatus(False, "summary0", {}, 0, 0) + + srv1 = NativeStorageServer(b"server_id1", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv1.get_connection_status = lambda: ConnectionStatus(False, "summary1", {}, 0, 0) + # arrange for this server to have some valid available space + srv1.get_available_space = lambda: 12345 class FakeClient(_Client): history = [] @@ -185,7 +190,8 @@ class RenderRoot(AsyncTestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server(b"test-srv", srv) + self.storage_broker.test_add_server(b"test-srv0", srv0) + self.storage_broker.test_add_server(b"test-srv1", srv1) root = Root(FakeClient(), now_fn=time.time) @@ -217,12 +223,20 @@ class RenderRoot(AsyncTestCase): }, "servers": [ { - "connection_status": "summary", - "nodeid": "server_id", + "connection_status": "summary0", + "nodeid": "server_id0", "last_received_data": 0, "version": None, "available_space": None, "nickname": "" + }, + { + "connection_status": "summary1", + "nodeid": "server_id1", + "last_received_data": 0, + "version": None, + "available_space": 12345, + "nickname": "" } ] }) From e8f5023ae2e8b6404f3d7ad37db34fb28a3c4333 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:59:34 -0700 Subject: [PATCH 320/916] its a bugfix --- newsfragments/3852.minor => 3852.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/3852.minor => 3852.bugfix (100%) diff --git a/newsfragments/3852.minor b/3852.bugfix similarity index 100% rename from newsfragments/3852.minor rename to 3852.bugfix From 9d823aef67d7328c9b8ee2d2ae75703b5cd3b26a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 11:05:35 -0700 Subject: [PATCH 321/916] newsfragment to correct spot --- 3852.bugfix => newsfragments/3852.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 3852.bugfix => newsfragments/3852.bugfix (100%) diff --git a/3852.bugfix b/newsfragments/3852.bugfix similarity index 100% rename from 3852.bugfix rename to newsfragments/3852.bugfix From 9644532916de535b738f1e85f5ce060c5e77c604 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 11:28:55 -0700 Subject: [PATCH 322/916] don't depend on order --- src/allmydata/test/web/test_root.py | 49 ++++++++++++++--------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 8c46b809a..228b8e449 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -215,29 +215,28 @@ class RenderRoot(AsyncTestCase): yield root.render(req) raw_js = b"".join(lines).decode("utf8") - self.assertThat( - json.loads(raw_js), - Equals({ - "introducers": { - "statuses": [] - }, - "servers": [ - { - "connection_status": "summary0", - "nodeid": "server_id0", - "last_received_data": 0, - "version": None, - "available_space": None, - "nickname": "" - }, - { - "connection_status": "summary1", - "nodeid": "server_id1", - "last_received_data": 0, - "version": None, - "available_space": 12345, - "nickname": "" - } - ] - }) + js = json.loads(raw_js) + servers = js["servers"] + self.assertEquals(len(servers), 2) + self.assertIn( + { + "connection_status": "summary0", + "nodeid": "server_id0", + "last_received_data": 0, + "version": None, + "available_space": None, + "nickname": "" + }, + servers + ) + self.assertIn( + { + "connection_status": "summary1", + "nodeid": "server_id1", + "last_received_data": 0, + "version": None, + "available_space": 12345, + "nickname": "" + }, + servers ) From b91835a2007764fc924d05f484563b303100f8b5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:06:26 -0700 Subject: [PATCH 323/916] update NEWS.txt for release --- NEWS.rst | 14 ++++++++++++++ newsfragments/3848.minor | 0 newsfragments/3849.minor | 0 newsfragments/3850.minor | 0 newsfragments/3852.bugfix | 1 - newsfragments/3854.bugfix | 1 - newsfragments/3856.minor | 0 7 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 newsfragments/3848.minor delete mode 100644 newsfragments/3849.minor delete mode 100644 newsfragments/3850.minor delete mode 100644 newsfragments/3852.bugfix delete mode 100644 newsfragments/3854.bugfix delete mode 100644 newsfragments/3856.minor diff --git a/NEWS.rst b/NEWS.rst index 15cb9459d..62d1587dd 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,20 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) +'''''''''''''''''''''''''''''''''' + +Bug Fixes +--------- + +- (`#3852 `_) +- Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. (`#3854 `_) + + +Misc/Other +---------- + +- `#3848 `_, `#3849 `_, `#3850 `_, `#3856 `_ Release 1.17.0 (2021-12-06) diff --git a/newsfragments/3848.minor b/newsfragments/3848.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3849.minor b/newsfragments/3849.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3850.minor b/newsfragments/3850.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3852.bugfix b/newsfragments/3852.bugfix deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3852.bugfix +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3854.bugfix b/newsfragments/3854.bugfix deleted file mode 100644 index d12e174f9..000000000 --- a/newsfragments/3854.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. \ No newline at end of file diff --git a/newsfragments/3856.minor b/newsfragments/3856.minor deleted file mode 100644 index e69de29bb..000000000 From 22734dccba2cc95752de1c360830b728d4ac83b2 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:13:44 -0700 Subject: [PATCH 324/916] fix text for 3852 --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 62d1587dd..c2d405f8f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -11,7 +11,7 @@ Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) Bug Fixes --------- -- (`#3852 `_) +- Fixed regression on Python 3 causing the JSON version of the Welcome page to sometimes produce a 500 error (`#3852 `_) - Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. (`#3854 `_) From e9ece061f4a1a6373f39fe37291e1775df3a0391 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:18:03 -0700 Subject: [PATCH 325/916] news --- newsfragments/3858.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3858.minor diff --git a/newsfragments/3858.minor b/newsfragments/3858.minor new file mode 100644 index 000000000..e69de29bb From f9ddd3b3bedf692ffdf598d9def96b3c79097602 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:21:44 -0700 Subject: [PATCH 326/916] fix NEWS title --- NEWS.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index c2d405f8f..0f9194cc4 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,8 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) -'''''''''''''''''''''''''''''''''' +Release 1.17.1 (2022-01-07) +''''''''''''''''''''''''''' Bug Fixes --------- From b5251eb0a12eaa07411473583facd5c21cee729f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:27:53 -0700 Subject: [PATCH 327/916] update relnotes --- relnotes.txt | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index dff4f192e..e9b298771 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1 -The Tahoe-LAFS team is pleased to announce version 1.17.0 of +The Tahoe-LAFS team is pleased to announce version 1.17.1 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,19 +15,12 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.16.0, released on -October 19, 2021. +The previous stable release of Tahoe-LAFS was v1.17.0, released on +December 6, 2021. -This release fixes several security issues raised as part of an audit -by Cure53. We developed fixes for these issues in a private -repository. Shortly after this release, public tickets will be updated -with further information (along with, of course, all the code). +This release fixes two Python3-releated regressions and 4 minor bugs. -There is also OpenMetrics support now and several bug fixes. - -In all, 46 issues have been fixed since the last release. - -Please see ``NEWS.rst`` for a more complete list of changes. +Please see ``NEWS.rst`` [1] for a complete list of changes. WHAT IS IT GOOD FOR? @@ -66,12 +59,12 @@ to v1.0 (which was released March 25, 2008). Clients from this release can read files and directories produced by clients of all versions since v1.0. -Network connections are limited by the Introducer protocol in -use. If the Introducer is running v1.10 or v1.11, then servers -from this release (v1.12) can serve clients of all versions -back to v1.0 . If it is running v1.12, then they can only -serve clients back to v1.10. Clients from this release can use -servers back to v1.10, but not older servers. +Network connections are limited by the Introducer protocol in use. If +the Introducer is running v1.10 or v1.11, then servers from this +release can serve clients of all versions back to v1.0 . If it is +running v1.12 or higher, then they can only serve clients back to +v1.10. Clients from this release can use servers back to v1.10, but +not older servers. Except for the new optional MDMF format, we have not made any intentional compatibility changes. However we do not yet have @@ -79,7 +72,7 @@ the test infrastructure to continuously verify that all new versions are interoperable with previous versions. We intend to build such an infrastructure in the future. -This is the twenty-first release in the version 1 series. This +This is the twenty-second release in the version 1 series. This series of Tahoe-LAFS will be actively supported and maintained for the foreseeable future, and future versions of Tahoe-LAFS will retain the ability to read and write files compatible @@ -139,7 +132,7 @@ Of Fame" [13]. ACKNOWLEDGEMENTS -This is the eighteenth release of Tahoe-LAFS to be created +This is the nineteenth release of Tahoe-LAFS to be created solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. @@ -147,16 +140,16 @@ Tahoe-LAFS possible. meejah on behalf of the Tahoe-LAFS team -December 6, 2021 +January 7, 2022 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From c7664762365e1e14dd2c52374e3206ecd9b077a5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:28:27 -0700 Subject: [PATCH 328/916] nix --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 04d6c4163..2b41e676e 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.17.0). + # Most of the time this is not exactly the release version (eg 1.17.1). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.17.0.post1"; + version = "1.17.1.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From aa81bfc937b1ee7bbe2b43e814cc7683eed1d29e Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:29:45 -0700 Subject: [PATCH 329/916] cleanup whitespace --- docs/Installation/install-tahoe.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Installation/install-tahoe.rst b/docs/Installation/install-tahoe.rst index 2fe47f4a8..8ceca2e01 100644 --- a/docs/Installation/install-tahoe.rst +++ b/docs/Installation/install-tahoe.rst @@ -28,15 +28,15 @@ To install Tahoe-LAFS on Windows: 3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. 4. Start PowerShell and enter the following command to verify python installation:: - + python --version 5. Enter the following command to install Tahoe-LAFS:: - + pip install tahoe-lafs 6. Verify installation by checking for the version:: - + tahoe --version If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`. @@ -56,13 +56,13 @@ If you are working on MacOS or a Linux distribution which does not have Tahoe-LA * **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation `_. 2. Install Tahoe-LAFS using pip:: - + pip install tahoe-lafs 3. Verify installation by checking for the version:: - + tahoe --version -If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. From f7477043c5025642ef0fbeb042310decb774bd01 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:29:52 -0700 Subject: [PATCH 330/916] unnecessary step --- docs/release-checklist.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index f943abb5d..2b954449e 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -70,7 +70,6 @@ Create Branch and Apply Updates - commit it - update "docs/known_issues.rst" if appropriate -- update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github - Create a (draft) PR; this should trigger CI (note that github doesn't let you create a PR without some changes on the branch so From 852ebe90e5bd5b04b5a75d1850df87673a78955f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:48:55 -0700 Subject: [PATCH 331/916] clean clone --- docs/release-checklist.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 2b954449e..edfe9e20f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -106,6 +106,11 @@ they will need to evaluate which contributors' signatures they trust. - tox -e deprecations,upcoming-deprecations +- clone to a clean, local checkout (to avoid extra files being included in the release) + + - cd /tmp + - git clone /home/meejah/src/tahoe-lafs + - build tarballs - tox -e tarballs From d2ff2a7376f99f08fba22ee3c3b28cba535a0117 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:02 -0700 Subject: [PATCH 332/916] whitespace --- docs/release-checklist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index edfe9e20f..3c984d122 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -158,7 +158,8 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - secure-copy all release artifacts to the download area on the tahoe-lafs.org host machine. `~source/downloads` on there maps to https://tahoe-lafs.org/downloads/ on the Web. -- scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads + - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads + - the following developers have access to do this: - exarkun From 1446c9c4adebda255276659dfef883f17770ca7f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:15 -0700 Subject: [PATCH 333/916] add 'push the tags' step --- docs/release-checklist.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3c984d122..7acca6bb3 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -166,6 +166,10 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - meejah - warner +Push the signed tag to the main repository: + +- git push origin_push tahoe-lafs-1.17.1 + For the actual release, the tarball and signature files need to be uploaded to PyPI as well. From 8cd4e7a4b5069c3fb30c195934974755e8f0c53c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:31 -0700 Subject: [PATCH 334/916] news --- newsfragments/3859.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3859.minor diff --git a/newsfragments/3859.minor b/newsfragments/3859.minor new file mode 100644 index 000000000..e69de29bb From ea83b16d1171b789c2041ed1e67e2ffa6dec3ff4 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 14:17:50 -0700 Subject: [PATCH 335/916] most people say 'origin' --- docs/release-checklist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 7acca6bb3..165aa8826 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -168,7 +168,7 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` Push the signed tag to the main repository: -- git push origin_push tahoe-lafs-1.17.1 +- git push origin tahoe-lafs-1.17.1 For the actual release, the tarball and signature files need to be uploaded to PyPI as well. From a753a71105a8865cb27c9a59258fe349d55ba06a Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 14:22:57 -0700 Subject: [PATCH 336/916] please the Sphinx --- docs/release-checklist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 165aa8826..d2f1b3eb8 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -157,7 +157,8 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - secure-copy all release artifacts to the download area on the tahoe-lafs.org host machine. `~source/downloads` on there maps to - https://tahoe-lafs.org/downloads/ on the Web. + https://tahoe-lafs.org/downloads/ on the Web: + - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads - the following developers have access to do this: From 57405ea722d9ea97b74d71c2dca9a7eacd1b4ad5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 14:15:16 -0500 Subject: [PATCH 337/916] Finish sketch of minimal immutable HTTP client code. --- src/allmydata/storage/http_client.py | 102 ++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cdcb94a94..3accb3c62 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,7 +16,7 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union, Set, List + from typing import Union, Set, List, Optional from treq.testing import StubTreq from base64 import b64encode @@ -28,6 +28,7 @@ from cbor2 import loads, dumps from twisted.web.http_headers import Headers +from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq @@ -70,9 +71,10 @@ class StorageClient(object): """Get a URL relative to the base URL.""" return self._base_url.click(path) - def _get_headers(self): # type: () -> Headers + def _get_headers(self, headers): # type: (Optional[Headers]) -> Headers """Return the basic headers to be used by default.""" - headers = Headers() + if headers is None: + headers = Headers() headers.addRawHeader( "Authorization", swissnum_auth_header(self._swissnum), @@ -86,13 +88,14 @@ class StorageClient(object): lease_renewal_secret=None, lease_cancel_secret=None, upload_secret=None, + headers=None, **kwargs ): """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. """ - headers = self._get_headers() + headers = self._get_headers(headers) for secret, value in [ (Secrets.LEASE_RENEW, lease_renewal_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), @@ -122,7 +125,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client: StorageClient):# # type: (StorageClient) -> None + def __init__(self, client: StorageClient): # # type: (StorageClient) -> None self._client = client @inlineCallbacks @@ -138,11 +141,12 @@ class StorageClientImmutables(object): """ Create a new storage index for an immutable. - TODO retry internally on failure, to ensure the operation fully - succeeded. If sufficient number of failures occurred, the result may - fire with an error, but there's no expectation that user code needs to - have a recovery codepath; it will most likely just report an error to - the user. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 retry + internally on failure, to ensure the operation fully succeeded. If + sufficient number of failures occurred, the result may fire with an + error, but there's no expectation that user code needs to have a + recovery codepath; it will most likely just report an error to the + user. Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. @@ -151,7 +155,22 @@ class StorageClientImmutables(object): message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) - self._client._request("POST", ) + response = yield self._client._request( + "POST", + url, + lease_renew_secret=lease_renew_secret, + lease_cancel_secret=lease_cancel_secret, + upload_secret=upload_secret, + data=message, + headers=Headers({"content-type": "application/cbor"}), + ) + decoded_response = yield _decode_cbor(response) + returnValue( + ImmutableCreateResult( + already_have=decoded_response["already-have"], + allocated=decoded_response["allocated"], + ) + ) @inlineCallbacks def write_share_chunk( @@ -160,14 +179,45 @@ class StorageClientImmutables(object): """ Upload a chunk of data for a specific share. - TODO The implementation should retry failed uploads transparently a number - of times, so that if a failure percolates up, the caller can assume the + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 The + implementation should retry failed uploads transparently a number of + times, so that if a failure percolates up, the caller can assume the failure isn't a short-term blip. Result fires when the upload succeeded, with a boolean indicating whether the _complete_ share (i.e. all chunks, not just this one) has been uploaded. """ + url = self._client._url( + "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + ) + response = yield self._client._request( + "POST", + url, + upload_secret=upload_secret, + data=data, + headers=Headers( + { + # The range is inclusive, thus the '- 1'. '*' means "length + # unknown", which isn't technically true but adding it just + # makes things slightly harder for calling API. + "content-range": "bytes {}-{}/*".format( + offset, offset + len(data) - 1 + ) + } + ), + ) + + if response.code == http.OK: + # Upload is still unfinished. + returnValue(False) + elif response.code == http.CREATED: + # Upload is done! + returnValue(True) + else: + raise ClientException( + response.code, + ) @inlineCallbacks def read_share_chunk( @@ -176,9 +226,10 @@ class StorageClientImmutables(object): """ Download a chunk of data from a share. - TODO Failed downloads should be transparently retried and redownloaded - by the implementation a few times so that if a failure percolates up, - the caller can assume the failure isn't a short-term blip. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. NOTE: the underlying HTTP protocol is much more flexible than this API, so a future refactor may expand this in order to simplify the calling @@ -186,3 +237,22 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ + url = self._client._url( + "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + ) + response = yield self._client._request( + "GET", + url, + headers=Headers( + { + # The range is inclusive, thus the -1. + "range": "bytes={}-{}".format(offset, offset + length - 1) + } + ), + ) + if response.code == 200: + returnValue(response.content.read()) + else: + raise ClientException( + response.code, + ) From db68defe8897c48fb4a0ad31b9959b89664882b8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 14:50:29 -0500 Subject: [PATCH 338/916] Sketch of basic immutable server-side logic. --- src/allmydata/storage/http_server.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 78752e9c5..d2c9f6b7a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -193,3 +193,31 @@ class HTTPServer(object): # TODO self._storage_server.allocate_buckets() with given inputs. # TODO add results to self._uploads. pass + + @_authorized_route( + _app, + {Secrets.UPLOAD}, "/v1/immutable//", methods=["PATCH"] + ) + def write_share_data(self, request, authorization, storage_index, share_number): + """Write data to an in-progress immutable upload.""" + # TODO parse the content-range header to get offset for writing + # TODO basic checks on validity of offset + # TODO basic check that body isn't infinite. require content-length? if so, needs t be in protocol spec. + data = request.content.read() + # TODO write to bucket at that offset. + + # TODO check if it conflicts with existing data (probably underlying code already handles that) if so, CONFLICT. + + # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. + + @_authorized_route( + _app, set(), "/v1/immutable//", methods=["GET"] + ) + def read_share_chunk(self, request, authorization, storage_index, share_number): + """Read a chunk for an already uploaded immutable.""" + # TODO read offset and length from Range header + # TODO basic checks on validity + # TODO lookup the share + # TODO if not found, 404 + # TODO otherwise, return data from that offset + From 040569b47acac5a00eddcbd03ef880c2b4f6727a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:11:16 -0500 Subject: [PATCH 339/916] Sketch of tests to write for basic HTTP immutable APIs. --- src/allmydata/test/test_storage_http.py | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 181b6d347..6cf7a521e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -263,3 +263,72 @@ class GenericHTTPAPITests(AsyncTestCase): b"maximum-immutable-share-size" ) self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + """ + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ From 2369de6873f20791f36c884760085010cca64941 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:45:15 -0500 Subject: [PATCH 340/916] Simple upload/download test for immutables. --- src/allmydata/test/test_storage_http.py | 69 ++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6cf7a521e..95708d211 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from base64 import b64encode +from os import urandom from twisted.internet.defer import inlineCallbacks @@ -33,7 +34,12 @@ from ..storage.http_server import ( ClientSecretsException, _authorized_route, ) -from ..storage.http_client import StorageClient, ClientException +from ..storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, +) def _post_process(params): @@ -270,12 +276,73 @@ class ImmutableHTTPAPITests(AsyncTestCase): Tests for immutable upload/download APIs. """ + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks def test_upload_can_be_downloaded(self): """ A single share can be uploaded in (possibly overlapping) chunks, and then a random chunk can be downloaded, and it will match the original file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, upload_secret, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) def test_multiple_shares_uploaded_to_different_place(self): """ From 004e5fbc9d2266585508dc376900e33ba1557a49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:47:32 -0500 Subject: [PATCH 341/916] Get to point where we get failing HTTP response. --- src/allmydata/storage/http_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3accb3c62..002ffc928 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -85,7 +85,7 @@ class StorageClient(object): self, method, url, - lease_renewal_secret=None, + lease_renew_secret=None, lease_cancel_secret=None, upload_secret=None, headers=None, @@ -97,7 +97,7 @@ class StorageClient(object): """ headers = self._get_headers(headers) for secret, value in [ - (Secrets.LEASE_RENEW, lease_renewal_secret), + (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), (Secrets.UPLOAD, upload_secret), ]: @@ -162,7 +162,7 @@ class StorageClientImmutables(object): lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, data=message, - headers=Headers({"content-type": "application/cbor"}), + headers=Headers({"content-type": ["application/cbor"]}), ) decoded_response = yield _decode_cbor(response) returnValue( @@ -201,9 +201,9 @@ class StorageClientImmutables(object): # The range is inclusive, thus the '- 1'. '*' means "length # unknown", which isn't technically true but adding it just # makes things slightly harder for calling API. - "content-range": "bytes {}-{}/*".format( - offset, offset + len(data) - 1 - ) + "content-range": [ + "bytes {}-{}/*".format(offset, offset + len(data) - 1) + ] } ), ) @@ -245,8 +245,8 @@ class StorageClientImmutables(object): url, headers=Headers( { - # The range is inclusive, thus the -1. - "range": "bytes={}-{}".format(offset, offset + length - 1) + # The range is inclusive. + "range": ["bytes={}-{}".format(offset, offset + length)] } ), ) From 6e2aaa8391e46bf02ee37a794dcc51c8ced84a25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 09:14:58 -0500 Subject: [PATCH 342/916] Refactor more integration-y tests out. --- integration/test_storage_http.py | 283 ++++++++++++++++++++++++ src/allmydata/test/test_storage_http.py | 274 +---------------------- 2 files changed, 284 insertions(+), 273 deletions(-) create mode 100644 integration/test_storage_http.py diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py new file mode 100644 index 000000000..714562cc4 --- /dev/null +++ b/integration/test_storage_http.py @@ -0,0 +1,283 @@ +""" +Connect the HTTP storage client to the HTTP storage server and make sure they +can talk to each other. +""" + +from future.utils import PY2 + +from os import urandom + +from twisted.internet.defer import inlineCallbacks +from fixtures import Fixture, TempDir +from treq.testing import StubTreq +from hyperlink import DecodedURL +from klein import Klein + +from allmydata.storage.server import StorageServer +from allmydata.storage.http_server import ( + HTTPServer, + _authorized_route, +) +from allmydata.storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, +) +from allmydata.storage.http_common import Secrets +from allmydata.test.common import AsyncTestCase + + +# TODO should be actual swissnum +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(AsyncTestCase): + """ + Tests for the HTTP routing infrastructure. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {} + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} + ) + self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") + + +class HttpTestFixture(Fixture): + """ + Setup HTTP tests' infrastructure, the storage server and corresponding + client. + """ + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) + + +class GenericHTTPAPITests(AsyncTestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + + We ignore available disk space and max immutable share size, since that + might change across calls. + """ + version = yield self.http.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + expected_version = self.http.storage_server.get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. + """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, upload_secret, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 95708d211..aaa455a03 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,30 +15,13 @@ if PY2: # fmt: on from base64 import b64encode -from os import urandom - -from twisted.internet.defer import inlineCallbacks from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir -from treq.testing import StubTreq -from klein import Klein -from hyperlink import DecodedURL - -from .common import AsyncTestCase, SyncTestCase -from ..storage.server import StorageServer +from .common import SyncTestCase from ..storage.http_server import ( - HTTPServer, _extract_secrets, Secrets, ClientSecretsException, - _authorized_route, -) -from ..storage.http_client import ( - StorageClient, - ClientException, - StorageClientImmutables, - ImmutableCreateResult, ) @@ -144,258 +127,3 @@ class ExtractSecretsTests(SyncTestCase): """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) - - -SWISSNUM_FOR_TEST = b"abcd" - - -class TestApp(object): - """HTTP API for testing purposes.""" - - _app = Klein() - _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - - @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) - def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"MAGIC"}: - return "GOOD SECRET" - else: - return "BAD: {}".format(authorization) - - -class RoutingTests(AsyncTestCase): - """ - Tests for the HTTP routing infrastructure. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() - # Could be a fixture, but will only be used in this test class so not - # going to bother: - self._http_server = TestApp() - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) - - @inlineCallbacks - def test_authorization_enforcement(self): - """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. - """ - # Without secret, get a 400 error. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {} - ) - self.assertEqual(response.code, 400) - - # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} - ) - self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") - - -class HttpTestFixture(Fixture): - """ - Setup HTTP tests' infrastructure, the storage server and corresponding - client. - """ - - def _setUp(self): - self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), - ) - - -class GenericHTTPAPITests(AsyncTestCase): - """ - Tests of HTTP client talking to the HTTP server, for generic HTTP API - endpoints and concerns. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(GenericHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_bad_authentication(self): - """ - If the wrong swissnum is used, an ``Unauthorized`` response code is - returned. - """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), - ) - with self.assertRaises(ClientException) as e: - yield client.get_version() - self.assertEqual(e.exception.args[0], 401) - - @inlineCallbacks - def test_version(self): - """ - The client can return the version. - - We ignore available disk space and max immutable share size, since that - might change across calls. - """ - version = yield self.http.client.get_version() - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - expected_version = self.http.storage_server.get_version() - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - self.assertEqual(version, expected_version) - - -class ImmutableHTTPAPITests(AsyncTestCase): - """ - Tests for immutable upload/download APIs. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ImmutableHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_upload_can_be_downloaded(self): - """ - A single share can be uploaded in (possibly overlapping) chunks, and - then a random chunk can be downloaded, and it will match the original - file. - - We don't exercise the full variation of overlapping chunks because - that's already done in test_storage.py. - """ - length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) - - im_client = StorageClientImmutables(self.http.client) - - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret - ) - self.assertEqual( - created, ImmutableCreateResult(already_have=set(), allocated={1}) - ) - - # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - def write(offset, length): - return im_client.write_share_chunk( - storage_index, - 1, - upload_secret, - offset, - expected_data[offset : offset + length], - ) - - finished = yield write(10, 10) - self.assertFalse(finished) - finished = yield write(30, 10) - self.assertFalse(finished) - finished = yield write(50, 10) - self.assertFalse(finished) - - # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) - self.assertFalse(finished) - - # Now fill in the holes: - finished = yield write(0, 10) - self.assertFalse(finished) - finished = yield write(40, 10) - self.assertFalse(finished) - finished = yield write(60, 40) - self.assertTrue(finished) - - # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, upload_secret, offset, length - ) - self.assertEqual(downloaded, expected_data[offset : offset + length]) - - def test_multiple_shares_uploaded_to_different_place(self): - """ - If a storage index has multiple shares, uploads to different shares are - stored separately and can be downloaded separately. - """ - - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - """ - - def test_bucket_allocation_new_upload_key(self): - """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. - """ - - def test_upload_with_wrong_upload_key_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - """ - - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - """ - - def test_mismatching_upload_fails(self): - """ - If an uploaded chunk conflicts with an already uploaded chunk, a - CONFLICT error is returned. - """ - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_with_negative_offset_fails(self): - """ - The offset for reads cannot be negative. - """ - - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. - """ From 2bccb01be4bbd33b0b25642049f6ab2ae2697e17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:16:21 -0500 Subject: [PATCH 343/916] Fix bug wrapping endpoints. --- src/allmydata/storage/http_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d2c9f6b7a..b371fc395 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -110,6 +110,7 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): def decorator(f): @app.route(*route_args, **route_kwargs) @_authorization_decorator(required_secrets) + @wraps(f) def handle_route(*args, **kwargs): return f(*args, **kwargs) From 018f53105e9ad6c29dd82e324f2405ef9c75eb54 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:16:39 -0500 Subject: [PATCH 344/916] Pass correct arguments. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 002ffc928..e38525583 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -115,7 +115,7 @@ class StorageClient(object): Return the version metadata for the server. """ url = self._url("/v1/version") - response = yield self._request("GET", url, {}) + response = yield self._request("GET", url) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) From c4bb3c21d13a757007c058c467ee1dbc602be521 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:18:34 -0500 Subject: [PATCH 345/916] Update test to match current API. --- integration/test_storage_http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py index 714562cc4..66a8b2af1 100644 --- a/integration/test_storage_http.py +++ b/integration/test_storage_http.py @@ -72,13 +72,14 @@ class RoutingTests(AsyncTestCase): """ # Without secret, get a 400 error. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {} + "GET", + "http://127.0.0.1/upload_secret", ) self.assertEqual(response.code, 400) # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" ) self.assertEqual(response.code, 200) self.assertEqual((yield response.content()), b"GOOD SECRET") From f5437d9be73b42b891f892336551c95074d84b4d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:51:56 -0500 Subject: [PATCH 346/916] Some progress towards bucket allocation endpoint, and defining the protocol better. --- docs/proposed/http-storage-node-protocol.rst | 5 +++ src/allmydata/storage/http_client.py | 12 ++++++-- src/allmydata/storage/http_server.py | 32 ++++++++++++++++---- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a8555cd26..26f1a2bb7 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -382,6 +382,11 @@ the server will respond with ``400 BAD REQUEST``. If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. +Encoding +~~~~~~~~ + +* ``storage_index`` should be base32 encoded (RFC3548) in URLs. + General ~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e38525583..5e964bfbe 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -34,6 +34,12 @@ from hyperlink import DecodedURL import treq from .http_common import swissnum_auth_header, Secrets +from .common import si_b2a + + +def _encode_si(si): # type: (bytes) -> str + """Encode the storage index into Unicode string.""" + return str(si_b2a(si), "ascii") class ClientException(Exception): @@ -151,7 +157,7 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client._url("/v1/immutable/" + str(storage_index, "ascii")) + url = self._client._url("/v1/immutable/" + _encode_si(storage_index)) message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) @@ -189,7 +195,7 @@ class StorageClientImmutables(object): been uploaded. """ url = self._client._url( - "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( "POST", @@ -238,7 +244,7 @@ class StorageClientImmutables(object): https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ url = self._client._url( - "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( "GET", diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b371fc395..23f0d2f1c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -28,6 +28,7 @@ from cbor2 import dumps, loads from .server import StorageServer from .http_common import swissnum_auth_header, Secrets +from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -174,6 +175,7 @@ class HTTPServer(object): ) def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" + storage_index = si_a2b(storage_index.encode("ascii")) info = loads(request.content.read()) upload_key = authorization[Secrets.UPLOAD] @@ -191,13 +193,29 @@ class HTTPServer(object): pass else: # New upload. - # TODO self._storage_server.allocate_buckets() with given inputs. - # TODO add results to self._uploads. - pass + already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( + storage_index, + renew_secret=authorization[Secrets.LEASE_RENEW], + cancel_secret=authorization[Secrets.LEASE_CANCEL], + sharenums=info["share-numbers"], + allocated_size=info["allocated-size"], + ) + self._uploads[storage_index] = StorageIndexUploads( + shares=sharenum_to_bucket, upload_key=authorization[Secrets.UPLOAD] + ) + return self._cbor( + request, + { + "already-have": set(already_got), + "allocated": set(sharenum_to_bucket), + }, + ) @_authorized_route( _app, - {Secrets.UPLOAD}, "/v1/immutable//", methods=["PATCH"] + {Secrets.UPLOAD}, + "/v1/immutable//", + methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" @@ -212,7 +230,10 @@ class HTTPServer(object): # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. @_authorized_route( - _app, set(), "/v1/immutable//", methods=["GET"] + _app, + set(), + "/v1/immutable//", + methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" @@ -221,4 +242,3 @@ class HTTPServer(object): # TODO lookup the share # TODO if not found, 404 # TODO otherwise, return data from that offset - From 3bed0678285deb5ac2064bbc4c045c0b75b4df2b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Jan 2022 08:34:17 -0500 Subject: [PATCH 347/916] Implement more of the writing logic. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 32 ++++++++++++++++++++++------ src/allmydata/storage/immutable.py | 11 ++++++++-- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5e964bfbe..697af91b5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -198,7 +198,7 @@ class StorageClientImmutables(object): "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( - "POST", + "PATCH", url, upload_secret=upload_secret, data=data, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 23f0d2f1c..28367752d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -219,15 +219,35 @@ class HTTPServer(object): ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" - # TODO parse the content-range header to get offset for writing - # TODO basic checks on validity of offset - # TODO basic check that body isn't infinite. require content-length? if so, needs t be in protocol spec. + storage_index = si_a2b(storage_index.encode("ascii")) + content_range = request.getHeader("content-range") + if content_range is None: + offset = 0 + else: + offset = int(content_range.split()[1].split("-")[0]) + + # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. + # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. + data = request.content.read() - # TODO write to bucket at that offset. + try: + bucket = self._uploads[storage_index].shares[share_number] + except (KeyError, IndexError): + # TODO return 404 + raise - # TODO check if it conflicts with existing data (probably underlying code already handles that) if so, CONFLICT. + finished = bucket.write(offset, data) - # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. + # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. + + if finished: + request.setResponseCode(http.CREATED) + else: + request.setResponseCode(http.OK) + + # TODO spec says we should return missing ranges. but client doesn't + # actually use them? So is it actually useful? + return b"" @_authorized_route( _app, diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index da9aa473f..5878e254a 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,7 +13,7 @@ if PY2: import os, stat, struct, time -from collections_extended import RangeMap +from collections_extended import RangeMap, MappedRange from foolscap.api import Referenceable @@ -375,7 +375,10 @@ class BucketWriter(object): def allocated_size(self): return self._max_size - def write(self, offset, data): + def write(self, offset, data): # type: (int, bytes) -> bool + """ + Write data at given offset, return whether the upload is complete. + """ # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() @@ -399,6 +402,10 @@ class BucketWriter(object): self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") + # Return whether the whole thing has been written. + # TODO needs property test + return self._already_written.ranges() == [MappedRange(0, self._max_size, True)] + def close(self): precondition(not self.closed) self._timeout.cancel() From 4ea6bf2381f5a43f1265be988b2b9c50ad017384 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sat, 15 Jan 2022 12:59:23 -0500 Subject: [PATCH 348/916] A test and some progress to making it pass. --- src/allmydata/storage/immutable.py | 12 +++++++---- src/allmydata/test/test_storage.py | 33 ++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 5878e254a..0bcef8246 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,7 +13,7 @@ if PY2: import os, stat, struct, time -from collections_extended import RangeMap, MappedRange +from collections_extended import RangeMap from foolscap.api import Referenceable @@ -402,9 +402,13 @@ class BucketWriter(object): self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") - # Return whether the whole thing has been written. - # TODO needs property test - return self._already_written.ranges() == [MappedRange(0, self._max_size, True)] + # Return whether the whole thing has been written. See + # https://github.com/mlenzen/collections-extended/issues/169 for why + # it's done this way. + print([tuple(mr) for mr in self._already_written.ranges()]) + return [tuple(mr) for mr in self._already_written.ranges()] == [ + (0, self._max_size, True) + ] def close(self): precondition(not self.closed) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bd74a1052..e4f01f6f1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,6 +13,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str +from array import array from io import ( BytesIO, ) @@ -34,7 +35,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.internet.task import Clock -from hypothesis import given, strategies +from hypothesis import given, strategies, example import itertools from allmydata import interfaces @@ -230,7 +231,6 @@ class Bucket(unittest.TestCase): br = BucketReader(self, bw.finalhome) self.assertEqual(br.read(0, length), expected_data) - @given( maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), maybe_overlapping_length=strategies.integers(min_value=1, max_value=100), @@ -264,6 +264,35 @@ class Bucket(unittest.TestCase): bw.write(40, b"1" * 10) bw.write(60, b"1" * 40) + @given( + offsets=strategies.lists( + strategies.integers(min_value=0, max_value=99), + min_size=20, + max_size=20 + ), + ) + @example(offsets=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 40, 70]) + def test_writes_return_when_finished( + self, offsets + ): + """ + The ``BucketWriter.write()`` return true if and only if the maximum + size has been reached via potentially overlapping writes. + """ + length = 100 + incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) + bw = BucketWriter( + self, incoming, final, length, self.make_lease(), Clock() + ) + local_written = [0] * 100 + for offset in offsets: + length = min(30, 100 - offset) + data = b"1" * length + for i in range(offset, offset+length): + local_written[i] = 1 + finished = bw.write(offset, data) + self.assertEqual(finished, sum(local_written) == 100) + def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share # file): From 25e2100219ddc067f5090072fb318930a1eb0660 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:06:21 -0500 Subject: [PATCH 349/916] Immutable writing now knows when it's finished. --- src/allmydata/storage/immutable.py | 8 +++----- src/allmydata/test/test_storage.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0bcef8246..d17f69c07 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -403,12 +403,10 @@ class BucketWriter(object): self.ss.count("write") # Return whether the whole thing has been written. See - # https://github.com/mlenzen/collections-extended/issues/169 for why + # https://github.com/mlenzen/collections-extended/issues/169 and + # https://github.com/mlenzen/collections-extended/issues/172 for why # it's done this way. - print([tuple(mr) for mr in self._already_written.ranges()]) - return [tuple(mr) for mr in self._already_written.ranges()] == [ - (0, self._max_size, True) - ] + return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): precondition(not self.closed) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e4f01f6f1..881bfb6fd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -279,10 +279,9 @@ class Bucket(unittest.TestCase): The ``BucketWriter.write()`` return true if and only if the maximum size has been reached via potentially overlapping writes. """ - length = 100 incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), Clock() + self, incoming, final, 100, self.make_lease(), Clock() ) local_written = [0] * 100 for offset in offsets: From d4ae7c89aa0ba41b45720bebfe90054bc9e53df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:20:40 -0500 Subject: [PATCH 350/916] First end-to-end immutable upload then download test passes. --- integration/test_storage_http.py | 2 +- src/allmydata/storage/http_client.py | 5 +++-- src/allmydata/storage/http_server.py | 21 ++++++++++++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py index 66a8b2af1..0e2cf89a6 100644 --- a/integration/test_storage_http.py +++ b/integration/test_storage_http.py @@ -223,7 +223,7 @@ class ImmutableHTTPAPITests(AsyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: downloaded = yield im_client.read_share_chunk( - storage_index, 1, upload_secret, offset, length + storage_index, 1, offset, length ) self.assertEqual(downloaded, expected_data[offset : offset + length]) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 697af91b5..b091b3ca7 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -252,12 +252,13 @@ class StorageClientImmutables(object): headers=Headers( { # The range is inclusive. - "range": ["bytes={}-{}".format(offset, offset + length)] + "range": ["bytes={}-{}".format(offset, offset + length - 1)] } ), ) if response.code == 200: - returnValue(response.content.read()) + body = yield response.content() + returnValue(body) else: raise ClientException( response.code, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 28367752d..50d955127 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -241,6 +241,7 @@ class HTTPServer(object): # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. if finished: + bucket.close() request.setResponseCode(http.CREATED) else: request.setResponseCode(http.OK) @@ -257,8 +258,22 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO read offset and length from Range header # TODO basic checks on validity - # TODO lookup the share + storage_index = si_a2b(storage_index.encode("ascii")) + range_header = request.getHeader("range") + if range_header is None: + offset = 0 + inclusive_end = None + else: + parts = range_header.split("=")[1].split("-") + offset = int(parts[0]) # TODO make sure valid + if len(parts) > 0: + inclusive_end = int(parts[1]) # TODO make sure valid + else: + inclusive_end = None + + assert inclusive_end != None # TODO support this case + # TODO if not found, 404 - # TODO otherwise, return data from that offset + bucket = self._storage_server.get_buckets(storage_index)[share_number] + return bucket.read(offset, inclusive_end - offset + 1) From 79cd9a3d6d236749943228ed5fcc2287c6187e48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:22:15 -0500 Subject: [PATCH 351/916] Fix lint. --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 881bfb6fd..27309a82a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,7 +13,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str -from array import array from io import ( BytesIO, ) From 7aed7dbd8a7219e257dd0dc638eecd57e26fb66a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:24:28 -0500 Subject: [PATCH 352/916] Make module import on Python 2 (so tests can pass). --- src/allmydata/storage/http_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b091b3ca7..8fea86396 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -13,6 +13,11 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on + from collections import defaultdict + + Optional = Set = defaultdict( + lambda: None + ) # some garbage to just make this module import else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. @@ -131,7 +136,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client: StorageClient): # # type: (StorageClient) -> None + def __init__(self, client): # type: (StorageClient) -> None self._client = client @inlineCallbacks From 28dbdbe019f53dc16e0b6ea4af0822eba61b0842 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:31:29 -0500 Subject: [PATCH 353/916] Make sure return type is consistent. --- src/allmydata/storage/immutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index d17f69c07..0949929a9 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -384,7 +384,7 @@ class BucketWriter(object): start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: - return + return False # Make sure we're not conflicting with existing data: end = offset + len(data) From 406a06a5080c691c9216849bba43100061b8ac3c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:38:06 -0500 Subject: [PATCH 354/916] Make sure we don't violate the Foolscap interface definition for this method. --- src/allmydata/storage/immutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0949929a9..e35ae9782 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -494,7 +494,7 @@ class FoolscapBucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._bucket_writer = bucket_writer def remote_write(self, offset, data): - return self._bucket_writer.write(offset, data) + self._bucket_writer.write(offset, data) def remote_close(self): return self._bucket_writer.close() From 23368fc9d95811c2088d430cc0dced995d212527 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:34:09 -0500 Subject: [PATCH 355/916] Move tests back into unittest module. --- integration/test_storage_http.py | 284 ------------------------ src/allmydata/test/test_storage_http.py | 275 ++++++++++++++++++++++- 2 files changed, 274 insertions(+), 285 deletions(-) delete mode 100644 integration/test_storage_http.py diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py deleted file mode 100644 index 0e2cf89a6..000000000 --- a/integration/test_storage_http.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -Connect the HTTP storage client to the HTTP storage server and make sure they -can talk to each other. -""" - -from future.utils import PY2 - -from os import urandom - -from twisted.internet.defer import inlineCallbacks -from fixtures import Fixture, TempDir -from treq.testing import StubTreq -from hyperlink import DecodedURL -from klein import Klein - -from allmydata.storage.server import StorageServer -from allmydata.storage.http_server import ( - HTTPServer, - _authorized_route, -) -from allmydata.storage.http_client import ( - StorageClient, - ClientException, - StorageClientImmutables, - ImmutableCreateResult, -) -from allmydata.storage.http_common import Secrets -from allmydata.test.common import AsyncTestCase - - -# TODO should be actual swissnum -SWISSNUM_FOR_TEST = b"abcd" - - -class TestApp(object): - """HTTP API for testing purposes.""" - - _app = Klein() - _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - - @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) - def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"MAGIC"}: - return "GOOD SECRET" - else: - return "BAD: {}".format(authorization) - - -class RoutingTests(AsyncTestCase): - """ - Tests for the HTTP routing infrastructure. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() - # Could be a fixture, but will only be used in this test class so not - # going to bother: - self._http_server = TestApp() - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) - - @inlineCallbacks - def test_authorization_enforcement(self): - """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. - """ - # Without secret, get a 400 error. - response = yield self.client._request( - "GET", - "http://127.0.0.1/upload_secret", - ) - self.assertEqual(response.code, 400) - - # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" - ) - self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") - - -class HttpTestFixture(Fixture): - """ - Setup HTTP tests' infrastructure, the storage server and corresponding - client. - """ - - def _setUp(self): - self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), - ) - - -class GenericHTTPAPITests(AsyncTestCase): - """ - Tests of HTTP client talking to the HTTP server, for generic HTTP API - endpoints and concerns. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(GenericHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_bad_authentication(self): - """ - If the wrong swissnum is used, an ``Unauthorized`` response code is - returned. - """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), - ) - with self.assertRaises(ClientException) as e: - yield client.get_version() - self.assertEqual(e.exception.args[0], 401) - - @inlineCallbacks - def test_version(self): - """ - The client can return the version. - - We ignore available disk space and max immutable share size, since that - might change across calls. - """ - version = yield self.http.client.get_version() - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - expected_version = self.http.storage_server.get_version() - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - self.assertEqual(version, expected_version) - - -class ImmutableHTTPAPITests(AsyncTestCase): - """ - Tests for immutable upload/download APIs. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ImmutableHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_upload_can_be_downloaded(self): - """ - A single share can be uploaded in (possibly overlapping) chunks, and - then a random chunk can be downloaded, and it will match the original - file. - - We don't exercise the full variation of overlapping chunks because - that's already done in test_storage.py. - """ - length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) - - im_client = StorageClientImmutables(self.http.client) - - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret - ) - self.assertEqual( - created, ImmutableCreateResult(already_have=set(), allocated={1}) - ) - - # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - def write(offset, length): - return im_client.write_share_chunk( - storage_index, - 1, - upload_secret, - offset, - expected_data[offset : offset + length], - ) - - finished = yield write(10, 10) - self.assertFalse(finished) - finished = yield write(30, 10) - self.assertFalse(finished) - finished = yield write(50, 10) - self.assertFalse(finished) - - # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) - self.assertFalse(finished) - - # Now fill in the holes: - finished = yield write(0, 10) - self.assertFalse(finished) - finished = yield write(40, 10) - self.assertFalse(finished) - finished = yield write(60, 40) - self.assertTrue(finished) - - # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, offset, length - ) - self.assertEqual(downloaded, expected_data[offset : offset + length]) - - def test_multiple_shares_uploaded_to_different_place(self): - """ - If a storage index has multiple shares, uploads to different shares are - stored separately and can be downloaded separately. - """ - - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - """ - - def test_bucket_allocation_new_upload_key(self): - """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. - """ - - def test_upload_with_wrong_upload_key_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - """ - - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - """ - - def test_mismatching_upload_fails(self): - """ - If an uploaded chunk conflicts with an already uploaded chunk, a - CONFLICT error is returned. - """ - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_with_negative_offset_fails(self): - """ - The offset for reads cannot be negative. - """ - - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. - """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index aaa455a03..af53efbde 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,13 +15,30 @@ if PY2: # fmt: on from base64 import b64encode +from os import urandom + +from twisted.internet.defer import inlineCallbacks from hypothesis import assume, given, strategies as st -from .common import SyncTestCase +from fixtures import Fixture, TempDir +from treq.testing import StubTreq +from klein import Klein +from hyperlink import DecodedURL + +from .common import AsyncTestCase, SyncTestCase +from ..storage.server import StorageServer from ..storage.http_server import ( + HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + _authorized_route, +) +from ..storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, ) @@ -127,3 +144,259 @@ class ExtractSecretsTests(SyncTestCase): """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) + + +# TODO should be actual swissnum +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(AsyncTestCase): + """ + Tests for the HTTP routing infrastructure. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", + "http://127.0.0.1/upload_secret", + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + ) + self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") + + +class HttpTestFixture(Fixture): + """ + Setup HTTP tests' infrastructure, the storage server and corresponding + client. + """ + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) + + +class GenericHTTPAPITests(AsyncTestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + + We ignore available disk space and max immutable share size, since that + might change across calls. + """ + version = yield self.http.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + expected_version = self.http.storage_server.get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. + """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ From 1bf2b2ee5f4bec2df84f2a8bfac8aaa40ca08c95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:52:44 -0500 Subject: [PATCH 356/916] Note follow-up issue. --- src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af53efbde..2689b429f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -349,12 +349,16 @@ class ImmutableHTTPAPITests(AsyncTestCase): """ If a storage index has multiple shares, uploads to different shares are stored separately and can be downloaded separately. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_bucket_allocated_with_new_shares(self): """ If some shares already exist, allocating shares indicates only the new ones were created. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_bucket_allocation_new_upload_key(self): @@ -362,41 +366,57 @@ class ImmutableHTTPAPITests(AsyncTestCase): If a bucket was allocated with one upload key, and a different upload key is used to allocate the bucket again, the previous download is cancelled. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_upload_with_wrong_upload_key_fails(self): """ Uploading with a key that doesn't match the one used to allocate the bucket will fail. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_upload_offset_cannot_be_negative(self): """ A negative upload offset will be rejected. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a CONFLICT error is returned. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_of_wrong_storage_index_fails(self): """ Reading from unknown storage index results in 404. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_with_negative_offset_fails(self): """ The offset for reads cannot be negative. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_with_negative_length_fails(self): """ The length for reads cannot be negative. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ From d5bac8e186859f16c7de5637b601b215a87782c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:56:08 -0500 Subject: [PATCH 357/916] Make sure upload secret semantics are still supporting the security goals. --- docs/proposed/http-storage-node-protocol.rst | 3 ++- src/allmydata/storage/http_server.py | 4 +--- src/allmydata/test/test_storage_http.py | 9 ++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 26f1a2bb7..bb1db750c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -493,7 +493,8 @@ Handling repeat calls: * If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. This is necessary to ensure retries work in the face of lost responses from the server. * If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. - In this case, all relevant in-progress uploads are canceled, and then the command is handled as usual. + In order to prevent storage servers from being able to mess with each other, this API call will fail, because the secret doesn't match. + The use case of restarting upload from scratch if the client dies can be implemented by having the client persist the upload secret. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50d955127..71c34124a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -187,9 +187,7 @@ class HTTPServer(object): # TODO add BucketWriters only for new shares pass else: - # New session. - # TODO cancel all existing BucketWriters, then do - # self._storage_server.allocate_buckets() with given inputs. + # TODO Fail, since the secret doesnt match. pass else: # New upload. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2689b429f..a7aad608e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -361,16 +361,15 @@ class ImmutableHTTPAPITests(AsyncTestCase): TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_bucket_allocation_new_upload_key(self): + def test_bucket_allocation_new_upload_secret(self): """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. + If a bucket was allocated with one upload secret, and a different upload + key is used to allocate the bucket again, the second allocation fails. TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_upload_with_wrong_upload_key_fails(self): + def test_upload_with_wrong_upload_secret_fails(self): """ Uploading with a key that doesn't match the one used to allocate the bucket will fail. From f09aa8c7969d8455a958c278d3fdb889aae15d71 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 11:16:06 -0500 Subject: [PATCH 358/916] Use pre-existing parser for Range and Content-Range headers. --- docs/proposed/http-storage-node-protocol.rst | 2 +- nix/tahoe-lafs.nix | 4 +- setup.py | 1 + src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 41 ++++++++++---------- src/allmydata/test/test_storage_http.py | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bb1db750c..560220d00 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -640,7 +640,7 @@ For example:: Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). -The ``Range`` header may be used to request exactly one ``bytes`` range. +The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported. diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 04d6c4163..1885dd9ca 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, cbor2 +, html5lib, pyutil, distro, configparser, klein, werkzeug, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.17.0). @@ -98,7 +98,7 @@ EOF service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended - klein cbor2 treq + klein werkzeug cbor2 treq ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index 7e7a955c6..36e82a2b2 100644 --- a/setup.py +++ b/setup.py @@ -143,6 +143,7 @@ install_requires = [ # HTTP server and client "klein", + "werkzeug", "treq", "cbor2" ] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8fea86396..cf453fcfc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -261,7 +261,7 @@ class StorageClientImmutables(object): } ), ) - if response.code == 200: + if response.code == http.PARTIAL_CONTENT: body = yield response.content() returnValue(body) else: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 71c34124a..bbb42dbe1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -22,6 +22,7 @@ from base64 import b64decode from klein import Klein from twisted.web import http import attr +from werkzeug.http import parse_range_header, parse_content_range_header # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -218,11 +219,12 @@ class HTTPServer(object): def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" storage_index = si_a2b(storage_index.encode("ascii")) - content_range = request.getHeader("content-range") - if content_range is None: - offset = 0 - else: - offset = int(content_range.split()[1].split("-")[0]) + content_range = parse_content_range_header(request.getHeader("content-range")) + # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + # 1. Malformed header should result in error + # 2. Non-bytes unit should result in error + # 3. Missing header means full upload in one request + offset = content_range.start # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. @@ -256,22 +258,21 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO basic checks on validity + # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + # 1. basic checks on validity on storage index, share number + # 2. missing range header should have response code 200 and return whole thing + # 3. malformed range header should result in error? or return everything? + # 4. non-bytes range results in error + # 5. ranges make sense semantically (positive, etc.) + # 6. multiple ranges fails with error + # 7. missing end of range means "to the end of share" storage_index = si_a2b(storage_index.encode("ascii")) - range_header = request.getHeader("range") - if range_header is None: - offset = 0 - inclusive_end = None - else: - parts = range_header.split("=")[1].split("-") - offset = int(parts[0]) # TODO make sure valid - if len(parts) > 0: - inclusive_end = int(parts[1]) # TODO make sure valid - else: - inclusive_end = None - - assert inclusive_end != None # TODO support this case + range_header = parse_range_header(request.getHeader("range")) + offset, end = range_header.ranges[0] + assert end != None # TODO support this case # TODO if not found, 404 bucket = self._storage_server.get_buckets(storage_index)[share_number] - return bucket.read(offset, inclusive_end - offset + 1) + data = bucket.read(offset, end - offset) + request.setResponseCode(http.PARTIAL_CONTENT) + return data diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index a7aad608e..540e40c16 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -339,7 +339,7 @@ class ImmutableHTTPAPITests(AsyncTestCase): self.assertTrue(finished) # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = yield im_client.read_share_chunk( storage_index, 1, offset, length ) From 5fa8c78f97c14a378ea51f457093647c88c4b597 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:04:20 -0500 Subject: [PATCH 359/916] Don't use reactor, since it's not necessary. --- src/allmydata/test/test_storage_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 540e40c16..8b71666b0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,7 +25,7 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL -from .common import AsyncTestCase, SyncTestCase +from .common import SyncTestCase from ..storage.server import StorageServer from ..storage.http_server import ( HTTPServer, @@ -164,7 +164,7 @@ class TestApp(object): return "BAD: {}".format(authorization) -class RoutingTests(AsyncTestCase): +class RoutingTests(SyncTestCase): """ Tests for the HTTP routing infrastructure. """ @@ -220,7 +220,7 @@ class HttpTestFixture(Fixture): ) -class GenericHTTPAPITests(AsyncTestCase): +class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API endpoints and concerns. @@ -272,7 +272,7 @@ class GenericHTTPAPITests(AsyncTestCase): self.assertEqual(version, expected_version) -class ImmutableHTTPAPITests(AsyncTestCase): +class ImmutableHTTPAPITests(SyncTestCase): """ Tests for immutable upload/download APIs. """ From 9a0a19c15a70287d82dbc49e3a102c5a7b62b1d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:07:58 -0500 Subject: [PATCH 360/916] Reminder we might want to support JSON too. --- src/allmydata/storage/http_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bbb42dbe1..236204d66 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -155,6 +155,8 @@ class HTTPServer(object): def _cbor(self, request, data): """Return CBOR-encoded data.""" + # TODO Might want to optionally send JSON someday, based on Accept + # headers, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 request.setHeader("Content-Type", "application/cbor") # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) From 587a510b06a406d47e4d61417830d1f69455fcce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:38:01 -0500 Subject: [PATCH 361/916] Note a better way to implement this. --- src/allmydata/storage/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 2daf081e4..0add9806b 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -353,6 +353,9 @@ class StorageServer(service.MultiService): max_space_per_bucket, lease_info, clock=self._clock) if self.no_storage: + # Really this should be done by having a separate class for + # this situation; see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3862 bw.throw_out_all_data = True bucketwriters[shnum] = bw self._bucket_writers[incominghome] = bw From 2a2ab1ead722f60cc7771819908dd1b2fb35f58c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:39:25 -0500 Subject: [PATCH 362/916] Use a set, not a list, for share numbers. --- src/allmydata/storage/http_client.py | 4 ++-- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cf453fcfc..36e745395 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -21,7 +21,7 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union, Set, List, Optional + from typing import Union, Set, Optional from treq.testing import StubTreq from base64 import b64encode @@ -148,7 +148,7 @@ class StorageClientImmutables(object): upload_secret, lease_renew_secret, lease_cancel_secret, - ): # type: (bytes, List[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + ): # type: (bytes, Set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] """ Create a new storage index for an immutable. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8b71666b0..a3e7d1640 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -303,7 +303,7 @@ class ImmutableHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = b"".join(bytes([i]) for i in range(16)) created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret + storage_index, {1}, 100, upload_secret, lease_secret, lease_secret ) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) From b952e738dd60e5b5b6f85cbe232f82a18c828846 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:42:26 -0500 Subject: [PATCH 363/916] Try to clarify. --- src/allmydata/storage/http_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 36e745395..dca8b761c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -210,8 +210,10 @@ class StorageClientImmutables(object): headers=Headers( { # The range is inclusive, thus the '- 1'. '*' means "length - # unknown", which isn't technically true but adding it just - # makes things slightly harder for calling API. + # unknown", which isn't technically true but it's not clear + # there's any value in passing it in. The server has to + # handle this case anyway, and requiring share length means + # a bit more work for the calling API with no benefit. "content-range": [ "bytes {}-{}/*".format(offset, offset + len(data) - 1) ] From 4b5c71ffbc21a0ff961c15fb27da521adf294bf8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:50:36 -0500 Subject: [PATCH 364/916] Bit more info. --- src/allmydata/storage/http_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 236204d66..84c1a2e69 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -223,9 +223,10 @@ class HTTPServer(object): storage_index = si_a2b(storage_index.encode("ascii")) content_range = parse_content_range_header(request.getHeader("content-range")) # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. Malformed header should result in error - # 2. Non-bytes unit should result in error + # 1. Malformed header should result in error 416 + # 2. Non-bytes unit should result in error 416 # 3. Missing header means full upload in one request + # 4. Impossible range should resul tin error 416 offset = content_range.start # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. From 65787e5603215fb2b9df3893662e9cefa8112da3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:57:52 -0500 Subject: [PATCH 365/916] Get rid of inlineCallbacks. --- src/allmydata/test/test_storage_http.py | 67 ++++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index a3e7d1640..948b6c718 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -17,8 +17,6 @@ if PY2: from base64 import b64encode from os import urandom -from twisted.internet.defer import inlineCallbacks - from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq @@ -164,6 +162,23 @@ class TestApp(object): return "BAD: {}".format(authorization) +def result_of(d): + """ + Synchronously extract the result of a Deferred. + """ + result = [] + error = [] + d.addCallbacks(result.append, error.append) + if result: + return result[0] + if error: + error[0].raiseException() + raise RuntimeError( + "We expected given Deferred to have result already, but it wasn't. " + + "This is probably a test design issue." + ) + + class RoutingTests(SyncTestCase): """ Tests for the HTTP routing infrastructure. @@ -182,25 +197,28 @@ class RoutingTests(SyncTestCase): treq=StubTreq(self._http_server._app.resource()), ) - @inlineCallbacks def test_authorization_enforcement(self): """ The requirement for secrets is enforced; if they are not given, a 400 response code is returned. """ # Without secret, get a 400 error. - response = yield self.client._request( - "GET", - "http://127.0.0.1/upload_secret", + response = result_of( + self.client._request( + "GET", + "http://127.0.0.1/upload_secret", + ) ) self.assertEqual(response.code, 400) # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + response = result_of( + self.client._request( + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + ) ) self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") + self.assertEqual(result_of(response.content()), b"GOOD SECRET") class HttpTestFixture(Fixture): @@ -232,7 +250,6 @@ class GenericHTTPAPITests(SyncTestCase): super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - @inlineCallbacks def test_bad_authentication(self): """ If the wrong swissnum is used, an ``Unauthorized`` response code is @@ -244,10 +261,9 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), ) with self.assertRaises(ClientException) as e: - yield client.get_version() + result_of(client.get_version()) self.assertEqual(e.exception.args[0], 401) - @inlineCallbacks def test_version(self): """ The client can return the version. @@ -255,7 +271,7 @@ class GenericHTTPAPITests(SyncTestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = yield self.http.client.get_version() + version = result_of(self.http.client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) @@ -283,7 +299,6 @@ class ImmutableHTTPAPITests(SyncTestCase): super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - @inlineCallbacks def test_upload_can_be_downloaded(self): """ A single share can be uploaded in (possibly overlapping) chunks, and @@ -302,8 +317,10 @@ class ImmutableHTTPAPITests(SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, {1}, 100, upload_secret, lease_secret, lease_secret + created = result_of( + im_client.create( + storage_index, {1}, 100, upload_secret, lease_secret, lease_secret + ) ) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) @@ -319,29 +336,29 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - finished = yield write(10, 10) + finished = result_of(write(10, 10)) self.assertFalse(finished) - finished = yield write(30, 10) + finished = result_of(write(30, 10)) self.assertFalse(finished) - finished = yield write(50, 10) + finished = result_of(write(50, 10)) self.assertFalse(finished) # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) + finished = result_of(write(15, 20)) self.assertFalse(finished) # Now fill in the holes: - finished = yield write(0, 10) + finished = result_of(write(0, 10)) self.assertFalse(finished) - finished = yield write(40, 10) + finished = result_of(write(40, 10)) self.assertFalse(finished) - finished = yield write(60, 40) + finished = result_of(write(60, 40)) self.assertTrue(finished) # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, offset, length + downloaded = result_of( + im_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) From c4d71a4636503df1afa5a7884c473c0674f427f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 13:10:42 -0500 Subject: [PATCH 366/916] Use abstractions for generating headers on client, note another place we should generate headers. --- src/allmydata/storage/http_client.py | 14 +++++--------- src/allmydata/storage/http_server.py | 6 ++++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index dca8b761c..4436f2fd2 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -31,7 +31,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps - +from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred @@ -209,13 +209,8 @@ class StorageClientImmutables(object): data=data, headers=Headers( { - # The range is inclusive, thus the '- 1'. '*' means "length - # unknown", which isn't technically true but it's not clear - # there's any value in passing it in. The server has to - # handle this case anyway, and requiring share length means - # a bit more work for the calling API with no benefit. "content-range": [ - "bytes {}-{}/*".format(offset, offset + len(data) - 1) + ContentRange("bytes", offset, offset+len(data)).to_header() ] } ), @@ -258,8 +253,9 @@ class StorageClientImmutables(object): url, headers=Headers( { - # The range is inclusive. - "range": ["bytes={}-{}".format(offset, offset + length - 1)] + "range": [ + Range("bytes", [(offset, offset + length)]).to_header() + ] } ), ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 84c1a2e69..6b792cf06 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -278,4 +278,10 @@ class HTTPServer(object): bucket = self._storage_server.get_buckets(storage_index)[share_number] data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) + # TODO set content-range on response. We we need to expand the + # BucketReader interface to return share's length. + # + # request.setHeader( + # "content-range", range_header.make_content_range(share_length).to_header() + # ) return data From e8e3a3e663458b4df5cab8553890a75db02c6942 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 11:37:46 -0500 Subject: [PATCH 367/916] Expand. --- src/allmydata/storage/http_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6b792cf06..73ef8e09e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -187,7 +187,8 @@ class HTTPServer(object): in_progress = self._uploads[storage_index] if in_progress.upload_key == upload_key: # Same session. - # TODO add BucketWriters only for new shares + # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. + # The backend code may already implement this logic. pass else: # TODO Fail, since the secret doesnt match. From a4cb4837e6ea122a2273bd1560def2550b438664 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 11:43:36 -0500 Subject: [PATCH 368/916] It's a secret, compare it securely. --- src/allmydata/storage/http_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 73ef8e09e..a19faf1fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -131,7 +131,7 @@ class StorageIndexUploads(object): shares = attr.ib() # type: Dict[int,BucketWriter] # The upload key. - upload_key = attr.ib() # type: bytes + upload_secret = attr.ib() # type: bytes class HTTPServer(object): @@ -180,12 +180,12 @@ class HTTPServer(object): """Allocate buckets.""" storage_index = si_a2b(storage_index.encode("ascii")) info = loads(request.content.read()) - upload_key = authorization[Secrets.UPLOAD] + upload_secret = authorization[Secrets.UPLOAD] if storage_index in self._uploads: # Pre-existing upload. in_progress = self._uploads[storage_index] - if in_progress.upload_key == upload_key: + if timing_safe_compare(in_progress.upload_secret, upload_secret): # Same session. # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. # The backend code may already implement this logic. @@ -203,7 +203,7 @@ class HTTPServer(object): allocated_size=info["allocated-size"], ) self._uploads[storage_index] = StorageIndexUploads( - shares=sharenum_to_bucket, upload_key=authorization[Secrets.UPLOAD] + shares=sharenum_to_bucket, upload_secret=authorization[Secrets.UPLOAD] ) return self._cbor( request, From d2e3b74098c2a94c104f395d8c293ee40f0862be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 12:36:58 -0500 Subject: [PATCH 369/916] Some progress towards upload progress result from the server. --- src/allmydata/storage/http_client.py | 24 ++++++++++--- src/allmydata/storage/http_server.py | 8 +++-- src/allmydata/storage/immutable.py | 10 ++++++ src/allmydata/test/test_storage.py | 6 +++- src/allmydata/test/test_storage_http.py | 48 +++++++++++++++++-------- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4436f2fd2..d4837d4ab 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -30,7 +30,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps - +from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http @@ -131,6 +131,17 @@ class StorageClient(object): returnValue(decoded_response) +@attr.s +class UploadProgress(object): + """ + Progress of immutable upload, per the server. + """ + # True when upload has finished. + finished = attr.ib(type=bool) + # Remaining ranges to upload. + required = attr.ib(type=RangeMap) + + class StorageClientImmutables(object): """ APIs for interacting with immutables. @@ -186,7 +197,7 @@ class StorageClientImmutables(object): @inlineCallbacks def write_share_chunk( self, storage_index, share_number, upload_secret, offset, data - ): # type: (bytes, int, bytes, int, bytes) -> Deferred[bool] + ): # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress] """ Upload a chunk of data for a specific share. @@ -218,14 +229,19 @@ class StorageClientImmutables(object): if response.code == http.OK: # Upload is still unfinished. - returnValue(False) + finished = False elif response.code == http.CREATED: # Upload is done! - returnValue(True) + finished = True else: raise ClientException( response.code, ) + body = loads((yield response.content())) + remaining = RangeMap() + for chunk in body["required"]: + remaining.set(True, chunk["begin"], chunk["end"]) + returnValue(UploadProgress(finished=finished, required=remaining)) @inlineCallbacks def read_share_chunk( diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a19faf1fa..1d1a9466c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,6 +23,7 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header +from collections_extended import RangeMap # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -250,9 +251,10 @@ class HTTPServer(object): else: request.setResponseCode(http.OK) - # TODO spec says we should return missing ranges. but client doesn't - # actually use them? So is it actually useful? - return b"" + required = [] + for start, end, _ in bucket.required_ranges().ranges(): + required.append({"begin": start, "end": end}) + return self._cbor(request, {"required": required}) @_authorized_route( _app, diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index e35ae9782..920bd3c5e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -372,6 +372,16 @@ class BucketWriter(object): self._clock = clock self._timeout = clock.callLater(30 * 60, self._abort_due_to_timeout) + def required_ranges(self): # type: () -> RangeMap + """ + Return which ranges still need to be written. + """ + result = RangeMap() + result.set(True, 0, self._max_size) + for start, end, _ in self._already_written.ranges(): + result.delete(start, end) + return result + def allocated_size(self): return self._max_size diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 27309a82a..b37f74c24 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -276,7 +276,8 @@ class Bucket(unittest.TestCase): ): """ The ``BucketWriter.write()`` return true if and only if the maximum - size has been reached via potentially overlapping writes. + size has been reached via potentially overlapping writes. The + remaining ranges can be checked via ``BucketWriter.required_ranges()``. """ incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( @@ -290,6 +291,9 @@ class Bucket(unittest.TestCase): local_written[i] = 1 finished = bw.write(offset, data) self.assertEqual(finished, sum(local_written) == 100) + required_ranges = bw.required_ranges() + for i in range(0, 100): + self.assertEqual(local_written[i] == 1, required_ranges.get(i) is None) def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 948b6c718..b1eeca4e7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -22,6 +22,7 @@ from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL +from collections_extended import RangeMap from .common import SyncTestCase from ..storage.server import StorageServer @@ -37,6 +38,7 @@ from ..storage.http_client import ( ClientException, StorageClientImmutables, ImmutableCreateResult, + UploadProgress, ) @@ -326,8 +328,12 @@ class ImmutableHTTPAPITests(SyncTestCase): created, ImmutableCreateResult(already_have=set(), allocated={1}) ) + remaining = RangeMap() + remaining.set(True, 0, 100) + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): + remaining.empty(offset, offset + length) return im_client.write_share_chunk( storage_index, 1, @@ -336,24 +342,38 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - finished = result_of(write(10, 10)) - self.assertFalse(finished) - finished = result_of(write(30, 10)) - self.assertFalse(finished) - finished = result_of(write(50, 10)) - self.assertFalse(finished) + upload_progress = result_of(write(10, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(30, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(50, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) # Then, an overlapping write with matching data (15-35): - finished = result_of(write(15, 20)) - self.assertFalse(finished) + upload_progress = result_of(write(15, 20)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) # Now fill in the holes: - finished = result_of(write(0, 10)) - self.assertFalse(finished) - finished = result_of(write(40, 10)) - self.assertFalse(finished) - finished = result_of(write(60, 40)) - self.assertTrue(finished) + upload_progress = result_of(write(0, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(40, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(60, 40)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=RangeMap()) + ) # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: From 764e493c98798f2b4248189e1c0faa3a9df27ffb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:32:27 -0500 Subject: [PATCH 370/916] News. --- newsfragments/3865.incompat | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3865.incompat diff --git a/newsfragments/3865.incompat b/newsfragments/3865.incompat new file mode 100644 index 000000000..59381b269 --- /dev/null +++ b/newsfragments/3865.incompat @@ -0,0 +1 @@ +Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. \ No newline at end of file From 8eb6ab47653f7e8be52c310332f8f6d9686cd2ae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:40:26 -0500 Subject: [PATCH 371/916] Switch to Python 3.7 as minimal version. --- .circleci/config.yml | 12 ++++++------ .github/workflows/ci.yml | 7 +++---- Makefile | 4 ++-- misc/python3/Makefile | 4 ++-- setup.py | 5 ++--- tox.ini | 5 ++--- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fc8e88e7..d55b80469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,8 +49,8 @@ workflows: - "pypy27-buster": {} - # Just one Python 3.6 configuration while the port is in-progress. - - "python36": + # Test against Python 3: + - "python37": {} # Other assorted tasks and configurations @@ -118,7 +118,7 @@ workflows: <<: *DOCKERHUB_CONTEXT - "build-image-pypy27-buster": <<: *DOCKERHUB_CONTEXT - - "build-image-python36-ubuntu": + - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -379,7 +379,7 @@ jobs: user: "nobody" - python36: + python37: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH @@ -392,7 +392,7 @@ jobs: # this reporter on Python 3. So drop that and just specify the # reporter. TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" ubuntu-20-04: @@ -577,7 +577,7 @@ jobs: PYTHON_VERSION: "2.7" - build-image-python36-ubuntu: + build-image-python37-ubuntu: <<: *BUILD_IMAGE environment: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8209108bf..5ae70a3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,12 +39,11 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 - 3.7 - 3.8 - 3.9 include: - # On macOS don't bother with 3.6-3.8, just to get faster builds. + # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest @@ -181,10 +180,10 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 + - 3.7 - 3.9 include: - # On macOS don't bother with 3.6, just to get faster builds. + # On macOS don't bother with 3.7, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest diff --git a/Makefile b/Makefile index 5d8bf18ba..33a40df02 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py36,pypy27' + tox --develop -p auto -e 'py27,py37,pypy27' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -51,7 +51,7 @@ test-venv-coverage: .PHONY: test-py3-all ## Run all tests under Python 3 test-py3-all: .tox/create-venvs.log - tox --develop -e py36 allmydata + tox --develop -e py37 allmydata # This is necessary only if you want to automatically produce a new # _version.py file from the current git history (without doing a build). diff --git a/misc/python3/Makefile b/misc/python3/Makefile index f0ef8b12a..43cb3e3ce 100644 --- a/misc/python3/Makefile +++ b/misc/python3/Makefile @@ -37,8 +37,8 @@ test-py3-all-diff: ../../.tox/make-test-py3-all.diff # `$ make .tox/make-test-py3-all.diff` $(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ + tox --develop --notest -e py37-coverage + (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ test-venv-coverage || true) | \ sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ tee "./misc/python3/$(@)" diff --git a/setup.py b/setup.py index 53057b808..9a1c76bd8 100644 --- a/setup.py +++ b/setup.py @@ -376,9 +376,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and we're working on support for 3.6 (the - # highest version that PyPy currently supports). - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + # We support Python 2.7, and Python 3.7 or later. + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See diff --git a/tox.ini b/tox.ini index 38cee1f9f..34d555aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,6 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -18,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -51,7 +50,7 @@ deps = # suffering we're trying to avoid with the above pins. certifi # VCS hooks support - py36,!coverage: pre-commit + py37,!coverage: pre-commit # We add usedevelop=False because testing against a true installation gives # more useful results. From fa2b4a11c76a1b55100ea51fa25f9d0dcda9ff6d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:50:40 -0500 Subject: [PATCH 372/916] Welcome to the WORLD OF TOMORROW --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d55b80469..5ef9c81a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,7 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:17.05.0-ce-git" + image: "docker:20.10" environment: DISTRO: "tahoelafsci/:foo-py2" From f04e121a7d31d1b18bdd5c5d40618f98ce3bb2ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:51:55 -0500 Subject: [PATCH 373/916] Try to use correct Docker image. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ef9c81a8..05686fcf8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -383,7 +383,7 @@ jobs: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" user: "nobody" environment: @@ -583,7 +583,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "18.04" - PYTHON_VERSION: "3" + PYTHON_VERSION: "3.7" build-image-ubuntu-20-04: From 02740f075bb69f74ae3ed198a426a0fb3750a5e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:56:11 -0500 Subject: [PATCH 374/916] Temporarily enable image builds on every push. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05686fcf8..6578f1d1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 31e4556bd1910c19a74e30b793d5098ca5f27b8c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:01:47 -0500 Subject: [PATCH 375/916] Need image with Docker _and_ git+ssh. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6578f1d1a..d28196097 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,9 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:20.10" + # CircleCI build images; https://github.com/CircleCI-Public/cimg-base + # for details. + image: "cimg/base:2022.01" environment: DISTRO: "tahoelafsci/:foo-py2" From 04cf206e0d78a174c0f80a80180340838f332b67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:06:58 -0500 Subject: [PATCH 376/916] Switch back to running image building on schedule. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d28196097..a650313ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From b64e6552a44a5ae0e4149dbcb363fc51fc469fe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:30:41 -0500 Subject: [PATCH 377/916] Fix assertion. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b1eeca4e7..dcefc9950 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -372,7 +372,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) upload_progress = result_of(write(60, 40)) self.assertEqual( - upload_progress, UploadProgress(finished=False, required=RangeMap()) + upload_progress, UploadProgress(finished=True, required=RangeMap()) ) # We can now read: From e9d6eb8d0ec862616b839f3aba0a2b98e5931e3e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:30:49 -0500 Subject: [PATCH 378/916] Need some fixes in this version. --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 36e82a2b2..75a034ff5 100644 --- a/setup.py +++ b/setup.py @@ -138,8 +138,11 @@ install_requires = [ # Backported configparser for Python 2: "configparser ; python_version < '3.0'", - # For the RangeMap datastructure. - "collections-extended", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python + # 2 doesn't actually need this, since HTTP storage protocol isn't supported + # there, so we just pick whatever version so that code imports. + "collections-extended >= 2.0.2 ; python_version > '3.0'", + "collections-extended ; python_version < '3.0'", # HTTP server and client "klein", From 0346dfea60d92c9864d0e8cd9ac3ec5cb20e719b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 09:56:54 -0500 Subject: [PATCH 379/916] Note we can do this now. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result From 0ad31e33eca75467c3bb5120c7ba05b1b3795323 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 09:57:03 -0500 Subject: [PATCH 380/916] Not used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index f0ef8b12a..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From 54996185dec11b7b2c78a1f90e6a17bee9ff3ed6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:05 -0500 Subject: [PATCH 381/916] No longer used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index 43cb3e3ce..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py37-coverage - (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From e1f9f7de94c68d8c4584ea650e6d9584614b3eb7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:18 -0500 Subject: [PATCH 382/916] Note for future improvement. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result From 2583236ad8ab70e3b44bfd1bccbaf4e02aba8400 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:56:45 -0500 Subject: [PATCH 383/916] Fix unused import. --- src/allmydata/storage/http_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1d1a9466c..d79e9a38b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,7 +23,6 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header -from collections_extended import RangeMap # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads From 08911a5bddf6f159a7614894ddd04a07425e9a63 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 12:18:23 -0500 Subject: [PATCH 384/916] news fragment --- newsfragments/3867.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3867.minor diff --git a/newsfragments/3867.minor b/newsfragments/3867.minor new file mode 100644 index 000000000..e69de29bb From e482745a0bd4b0cfeb24cd782c99a4e3cfa42f57 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 12:19:24 -0500 Subject: [PATCH 385/916] drop all of the hand-rolled nix packaging expressions --- nix/autobahn.nix | 34 ---------- nix/cbor2.nix | 20 ------ nix/collections-extended.nix | 19 ------ nix/default.nix | 7 -- nix/eliot.nix | 31 --------- nix/future.nix | 35 ---------- nix/overlays.nix | 36 ---------- nix/py3.nix | 7 -- nix/pyutil.nix | 48 ------------- nix/tahoe-lafs.nix | 126 ----------------------------------- nix/twisted.nix | 63 ------------------ 11 files changed, 426 deletions(-) delete mode 100644 nix/autobahn.nix delete mode 100644 nix/cbor2.nix delete mode 100644 nix/collections-extended.nix delete mode 100644 nix/default.nix delete mode 100644 nix/eliot.nix delete mode 100644 nix/future.nix delete mode 100644 nix/overlays.nix delete mode 100644 nix/py3.nix delete mode 100644 nix/pyutil.nix delete mode 100644 nix/tahoe-lafs.nix delete mode 100644 nix/twisted.nix diff --git a/nix/autobahn.nix b/nix/autobahn.nix deleted file mode 100644 index 83148c4f8..000000000 --- a/nix/autobahn.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, isPy3k, - six, txaio, twisted, zope_interface, cffi, futures, - mock, pytest, cryptography, pynacl -}: -buildPythonPackage rec { - pname = "autobahn"; - version = "19.8.1"; - - src = fetchPypi { - inherit pname version; - sha256 = "294e7381dd54e73834354832604ae85567caf391c39363fed0ea2bfa86aa4304"; - }; - - propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++ - (lib.optionals (!isPy3k) [ futures ]); - - checkInputs = [ mock pytest ]; - checkPhase = '' - runHook preCheck - USE_TWISTED=true py.test $out - runHook postCheck - ''; - - # Tests do no seem to be compatible yet with pytest 5.1 - # https://github.com/crossbario/autobahn-python/issues/1235 - doCheck = false; - - meta = with lib; { - description = "WebSocket and WAMP in Python for Twisted and asyncio."; - homepage = "https://crossbar.io/autobahn"; - license = licenses.mit; - maintainers = with maintainers; [ nand0p ]; - }; -} diff --git a/nix/cbor2.nix b/nix/cbor2.nix deleted file mode 100644 index 16ca8ff63..000000000 --- a/nix/cbor2.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, setuptools_scm }: -buildPythonPackage rec { - pname = "cbor2"; - version = "5.2.0"; - - src = fetchPypi { - sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3"; - inherit pname version; - }; - - doCheck = false; - - propagatedBuildInputs = [ setuptools_scm ]; - - meta = with lib; { - homepage = https://github.com/agronholm/cbor2; - description = "CBOR encoder/decoder"; - license = licenses.mit; - }; -} diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix deleted file mode 100644 index 3f1ad165a..000000000 --- a/nix/collections-extended.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi }: -buildPythonPackage rec { - pname = "collections-extended"; - version = "1.0.3"; - - src = fetchPypi { - inherit pname version; - sha256 = "0lb69x23asd68n0dgw6lzxfclavrp2764xsnh45jm97njdplznkw"; - }; - - # Tests aren't in tarball, for 1.0.3 at least. - doCheck = false; - - meta = with lib; { - homepage = https://github.com/mlenzen/collections-extended; - description = "Extra Python Collections - bags (multisets), setlists (unique list / indexed set), RangeMap and IndexedDict"; - license = licenses.asl20; - }; -} diff --git a/nix/default.nix b/nix/default.nix deleted file mode 100644 index bd7460c2f..000000000 --- a/nix/default.nix +++ /dev/null @@ -1,7 +0,0 @@ -# This is the main entrypoint for the Tahoe-LAFS derivation. -{ pkgs ? import { } }: -# Add our Python packages to nixpkgs to simplify the expression for the -# Tahoe-LAFS derivation. -let pkgs' = pkgs.extend (import ./overlays.nix); -# Evaluate the expression for our Tahoe-LAFS derivation. -in pkgs'.python2.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/eliot.nix b/nix/eliot.nix deleted file mode 100644 index c5975e990..000000000 --- a/nix/eliot.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons -, hypothesis, testtools, pytest }: -buildPythonPackage rec { - pname = "eliot"; - version = "1.7.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8"; - }; - - postPatch = '' - substituteInPlace setup.py \ - --replace "boltons >= 19.0.1" boltons - ''; - - # A seemingly random subset of the test suite fails intermittently. After - # Tahoe-LAFS is ported to Python 3 we can update to a newer Eliot and, if - # the test suite continues to fail, maybe it will be more likely that we can - # have upstream fix it for us. - doCheck = false; - - checkInputs = [ testtools pytest hypothesis ]; - propagatedBuildInputs = [ zope_interface pyrsistent boltons ]; - - meta = with lib; { - homepage = https://github.com/itamarst/eliot/; - description = "Logging library that tells you why it happened"; - license = licenses.asl20; - }; -} diff --git a/nix/future.nix b/nix/future.nix deleted file mode 100644 index 814b7c1b5..000000000 --- a/nix/future.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -}: - -buildPythonPackage rec { - pname = "future"; - version = "0.18.2"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi"; - }; - - doCheck = false; - - meta = { - description = "Clean single-source support for Python 3 and 2"; - longDescription = '' - python-future is the missing compatibility layer between Python 2 and - Python 3. It allows you to use a single, clean Python 3.x-compatible - codebase to support both Python 2 and Python 3 with minimal overhead. - - It provides future and past packages with backports and forward ports - of features from Python 3 and 2. It also comes with futurize and - pasteurize, customized 2to3-based scripts that helps you to convert - either Py2 or Py3 code easily to support both Python 2 and 3 in a - single clean Py3-style codebase, module by module. - ''; - homepage = https://python-future.org; - downloadPage = https://github.com/PythonCharmers/python-future/releases; - license = with lib.licenses; [ mit ]; - maintainers = with lib.maintainers; [ prikhi ]; - }; -} diff --git a/nix/overlays.nix b/nix/overlays.nix deleted file mode 100644 index 92f36e93e..000000000 --- a/nix/overlays.nix +++ /dev/null @@ -1,36 +0,0 @@ -self: super: { - python27 = super.python27.override { - packageOverrides = python-self: python-super: { - # eliot is not part of nixpkgs at all at this time. - eliot = python-self.pythonPackages.callPackage ./eliot.nix { }; - - # NixOS autobahn package has trollius as a dependency, although - # it is optional. Trollius is unmaintained and fails on CI. - autobahn = python-super.pythonPackages.callPackage ./autobahn.nix { }; - - # Porting to Python 3 is greatly aided by the future package. A - # slightly newer version than appears in nixos 19.09 is helpful. - future = python-super.pythonPackages.callPackage ./future.nix { }; - - # Need version of pyutil that supports Python 3. The version in 19.09 - # is too old. - pyutil = python-super.pythonPackages.callPackage ./pyutil.nix { }; - - # Need a newer version of Twisted, too. - twisted = python-super.pythonPackages.callPackage ./twisted.nix { }; - - # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - - # cbor2 is not part of nixpkgs at this time. - cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { }; - }; - }; - - python39 = super.python39.override { - packageOverrides = python-self: python-super: { - # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - }; - }; -} diff --git a/nix/py3.nix b/nix/py3.nix deleted file mode 100644 index 34ede49dd..000000000 --- a/nix/py3.nix +++ /dev/null @@ -1,7 +0,0 @@ -# This is the main entrypoint for the Tahoe-LAFS derivation. -{ pkgs ? import { } }: -# Add our Python packages to nixpkgs to simplify the expression for the -# Tahoe-LAFS derivation. -let pkgs' = pkgs.extend (import ./overlays.nix); -# Evaluate the expression for our Tahoe-LAFS derivation. -in pkgs'.python39.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/pyutil.nix b/nix/pyutil.nix deleted file mode 100644 index 6852c2acc..000000000 --- a/nix/pyutil.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ stdenv -, buildPythonPackage -, fetchPypi -, setuptoolsDarcs -, setuptoolsTrial -, simplejson -, twisted -, isPyPy -}: - -buildPythonPackage rec { - pname = "pyutil"; - version = "3.3.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848"; - }; - - buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []); - propagatedBuildInputs = [ twisted ]; - - # Tests fail because they try to write new code into the twisted - # package, apparently some kind of plugin. - doCheck = false; - - prePatch = stdenv.lib.optionalString isPyPy '' - grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g" - ''; - - meta = with stdenv.lib; { - description = "Pyutil, a collection of mature utilities for Python programmers"; - - longDescription = '' - These are a few data structures, classes and functions which - we've needed over many years of Python programming and which - seem to be of general use to other Python programmers. Many of - the modules that have existed in pyutil over the years have - subsequently been obsoleted by new features added to the - Python language or its standard library, thus showing that - we're not alone in wanting tools like these. - ''; - - homepage = "http://allmydata.org/trac/pyutil"; - license = licenses.gpl2Plus; - }; - -} \ No newline at end of file diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix deleted file mode 100644 index 2b41e676e..000000000 --- a/nix/tahoe-lafs.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ fetchFromGitHub, lib -, git, python -, twisted, foolscap, zfec -, setuptools, setuptoolsTrial, pyasn1, zope_interface -, service-identity, pyyaml, magic-wormhole, treq, appdirs -, beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, cbor2 -}: -python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.17.1). - # Give it a `post` component to make it look newer than the release version - # and we'll bump this up at the time of each release. - # - # It's difficult to read the version from Git the way the Python code does - # for two reasons. First, doing so involves populating the Nix expression - # with values from the source. Nix calls this "import from derivation" or - # "IFD" (). This is - # discouraged in most cases - including this one, I think. Second, the - # Python code reads the contents of `.git` to determine its version. `.git` - # is not a reproducable artifact (in the sense of "reproducable builds") so - # it is excluded from the source tree by default. When it is included, the - # package tends to be frequently spuriously rebuilt. - version = "1.17.1.post1"; - name = "tahoe-lafs-${version}"; - src = lib.cleanSourceWith { - src = ../.; - filter = name: type: - let - basename = baseNameOf name; - - split = lib.splitString "."; - join = builtins.concatStringsSep "."; - ext = join (builtins.tail (split basename)); - - # Build up a bunch of knowledge about what kind of file this is. - isTox = type == "directory" && basename == ".tox"; - isTrialTemp = type == "directory" && basename == "_trial_temp"; - isVersion = basename == "_version.py"; - isBytecode = ext == "pyc" || ext == "pyo"; - isBackup = lib.hasSuffix "~" basename; - isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename; - isSymlink = type == "symlink"; - isGit = type == "directory" && basename == ".git"; - in - # Exclude all these things - ! (isTox - || isTrialTemp - || isVersion - || isBytecode - || isBackup - || isTemporary - || isSymlink - || isGit - ); - }; - - postPatch = '' - # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around - # that. - for i in $(find src/allmydata/test -type f) - do - sed -i "$i" -e"s/localhost/127.0.0.1/g" - done - - # Some tests are flaky or fail to skip when dependencies are missing. - # This list is over-zealous because it's more work to disable individual - # tests with in a module. - - # Many of these tests don't properly skip when i2p or tor dependencies are - # not supplied (and we are not supplying them). - rm src/allmydata/test/test_i2p_provider.py - rm src/allmydata/test/test_connections.py - rm src/allmydata/test/cli/test_create.py - - # Generate _version.py ourselves since we can't rely on the Python code - # extracting the information from the .git directory we excluded. - cat > src/allmydata/_version.py < /dev/null - ''; - - checkPhase = '' - ${python.interpreter} -m unittest discover -s twisted/test - ''; - # Tests require network - doCheck = false; - - meta = with stdenv.lib; { - homepage = https://twistedmatrix.com/; - description = "Twisted, an event-driven networking engine written in Python"; - longDescription = '' - Twisted is an event-driven networking engine written in Python - and licensed under the MIT license. - ''; - license = licenses.mit; - maintainers = [ ]; - }; -} From 8a1d4617c23b8addf2955931ef1a3e234de916b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:00:36 -0500 Subject: [PATCH 386/916] Add new Nix packaging using mach-nix.buildPythonPackage --- default.nix | 52 ++++++++++++++ nix/sources.json | 62 +++++++++++++++++ nix/sources.nix | 174 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 288 insertions(+) create mode 100644 default.nix create mode 100644 nix/sources.json create mode 100644 nix/sources.nix diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..4c9ed1cc9 --- /dev/null +++ b/default.nix @@ -0,0 +1,52 @@ +let + sources = import nix/sources.nix; +in +{ pkgsVersion ? "nixpkgs-21.11" +, pkgs ? import sources.${pkgsVersion} { } +, pypiData ? sources.pypi-deps-db +, pythonVersion ? "python37" +, mach-nix ? import sources.mach-nix { + inherit pkgs pypiData; + python = pythonVersion; + } +}: +# The project name, version, and most other metadata are automatically +# extracted from the source. Some requirements are not properly extracted +# and those cases are handled below. The version can only be extracted if +# `setup.py update_version` has been run (this is not at all ideal but it +# seems difficult to fix) - so for now just be sure to run that first. +mach-nix.buildPythonPackage { + # Define the location of the Tahoe-LAFS source to be packaged. Clean up all + # as many of the non-source files (eg the `.git` directory, `~` backup + # files, nix's own `result` symlink, etc) as possible to avoid needing to + # re-build when files that make no difference to the package have changed. + src = pkgs.lib.cleanSource ./.; + + # Define some extra requirements that mach-nix does not automatically detect + # from inspection of the source. We typically don't need to put version + # constraints on any of these requirements. The pypi-deps-db we're + # operating with makes dependency resolution deterministic so as long as it + # works once it will always work. It could be that in the future we update + # pypi-deps-db and an incompatibility arises - in which case it would make + # sense to apply some version constraints here. + requirementsExtra = '' + # mach-nix does not yet support pyproject.toml which means it misses any + # build-time requirements of our dependencies which are declared in such a + # file. Tell it about them here. + setuptools_rust + + # mach-nix does not yet parse environment markers correctly. It misses + # all of our requirements which have an environment marker. Duplicate them + # here. + foolscap + eliot + pyrsistent + ''; + + providers = { + # Through zfec 1.5.5 the wheel has an incorrect runtime dependency + # declared on argparse, not available for recent versions of Python 3. + # Force mach-nix to use the sdist instead, side-stepping this issue. + zfec = "sdist"; + }; +} diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 000000000..1169911e2 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,62 @@ +{ + "mach-nix": { + "branch": "master", + "description": "Create highly reproducible python environments", + "homepage": "", + "owner": "davhau", + "repo": "mach-nix", + "rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b", + "sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf", + "type": "tarball", + "url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "niv": { + "branch": "master", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "nmattia", + "repo": "niv", + "rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c", + "sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm", + "type": "tarball", + "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "release-20.03", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "eb73405ecceb1dc505b7cbbd234f8f94165e2696", + "sha256": "06k21wbyhhvq2f1xczszh3c2934p0m02by3l2ixvd6nkwrqklax7", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb73405ecceb1dc505b7cbbd234f8f94165e2696.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-21.11": { + "branch": "nixos-21.11", + "description": "Nix Packages collection", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", + "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "pypi-deps-db": { + "branch": "master", + "description": "Probably the most complete python dependency database", + "homepage": "", + "owner": "DavHau", + "repo": "pypi-deps-db", + "rev": "0f6de8bf1f186c275af862ec9667abb95aae8542", + "sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y", + "type": "tarball", + "url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 000000000..1938409dd --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,174 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + if spec ? ref then spec.ref else + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + in + builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else {}; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } From fae80e5da9c558a4700e1f4d78169359ebd02d0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:23:54 -0500 Subject: [PATCH 387/916] Fix zfec packaging --- default.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 4c9ed1cc9..b6918ecb7 100644 --- a/default.nix +++ b/default.nix @@ -46,7 +46,21 @@ mach-nix.buildPythonPackage { providers = { # Through zfec 1.5.5 the wheel has an incorrect runtime dependency # declared on argparse, not available for recent versions of Python 3. - # Force mach-nix to use the sdist instead, side-stepping this issue. + # Force mach-nix to use the sdist instead. This allows us to apply a + # patch that removes the offending declaration. zfec = "sdist"; }; + + # Define certain overrides to the way Python dependencies are built. + _ = { + # Apply the argparse declaration fix to zfec sdist. + zfec.patches = with pkgs; [ + (fetchpatch { + name = "fix-argparse.patch"; + url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch"; + sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; + }) + ]; + }; + } From c21ca210e35cb40e79105507a9e91675a9fcfef0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:23:59 -0500 Subject: [PATCH 388/916] a note about providers --- default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/default.nix b/default.nix index b6918ecb7..970fd75ca 100644 --- a/default.nix +++ b/default.nix @@ -43,6 +43,9 @@ mach-nix.buildPythonPackage { pyrsistent ''; + # Specify where mach-nix should find packages for our Python dependencies. + # There are some reasonable defaults so we only need to specify certain + # packages where the default configuration runs into some issue. providers = { # Through zfec 1.5.5 the wheel has an incorrect runtime dependency # declared on argparse, not available for recent versions of Python 3. From 86bcfaa14d3378802a441421649a42b8c7ac3cfd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:24:05 -0500 Subject: [PATCH 389/916] Update CircleCI configuration to the new packaging --- .circleci/config.yml | 21 +++++++++------------ nix/sources.json | 12 ++++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a650313ed..6fa1106fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,10 +39,10 @@ workflows: - "centos-8": {} - - "nixos-19-09": + - "nixos-21-05": {} - - "nixos-21-05": + - "nixos-21-11": {} # Test against PyPy 2.7 @@ -441,15 +441,16 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-19-09: &NIXOS + nixos-21.05: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" - SOURCE: "nix/" + # Reference the name of a niv-managed nixpkgs source (see `niv show` and + # nix/sources.json) + NIXPKGS: "nixpkgs-21.05" steps: - "checkout" @@ -466,17 +467,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 "$SOURCE" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "$NIXPKGS" - nixos-21-05: + nixos-21-11: <<: *NIXOS environment: - # Note this doesn't look more similar to the 19.09 NIX_PATH URL because - # there was some internal shuffling by the NixOS project about how they - # publish stable revisions. - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz" - SOURCE: "nix/py3.nix" + NIXPKGS: "nixpkgs-21.11" typechecks: docker: diff --git a/nix/sources.json b/nix/sources.json index 1169911e2..e0235a3fb 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -23,23 +23,23 @@ "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs": { - "branch": "release-20.03", + "nixpkgs-21.05": { + "branch": "nixos-21.05", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "eb73405ecceb1dc505b7cbbd234f8f94165e2696", - "sha256": "06k21wbyhhvq2f1xczszh3c2934p0m02by3l2ixvd6nkwrqklax7", + "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00", + "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/eb73405ecceb1dc505b7cbbd234f8f94165e2696.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.11": { "branch": "nixos-21.11", "description": "Nix Packages collection", "homepage": "", - "owner": "nixos", + "owner": "NixOS", "repo": "nixpkgs", "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", From b47457646c4a6e1c2a83577628b0e0b13ab39d77 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:26:57 -0500 Subject: [PATCH 390/916] Correct naming of the CircleCI job --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fa1106fc..50191555f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -441,7 +441,7 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-21.05: &NIXOS + nixos-21-05: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH From 9c964f4acd46d71d03a4b5e753b439bc9b63cc88 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:52:10 -0500 Subject: [PATCH 391/916] generate the version info --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50191555f..9d9b967f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,6 +454,12 @@ jobs: steps: - "checkout" + - "run": + name: "Generation version" + command: | + # The Nix package doesn't know how to do this part, unfortunately. + nix-shell -p python --run 'python setup.py update_version' + - "run": name: "Build and Test" command: | From 5cab1f7a4c147b6079583582231b021e5a837d7a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:57:09 -0500 Subject: [PATCH 392/916] Get Python this way? --- .circleci/config.yml | 2 +- .circleci/python.nix | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .circleci/python.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d9b967f1..e4d6c9e93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,7 +458,7 @@ jobs: name: "Generation version" command: | # The Nix package doesn't know how to do this part, unfortunately. - nix-shell -p python --run 'python setup.py update_version' + nix-shell .circleci/python.nix --run 'python setup.py update_version' - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/python.nix new file mode 100644 index 000000000..a830ee61b --- /dev/null +++ b/.circleci/python.nix @@ -0,0 +1,11 @@ +# Define a helper environment for incidental Python tasks required on CI. +let + sources = import ../nix/sources.nix; +in +{ pkgs ? import sources."nixpkgs-21.11" { } +}: +pkgs.mkShell { + buildInputs = [ + pkgs.python3 + ]; +} From dea4c7e131c097f875118f363c3c6e4472b8dc86 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:59:32 -0500 Subject: [PATCH 393/916] get setuptools --- .circleci/python.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/python.nix b/.circleci/python.nix index a830ee61b..6e3d79cc1 100644 --- a/.circleci/python.nix +++ b/.circleci/python.nix @@ -6,6 +6,8 @@ in }: pkgs.mkShell { buildInputs = [ - pkgs.python3 + (pkgs.python3.withPackages (ps: [ + ps.setuptools + ])) ]; } From 013e1810e4b0c94bf89701980f830490f1c45e22 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:59:37 -0500 Subject: [PATCH 394/916] try to use a single nixpkgs in each job --- .circleci/config.yml | 6 ++++-- .circleci/python.nix | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4d6c9e93..f76197bd7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,9 +456,11 @@ jobs: - "checkout" - "run": name: "Generation version" - command: | + command: >- # The Nix package doesn't know how to do this part, unfortunately. - nix-shell .circleci/python.nix --run 'python setup.py update_version' + nix-shell .circleci/python.nix + --argstr pkgsVersion "$NIXPKGS" + --run 'python setup.py update_version' - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/python.nix index 6e3d79cc1..ecaf9e27c 100644 --- a/.circleci/python.nix +++ b/.circleci/python.nix @@ -2,7 +2,8 @@ let sources = import ../nix/sources.nix; in -{ pkgs ? import sources."nixpkgs-21.11" { } +{ pkgsVersion +, pkgs ? import sources.${pkgsVersion} { } }: pkgs.mkShell { buildInputs = [ From 78c4b98b086241b43b08d8e7e2c531c1e63133f6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:01:40 -0500 Subject: [PATCH 395/916] that comment handles the >- yaml string type badly --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f76197bd7..01aa75e80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -455,9 +455,9 @@ jobs: steps: - "checkout" - "run": + # The Nix package doesn't know how to do this part, unfortunately. name: "Generation version" command: >- - # The Nix package doesn't know how to do this part, unfortunately. nix-shell .circleci/python.nix --argstr pkgsVersion "$NIXPKGS" --run 'python setup.py update_version' From 5b7f5a9f889c77c2946bfb0027e3a298f9338d33 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:04:21 -0500 Subject: [PATCH 396/916] fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01aa75e80..406a8f200 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,7 +456,7 @@ jobs: - "checkout" - "run": # The Nix package doesn't know how to do this part, unfortunately. - name: "Generation version" + name: "Generate version" command: >- nix-shell .circleci/python.nix --argstr pkgsVersion "$NIXPKGS" From b2acd0f7d0192bb9c03291e5fda56f9d76f3f43c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:05:59 -0500 Subject: [PATCH 397/916] >- and indentation changes don't interact well blackslashes are more likely to be understood, I guess --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 406a8f200..55c5730a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -457,9 +457,9 @@ jobs: - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" - command: >- - nix-shell .circleci/python.nix - --argstr pkgsVersion "$NIXPKGS" + command: | + nix-shell .circleci/python.nix \ + --argstr pkgsVersion "$NIXPKGS" \ --run 'python setup.py update_version' - "run": From 83a172210c1edf4298aedaed08c73371c22743ae Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:22:35 -0500 Subject: [PATCH 398/916] Switch to Nix 2.3. mach-nix is not compatible with older versions. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55c5730a5..3752fb7c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -445,7 +445,7 @@ jobs: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixorg/nix:circleci" + image: "nixos/nix:2.3.16" environment: # Reference the name of a niv-managed nixpkgs source (see `niv show` and From 5edd96ce6b018f93de2914e5c7a83d48e45f6998 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:31:56 -0500 Subject: [PATCH 399/916] Change around environment management so we can install ssh too The new image does not come with it --- .circleci/config.yml | 14 +++++++++++--- .circleci/{python.nix => env.nix} | 11 +++++------ 2 files changed, 16 insertions(+), 9 deletions(-) rename .circleci/{python.nix => env.nix} (63%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3752fb7c0..499bb16b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -453,14 +453,22 @@ jobs: NIXPKGS: "nixpkgs-21.05" steps: + - "run": + name: "Install Basic Dependencies" + command: | + nix-env \ + -f .circleci/env.nix \ + --argstr pkgsVersion "$NIXPKGS" \ + --install \ + -A ssh python3 + - "checkout" + - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" command: | - nix-shell .circleci/python.nix \ - --argstr pkgsVersion "$NIXPKGS" \ - --run 'python setup.py update_version' + python setup.py update_version - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/env.nix similarity index 63% rename from .circleci/python.nix rename to .circleci/env.nix index ecaf9e27c..0225b00c8 100644 --- a/.circleci/python.nix +++ b/.circleci/env.nix @@ -5,10 +5,9 @@ in { pkgsVersion , pkgs ? import sources.${pkgsVersion} { } }: -pkgs.mkShell { - buildInputs = [ - (pkgs.python3.withPackages (ps: [ - ps.setuptools - ])) - ]; +{ + ssh = pkgs.openssh; + python = pkgs.python3.withPackages (ps: [ + ps.setuptools + ]); } From e7bba3dad0909c4c5585270de1deba8d53eae643 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:36:59 -0500 Subject: [PATCH 400/916] cannot use the source before we do the checkout... --- .circleci/config.yml | 11 +++++------ .circleci/env.nix | 13 ------------- 2 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 .circleci/env.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 499bb16b6..21e560a89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,17 +450,16 @@ jobs: environment: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) - NIXPKGS: "nixpkgs-21.05" + NIXPKGS: "21.05" steps: - "run": name: "Install Basic Dependencies" command: | nix-env \ - -f .circleci/env.nix \ - --argstr pkgsVersion "$NIXPKGS" \ + -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ --install \ - -A ssh python3 + -A git openssh python3 - "checkout" @@ -483,13 +482,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "$NIXPKGS" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" nixos-21-11: <<: *NIXOS environment: - NIXPKGS: "nixpkgs-21.11" + NIXPKGS: "21.11" typechecks: docker: diff --git a/.circleci/env.nix b/.circleci/env.nix deleted file mode 100644 index 0225b00c8..000000000 --- a/.circleci/env.nix +++ /dev/null @@ -1,13 +0,0 @@ -# Define a helper environment for incidental Python tasks required on CI. -let - sources = import ../nix/sources.nix; -in -{ pkgsVersion -, pkgs ? import sources.${pkgsVersion} { } -}: -{ - ssh = pkgs.openssh; - python = pkgs.python3.withPackages (ps: [ - ps.setuptools - ]); -} From e4ed98fa64a55984098dc8e147d7eca150c8e366 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:39:30 -0500 Subject: [PATCH 401/916] maybe this is where they may be found --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21e560a89..ed45e3c3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ --install \ - -A git openssh python3 + -A nixos.git nixos.openssh nixos.python3 - "checkout" From 7ee55d07e570af74aceb59fac88c6fccad13a2b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:47:43 -0500 Subject: [PATCH 402/916] Use nix-env less wrong, maybe --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed45e3c3b..294eacaf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -457,9 +457,9 @@ jobs: name: "Install Basic Dependencies" command: | nix-env \ - -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ + --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A nixos.git nixos.openssh nixos.python3 + -A git openssh python3 - "checkout" From 17d2119521b4adbb6d438e15e596f46494f663bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:55:34 -0500 Subject: [PATCH 403/916] get setuptools in there --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 294eacaf4..de1966c86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh python3 + -A git openssh 'python3.withPackages (ps: [ ps.setuptools ])' - "checkout" From a8033e2c2f734477aa9882f858477be7cb4266fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:59:29 -0500 Subject: [PATCH 404/916] cannot get python env that way we don't need python until later anyway --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index de1966c86..2b72a4e78 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh 'python3.withPackages (ps: [ ps.setuptools ])' + -A git openssh - "checkout" @@ -467,7 +467,9 @@ jobs: # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" command: | - python setup.py update_version + nix-shell \ + -p 'python3.withPackages (ps: [ ps.setuptools ])' \ + --run 'python setup.py update_version' - "run": name: "Build and Test" From 0fb56c9a4890347a1124fcb98f6b459f46699e3a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:03:21 -0500 Subject: [PATCH 405/916] I checked, git is there. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b72a4e78..11136e04e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,12 +454,14 @@ jobs: steps: - "run": + # The nixos/nix image does not include ssh. Install it so the + # `checkout` step will succeed. name: "Install Basic Dependencies" command: | nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh + -A openssh - "checkout" From 136734c198d5bec305b1763aec41a36b321e3308 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:09:52 -0500 Subject: [PATCH 406/916] try to use cachix --- .circleci/config.yml | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11136e04e..7153fd370 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,20 +451,33 @@ jobs: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) NIXPKGS: "21.05" + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us + # to push to CACHIX_NAME. + CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": # The nixos/nix image does not include ssh. Install it so the - # `checkout` step will succeed. + # `checkout` step will succeed. We also want cachix for + # Nix-friendly caching. name: "Install Basic Dependencies" command: | nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A openssh + -A openssh cachix bash - "checkout" + - run: + name: "Cachix setup" + # Record the store paths that exist before we did much. There's no + # reason to cache these, they're either in the image or have to be + # retrieved before we can use cachix to restore from cache. + command: | + cachix use "${CACHIX_NAME}" + nix path-info --all > /tmp/store-path-pre-build + - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" @@ -488,6 +501,26 @@ jobs: # them in parallel. nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" + - run: + # Send any new store objects to cachix. + name: "Push to Cachix" + when: "always" + command: | + # Cribbed from + # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/ + if [ -n "$CIRCLE_PR_NUMBER" ]; then + # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the + # number of the PR being built". Sorry, dear reader, you have + # guessed poorly. It is also conditionally set based on whether + # this is a PR from a fork or not. + # + # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables + echo "Skipping Cachix push for forked PR." + else + # https://docs.cachix.org/continuous-integration-setup/circleci.html + bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" + fi + nixos-21-11: <<: *NIXOS From ccb6e65c0453ec38f582ae5c1163dee9ab49495e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:26:19 -0500 Subject: [PATCH 407/916] make sure CACHIX_NAME is set for both nixos jobs --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7153fd370..8e860d497 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,9 +451,6 @@ jobs: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) NIXPKGS: "21.05" - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us - # to push to CACHIX_NAME. - CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": @@ -471,6 +468,11 @@ jobs: - run: name: "Cachix setup" + environment: + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" # Record the store paths that exist before we did much. There's no # reason to cache these, they're either in the image or have to be # retrieved before we can use cachix to restore from cache. From f5e1af00c0689e60a54931fa0720f031a7e83f06 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:35:23 -0500 Subject: [PATCH 408/916] try using parameters to avoid environment collision the `cachix push` later on also needs CACHIX_NAME so defining it on a single step is not great --- .circleci/config.yml | 47 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e860d497..1ef0e820d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,11 +39,11 @@ workflows: - "centos-8": {} - - "nixos-21-05": - {} + - "nixos": + nixpkgs: "21.05" - - "nixos-21-11": - {} + - "nixos": + nixpkgs: "21.11" # Test against PyPy 2.7 - "pypy27-buster": @@ -441,16 +441,24 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-21-05: &NIXOS + nixos: + parameters: + nixpkgs: + description: >- + Reference the name of a niv-managed nixpkgs source (see `niv show` + and nix/sources.json) + type: "string" + docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.3.16" environment: - # Reference the name of a niv-managed nixpkgs source (see `niv show` and - # nix/sources.json) - NIXPKGS: "21.05" + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": @@ -460,7 +468,7 @@ jobs: name: "Install Basic Dependencies" command: | nix-env \ - --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ + --file https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz \ --install \ -A openssh cachix bash @@ -468,11 +476,6 @@ jobs: - run: name: "Cachix setup" - environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and - # allows us to push to CACHIX_NAME. We only need this set for - # `cachix use` in this step. - CACHIX_NAME: "tahoe-lafs-opensource" # Record the store paths that exist before we did much. There's no # reason to cache these, they're either in the image or have to be # retrieved before we can use cachix to restore from cache. @@ -489,7 +492,7 @@ jobs: --run 'python setup.py update_version' - "run": - name: "Build and Test" + name: "Build" command: | # CircleCI build environment looks like it has a zillion and a # half cores. Don't let Nix autodetect this high core count @@ -501,7 +504,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<>" + + - "run": + name: "Test" + command: | + # Let it go somewhat wild for the test suite itself + nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<>" tests.nix - run: # Send any new store objects to cachix. @@ -523,12 +532,6 @@ jobs: bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" fi - nixos-21-11: - <<: *NIXOS - - environment: - NIXPKGS: "21.11" - typechecks: docker: - <<: *DOCKERHUB_AUTH From 6154be1a96c967bacc5f507ccac4e6dbe003e737 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:37:12 -0500 Subject: [PATCH 409/916] Give the NixOS job instantiations nice names --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ef0e820d..3b76c0fb9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,11 @@ workflows: {} - "nixos": + name: "NixOS 21.05" nixpkgs: "21.05" - "nixos": + name: "NixOS 21.11" nixpkgs: "21.11" # Test against PyPy 2.7 From 60dd2ee413dcc5f2b2dc2bc484a9fc6a9a73e5d7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:17:37 -0500 Subject: [PATCH 410/916] Document the parameters and also accept an `extras` parameter --- default.nix | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/default.nix b/default.nix index 970fd75ca..d5acbbdd7 100644 --- a/default.nix +++ b/default.nix @@ -1,14 +1,27 @@ let sources = import nix/sources.nix; in -{ pkgsVersion ? "nixpkgs-21.11" -, pkgs ? import sources.${pkgsVersion} { } -, pypiData ? sources.pypi-deps-db -, pythonVersion ? "python37" -, mach-nix ? import sources.mach-nix { +{ + pkgsVersion ? "nixpkgs-21.11" # a string which choses a nixpkgs from the + # niv-managed sources data + +, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself + +, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use + # for dependency resolution + +, pythonVersion ? "python37" # a string chosing the python derivation from + # nixpkgs to target + +, extras ? [] # a list of strings identifying tahoe-lafs extras, the + # dependencies of which the resulting package will also depend + # on + +, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build + # the tahoe-lafs package inherit pkgs pypiData; python = pythonVersion; - } +} }: # The project name, version, and most other metadata are automatically # extracted from the source. Some requirements are not properly extracted @@ -22,6 +35,9 @@ mach-nix.buildPythonPackage { # re-build when files that make no difference to the package have changed. src = pkgs.lib.cleanSource ./.; + # Select whichever package extras were requested. + inherit extras; + # Define some extra requirements that mach-nix does not automatically detect # from inspection of the source. We typically don't need to put version # constraints on any of these requirements. The pypi-deps-db we're From f03f5fb8d7d3205c25edc94baee2c8886b3092cb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:18:43 -0500 Subject: [PATCH 411/916] Add an expression for running the test suite --- default.nix | 6 +++++- tests.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests.nix diff --git a/default.nix b/default.nix index d5acbbdd7..044f59e7b 100644 --- a/default.nix +++ b/default.nix @@ -80,6 +80,10 @@ mach-nix.buildPythonPackage { sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; }) ]; - }; + # Remove a click-default-group patch for a test suite problem which no + # longer applies because the project apparently no longer has a test suite + # in its source distribution. + click-default-group.patches = []; + }; } diff --git a/tests.nix b/tests.nix new file mode 100644 index 000000000..364407e87 --- /dev/null +++ b/tests.nix @@ -0,0 +1,26 @@ +let + sources = import nix/sources.nix; +in +# See default.nix for documentation about parameters. +{ pkgsVersion ? "nixpkgs-21.11" +, pkgs ? import sources.${pkgsVersion} { } +, pypiData ? sources.pypi-deps-db +, pythonVersion ? "python37" +, mach-nix ? import sources.mach-nix { + inherit pkgs pypiData; + python = pythonVersion; + } +}@args: +let + # Get the package with all of its test requirements. + tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); + + # Put it into a Python environment. + python-env = pkgs.${pythonVersion}.withPackages (ps: [ + tahoe-lafs + ]); +in +# Make a derivation that runs the unit test suite. +pkgs.runCommand "tahoe-lafs-tests" { } '' + ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata +'' From 16fd427b153551f139a39c5e9d371a2611da3a39 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:27:10 -0500 Subject: [PATCH 412/916] Get undetected txi2p-tahoe test dependency into the test environment --- default.nix | 6 +++++- tests.nix | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 044f59e7b..03ab89a4e 100644 --- a/default.nix +++ b/default.nix @@ -28,7 +28,7 @@ in # and those cases are handled below. The version can only be extracted if # `setup.py update_version` has been run (this is not at all ideal but it # seems difficult to fix) - so for now just be sure to run that first. -mach-nix.buildPythonPackage { +mach-nix.buildPythonPackage rec { # Define the location of the Tahoe-LAFS source to be packaged. Clean up all # as many of the non-source files (eg the `.git` directory, `~` backup # files, nix's own `result` symlink, etc) as possible to avoid needing to @@ -86,4 +86,8 @@ mach-nix.buildPythonPackage { # in its source distribution. click-default-group.patches = []; }; + + passthru.meta.mach-nix = { + inherit providers _; + }; } diff --git a/tests.nix b/tests.nix index 364407e87..5b6eae497 100644 --- a/tests.nix +++ b/tests.nix @@ -16,9 +16,15 @@ let tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); # Put it into a Python environment. - python-env = pkgs.${pythonVersion}.withPackages (ps: [ - tahoe-lafs - ]); + python-env = mach-nix.mkPython { + inherit (tahoe-lafs.meta.mach-nix) providers _; + packagesExtra = [ tahoe-lafs ]; + requirements = '' + # txi2p-tahoe is another dependency with an environment marker that + # mach-nix doesn't automatically pick up. + txi2p-tahoe + ''; + }; in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' From f5de5fc1271dc434e044fc3c04f523a031b80b05 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:41:38 -0500 Subject: [PATCH 413/916] produce and output so the build appears to be a success --- tests.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests.nix b/tests.nix index 5b6eae497..3816d6ad8 100644 --- a/tests.nix +++ b/tests.nix @@ -29,4 +29,14 @@ in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata + + # It's not cool to put the whole _trial_temp into $out because it has weird + # files in it we don't want in the store. Plus, even all of the less weird + # files are mostly just trash that's not meaningful if the test suite passes + # (which is the only way we get $out anyway). + # + # The build log itself is typically available from `nix-store --read-log` so + # we don't need to record that either. + echo "passed" >$out + '' From e4505cd7b88c981983229fc5f040bc02d4eb3f66 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 09:56:57 -0500 Subject: [PATCH 414/916] change the strategy for building the test environment it's not clear to me if this is conceptually better or worse than what it replaces but it is about 25% faster --- default.nix | 9 +++++--- tests.nix | 61 ++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/default.nix b/default.nix index 03ab89a4e..7abaa4c5a 100644 --- a/default.nix +++ b/default.nix @@ -13,9 +13,12 @@ in , pythonVersion ? "python37" # a string chosing the python derivation from # nixpkgs to target -, extras ? [] # a list of strings identifying tahoe-lafs extras, the - # dependencies of which the resulting package will also depend - # on +, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, + # the dependencies of which the resulting package + # will also depend on. Include all of the runtime + # extras by default because the incremental cost of + # including them is a lot smaller than the cost of + # re-building the whole thing to add them. , mach-nix ? import sources.mach-nix { # the mach-nix package to use to build # the tahoe-lafs package diff --git a/tests.nix b/tests.nix index 3816d6ad8..53a8885c0 100644 --- a/tests.nix +++ b/tests.nix @@ -12,17 +12,62 @@ in } }@args: let - # Get the package with all of its test requirements. - tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); + # We would like to know the test requirements but mach-nix does not directly + # expose this information to us. However, it is perfectly capable of + # determining it if we ask right... This is probably not meant to be a + # public mach-nix API but we pinned mach-nix so we can deal with mach-nix + # upgrade breakage in our own time. + mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" { + inherit pkgs; + lib = pkgs.lib; + }; + tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test; - # Put it into a Python environment. + # Get the Tahoe-LAFS package itself. This does not include test + # requirements and we don't ask for test requirements so that we can just + # re-use the normal package if it is already built. + tahoe-lafs = import ./. args; + + # If we want to get tahoe-lafs into a Python environment with a bunch of + # *other* Python modules and let them interact in the usual way then we have + # to ask mach-nix for tahoe-lafs and those other Python modules in the same + # way - i.e., using `requirements`. The other tempting mechanism, + # `packagesExtra`, inserts an extra layer of Python environment and prevents + # normal interaction between Python modules (as well as usually producing + # file collisions in the packages that are both runtime and test + # dependencies). To get the tahoe-lafs we just built into the environment, + # put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs + # from nixpkgs. + overridesPre = [(self: super: { inherit tahoe-lafs; })]; + providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; }; + + # Make the Python environment in which we can run the tests. python-env = mach-nix.mkPython { - inherit (tahoe-lafs.meta.mach-nix) providers _; - packagesExtra = [ tahoe-lafs ]; + # Get the packaging fixes we already know we need from putting together + # the runtime package. + inherit (tahoe-lafs.meta.mach-nix) _; + # Share the runtime package's provider configuration - combined with our + # own that causes the right tahoe-lafs to be picked up. + inherit providers overridesPre; requirements = '' - # txi2p-tahoe is another dependency with an environment marker that - # mach-nix doesn't automatically pick up. - txi2p-tahoe + # Here we pull in the Tahoe-LAFS package itself. + tahoe-lafs + + # Unfortunately mach-nix misses all of the Python dependencies of the + # tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a + # bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will + # fix this soon. + # + # https://github.com/DavHau/mach-nix/issues/123 + # https://github.com/DavHau/mach-nix/pull/386 + ${tahoe-lafs.requirements} + + # And then all of the test-only dependencies. + ${builtins.concatStringsSep "\n" tests_require} + + # txi2p-tahoe is another dependency with an environment marker that + # mach-nix doesn't automatically pick up. + txi2p-tahoe ''; }; in From d6e82d1d56e1a6787645c134de66f64b1048aa83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:37:43 -0500 Subject: [PATCH 415/916] explain this unfortunate cache step --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b76c0fb9..daf985567 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -530,7 +530,24 @@ jobs: # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables echo "Skipping Cachix push for forked PR." else + # If this *isn't* a build from a fork then we have the Cachix + # write key in our environment and we can push any new objects + # to Cachix. + # + # To decide what to push, we inspect the list of store objects + # that existed before and after we did most of our work. Any + # that are new after the work is probably a useful thing to have + # around so push it to the cache. We exclude all derivation + # objects (.drv files) because they're cheap to reconstruct and + # by the time you know their cache key you've already done all + # the work anyway. + # + # This shell expression for finding the objects and pushing them + # was from the Cachix docs: + # # https://docs.cachix.org/continuous-integration-setup/circleci.html + # + # but they seem to have removed it now. bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" fi From 005a7622699c74c594ecdbe2b5be53cfa84ba8e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:46:10 -0500 Subject: [PATCH 416/916] spelling --- default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 7abaa4c5a..cecb5579a 100644 --- a/default.nix +++ b/default.nix @@ -2,7 +2,7 @@ let sources = import nix/sources.nix; in { - pkgsVersion ? "nixpkgs-21.11" # a string which choses a nixpkgs from the + pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the # niv-managed sources data , pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself @@ -10,7 +10,7 @@ in , pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use # for dependency resolution -, pythonVersion ? "python37" # a string chosing the python derivation from +, pythonVersion ? "python37" # a string choosing the python derivation from # nixpkgs to target , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, From 9ba17ba8d1731ba004587144d0e5b0b63f870fcd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:46:13 -0500 Subject: [PATCH 417/916] explain sources.nix a bit --- default.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/default.nix b/default.nix index cecb5579a..a8a7ba1c8 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,23 @@ let + # sources.nix contains information about which versions of some of our + # dependencies we should use. since we use it to pin nixpkgs and the PyPI + # package database, roughly all the rest of our dependencies are *also* + # pinned - indirectly. + # + # sources.nix is managed using a tool called `niv`. as an example, to + # update to the most recent version of nixpkgs from the 21.11 maintenance + # release, in the top-level tahoe-lafs checkout directory you run: + # + # niv update nixpkgs-21.11 + # + # or, to update the PyPI package database -- which is necessary to make any + # newly released packages visible -- you likewise run: + # + # niv update pypi-deps-db + # + # niv also supports chosing a specific revision, following a different + # branch, etc. find complete documentation for the tool at + # https://github.com/nmattia/niv sources = import nix/sources.nix; in { From d23fdcdb8ab32bd11530db344b0448ecaae2b041 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jan 2022 12:03:17 -0500 Subject: [PATCH 418/916] Sketch of first IStorageServer test with HTTP server/client. --- src/allmydata/storage_client.py | 38 ++++++++++++ src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++++-- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 526e4e70d..13b4dfd01 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -75,6 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict +from allmydata.storage.http_client import StorageClient # who is responsible for de-duplication? @@ -1024,3 +1025,40 @@ class _StorageServer(object): shnum, reason, ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + + +# WORK IN PROGRESS, for now it doesn't actually implement whole thing. +@implementer(IStorageServer) # type: ignore +@attr.s +class _HTTPStorageServer(object): + """ + Talk to remote storage server over HTTP. + """ + _http_client = attr.ib(type=StorageClient) + + @staticmethod + def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer + """ + Create an ``IStorageServer`` from a HTTP ``StorageClient``. + """ + return _HTTPStorageServer(_http_client=http_client) + + def get_version(self): + return self._http_client.get_version() + + def allocate_buckets( + self, + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ): + pass + + def get_buckets( + self, + storage_index, + ): + pass diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a17264713..fb9765624 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1,6 +1,8 @@ """ Tests for the ``IStorageServer`` interface. +Keep in mind that ``IStorageServer`` is actually the storage _client_ interface. + Note that for performance, in the future we might want the same node to be reused across tests, so each test should be careful to generate unique storage indexes. @@ -22,6 +24,10 @@ from random import Random from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock +from twisted.internet import reactor +from twisted.web.server import Site +from hyperlink import DecodedURL +from treq.api import get_global_pool as get_treq_pool from foolscap.api import Referenceable, RemoteException @@ -29,6 +35,10 @@ from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! +from allmydata.storage.http_server import HTTPServer +from allmydata.storage.http_client import StorageClient +from allmydata.util.iputil import allocate_tcp_port + # Use random generator with known seed, so results are reproducible if tests # are run in the same order. @@ -998,11 +1008,11 @@ class IStorageServerMutableAPIsTestsMixin(object): self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) -class _FoolscapMixin(SystemTestMixin): - """Run tests on Foolscap version of ``IStorageServer.""" +class _SharedMixin(SystemTestMixin): + """Base class for Foolscap and HTTP mixins.""" - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) + def _get_istorage_server(self): + raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): @@ -1010,8 +1020,6 @@ class _FoolscapMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_client = self._get_native_server().get_storage_server() - self.assertTrue(IStorageServer.providedBy(self.storage_client)) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1021,6 +1029,7 @@ class _FoolscapMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock + self.storage_client = self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1035,6 +1044,25 @@ class _FoolscapMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + raise NotImplementedError("implement in subclass") + + +class _FoolscapMixin(_SharedMixin): + """Run tests on Foolscap version of ``IStorageServer``.""" + + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) + + def _get_istorage_server(self): + client = self._get_native_server().get_storage_server() + self.assertTrue(IStorageServer.providedBy(client)) + return client + @inlineCallbacks def disconnect(self): """ @@ -1046,12 +1074,43 @@ class _FoolscapMixin(SystemTestMixin): assert self.storage_client is not current +class _HTTPMixin(_SharedMixin): + """Run tests on the HTTP version of ``IStorageServer``.""" + + def _get_istorage_server(self): + swissnum = b"1234" + self._http_storage_server = HTTPServer(self.server, swissnum) + self._port_number = allocate_tcp_port() + self._listening_port = reactor.listenTCP( + self._port_number, Site(self._http_storage_server.get_resource()), + interface="127.0.0.1" + ) + return StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum + ) + # Eventually should also: + # self.assertTrue(IStorageServer.providedBy(client)) + + @inlineCallbacks + def tearDown(self): + yield _SharedMixin.tearDown(self) + self._listening_port.stopListening() + yield get_treq_pool().closeCachedConnections() + + class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" +class HTTPSharedAPIsTests( + _HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for shared ``IStorageServer`` APIs.""" + + class FoolscapImmutableAPIsTests( _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): From e672029e6d1a1b2e6e209d6e5f6eca5f23842a40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 10:49:43 -0500 Subject: [PATCH 419/916] First HTTP test passes. --- src/allmydata/test/test_istorageserver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fb9765624..11758bf15 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -26,8 +26,9 @@ from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock from twisted.internet import reactor from twisted.web.server import Site +from twisted.web.client import HTTPConnectionPool from hyperlink import DecodedURL -from treq.api import get_global_pool as get_treq_pool +from treq.api import set_global_pool as set_treq_pool from foolscap.api import Referenceable, RemoteException @@ -1078,6 +1079,7 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def _get_istorage_server(self): + set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" self._http_storage_server = HTTPServer(self.server, swissnum) self._port_number = allocate_tcp_port() @@ -1095,8 +1097,7 @@ class _HTTPMixin(_SharedMixin): @inlineCallbacks def tearDown(self): yield _SharedMixin.tearDown(self) - self._listening_port.stopListening() - yield get_treq_pool().closeCachedConnections() + yield self._listening_port.stopListening() class FoolscapSharedAPIsTests( From 03bc39ed771f2c0446a511821d280d9772cfce2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 11:30:41 -0500 Subject: [PATCH 420/916] Try to fix nix builds. --- default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index a8a7ba1c8..095c54578 100644 --- a/default.nix +++ b/default.nix @@ -73,12 +73,13 @@ mach-nix.buildPythonPackage rec { # file. Tell it about them here. setuptools_rust - # mach-nix does not yet parse environment markers correctly. It misses - # all of our requirements which have an environment marker. Duplicate them - # here. + # mach-nix does not yet parse environment markers (e.g. "python > '3.0'") + # correctly. It misses all of our requirements which have an environment marker. + # Duplicate them here. foolscap eliot pyrsistent + collections-extended ''; # Specify where mach-nix should find packages for our Python dependencies. From 66abe5dfca17b67e22474210d72eee92dcede3c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 12:02:52 -0500 Subject: [PATCH 421/916] First passing immutable-API-over-HTTP IStorageServer tests. --- src/allmydata/storage_client.py | 68 +++++++++++++++++++++-- src/allmydata/test/test_istorageserver.py | 20 +++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 13b4dfd01..ca977c9d9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -40,7 +40,7 @@ if PY2: from six import ensure_text import re, time, hashlib - +from os import urandom # On Python 2 this will be the backport. from configparser import NoSectionError @@ -75,7 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict -from allmydata.storage.http_client import StorageClient +from allmydata.storage.http_client import StorageClient, StorageClientImmutables # who is responsible for de-duplication? @@ -1027,6 +1027,48 @@ class _StorageServer(object): ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + +@attr.s +class _FakeRemoteReference(object): + """ + Emulate a Foolscap RemoteReference, calling a local object instead. + """ + local_object = attr.ib(type=object) + + def callRemote(self, action, *args, **kwargs): + return getattr(self.local_object, action)(*args, **kwargs) + + +@attr.s +class _HTTPBucketWriter(object): + """ + Emulate a ``RIBucketWriter``. + """ + client = attr.ib(type=StorageClientImmutables) + storage_index = attr.ib(type=bytes) + share_number = attr.ib(type=int) + upload_secret = attr.ib(type=bytes) + finished = attr.ib(type=bool, default=False) + + def abort(self): + pass # TODO in later ticket + + @defer.inlineCallbacks + def write(self, offset, data): + result = yield self.client.write_share_chunk( + self.storage_index, self.share_number, self.upload_secret, offset, data + ) + if result.finished: + self.finished = True + defer.returnValue(None) + + def close(self): + # A no-op in HTTP protocol. + if not self.finished: + return defer.fail(RuntimeError("You didn't finish writing?!")) + return defer.succeed(None) + + # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @implementer(IStorageServer) # type: ignore @attr.s @@ -1041,11 +1083,12 @@ class _HTTPStorageServer(object): """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(_http_client=http_client) + return _HTTPStorageServer(http_client=http_client) def get_version(self): return self._http_client.get_version() + @defer.inlineCallbacks def allocate_buckets( self, storage_index, @@ -1055,7 +1098,24 @@ class _HTTPStorageServer(object): allocated_size, canary, ): - pass + upload_secret = urandom(20) + immutable_client = StorageClientImmutables(self._http_client) + result = immutable_client.create( + storage_index, sharenums, allocated_size, upload_secret, renew_secret, + cancel_secret + ) + result = yield result + defer.returnValue( + (result.already_have, { + share_num: _FakeRemoteReference(_HTTPBucketWriter( + client=immutable_client, + storage_index=storage_index, + share_number=share_num, + upload_secret=upload_secret + )) + for share_num in result.allocated + }) + ) def get_buckets( self, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 11758bf15..72c07ab82 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -39,6 +39,7 @@ from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer from allmydata.storage.http_client import StorageClient from allmydata.util.iputil import allocate_tcp_port +from allmydata.storage_client import _HTTPStorageServer # Use random generator with known seed, so results are reproducible if tests @@ -1084,12 +1085,15 @@ class _HTTPMixin(_SharedMixin): self._http_storage_server = HTTPServer(self.server, swissnum) self._port_number = allocate_tcp_port() self._listening_port = reactor.listenTCP( - self._port_number, Site(self._http_storage_server.get_resource()), - interface="127.0.0.1" + self._port_number, + Site(self._http_storage_server.get_resource()), + interface="127.0.0.1", ) - return StorageClient( - DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), - swissnum + return _HTTPStorageServer.from_http_client( + StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum, + ) ) # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) @@ -1118,6 +1122,12 @@ class FoolscapImmutableAPIsTests( """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" +class HTTPImmutableAPIsTests( + _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + + class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): From 5dfaa82ed294fe097fdd75e3b576b0ae7724e346 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:47:51 -0500 Subject: [PATCH 422/916] Skip tests that don't pass. --- src/allmydata/test/test_istorageserver.py | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 72c07ab82..65a67c586 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -21,6 +21,7 @@ if PY2: # fmt: on from random import Random +from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock @@ -1013,11 +1014,18 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" + SKIP_TESTS = set() + def _get_istorage_server(self): raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): + if self._testMethodName in self.SKIP_TESTS: + raise SkipTest( + "Test {} is still not supported".format(self._testMethodName) + ) + AsyncTestCase.setUp(self) self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) @@ -1127,6 +1135,23 @@ class HTTPImmutableAPIsTests( ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + # These will start passing in future PRs as HTTP protocol is implemented. + SKIP_TESTS = { + "test_abort", + "test_add_lease_renewal", + "test_add_new_lease", + "test_advise_corrupt_share", + "test_allocate_buckets_repeat", + "test_bucket_advise_corrupt_share", + "test_disconnection", + "test_get_buckets_skips_unfinished_buckets", + "test_matching_overlapping_writes", + "test_non_matching_overlapping_writes", + "test_read_bucket_at_offset", + "test_written_shares_are_readable", + "test_written_shares_are_allocated", + } + class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase From 5cda7ad8b9221df9e5208a6592d39014354ded24 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:52:04 -0500 Subject: [PATCH 423/916] News file. --- newsfragments/3868.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3868.minor diff --git a/newsfragments/3868.minor b/newsfragments/3868.minor new file mode 100644 index 000000000..e69de29bb From c2e524ddb8fd3b655aad3577a6524b08f359ce4a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:55:13 -0500 Subject: [PATCH 424/916] Make mypy happy. --- src/allmydata/test/test_istorageserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 65a67c586..96d4f687f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,6 +19,8 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +else: + from typing import Set from random import Random from unittest import SkipTest @@ -1014,7 +1016,7 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" - SKIP_TESTS = set() + SKIP_TESTS = set() # type: Set[str] def _get_istorage_server(self): raise NotImplementedError("implement in subclass") From c72e7b0585edf515506a66c1b6b150315b81757d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:20:23 -0500 Subject: [PATCH 425/916] Implement HTTP share listing endpoint. --- docs/proposed/http-storage-node-protocol.rst | 4 +-- src/allmydata/storage/http_client.py | 29 ++++++++++++++---- src/allmydata/storage/http_server.py | 16 ++++++++++ src/allmydata/test/test_storage_http.py | 31 ++++++++++++++++++++ 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 560220d00..f47a50d3e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -630,8 +630,8 @@ Reading ``GET /v1/immutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Retrieve a list indicating all shares available for the indicated storage index. -For example:: +Retrieve a list (semantically, a set) indicating all shares available for the +indicated storage index. For example:: [1, 5] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d4837d4ab..8a1d03192 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -136,6 +136,7 @@ class UploadProgress(object): """ Progress of immutable upload, per the server. """ + # True when upload has finished. finished = attr.ib(type=bool) # Remaining ranges to upload. @@ -221,7 +222,7 @@ class StorageClientImmutables(object): headers=Headers( { "content-range": [ - ContentRange("bytes", offset, offset+len(data)).to_header() + ContentRange("bytes", offset, offset + len(data)).to_header() ] } ), @@ -268,11 +269,7 @@ class StorageClientImmutables(object): "GET", url, headers=Headers( - { - "range": [ - Range("bytes", [(offset, offset + length)]).to_header() - ] - } + {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), ) if response.code == http.PARTIAL_CONTENT: @@ -282,3 +279,23 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) + + @inlineCallbacks + def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] + """ + Return the set of shares for a given storage index. + """ + url = self._client._url( + "/v1/immutable/{}/shares".format(_encode_si(storage_index)) + ) + response = yield self._client._request( + "GET", + url, + ) + if response.code == http.OK: + body = yield response.content() + returnValue(set(loads(body))) + else: + raise ClientException( + response.code, + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d79e9a38b..f4ba865ca 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -255,6 +255,22 @@ class HTTPServer(object): required.append({"begin": start, "end": end}) return self._cbor(request, {"required": required}) + @_authorized_route( + _app, + set(), + "/v1/immutable//shares", + methods=["GET"], + ) + def list_shares(self, request, authorization, storage_index): + """ + List shares for the given storage index. + """ + storage_index = si_a2b(storage_index.encode("ascii")) + + # TODO in future ticket, handle KeyError as 404 + share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) + return self._cbor(request, share_numbers) + @_authorized_route( _app, set(), diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index dcefc9950..4ddfff62e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -382,6 +382,37 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) + def test_list_shares(self): + """ + Once a share is finished uploading, it's possible to list it. + """ + im_client = StorageClientImmutables(self.http.client) + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1, 2, 3}, 10, upload_secret, lease_secret, lease_secret + ) + ) + + # Initially there are no shares: + self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + + # Upload shares 1 and 3: + for share_number in [1, 3]: + progress = result_of(im_client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + b"0123456789", + )) + self.assertTrue(progress.finished) + + # Now shares 1 and 3 exist: + self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + def test_multiple_shares_uploaded_to_different_place(self): """ If a storage index has multiple shares, uploads to different shares are From 48a9bf745724c5c5333e46ad48f3ae6c3771c8fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:25:13 -0500 Subject: [PATCH 426/916] Hook up more IStorageServer tests that can now pass with HTTP. --- src/allmydata/storage_client.py | 33 +++++++++++++++++++++-- src/allmydata/test/test_istorageserver.py | 2 -- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ca977c9d9..e7dbb27a8 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1042,7 +1042,7 @@ class _FakeRemoteReference(object): @attr.s class _HTTPBucketWriter(object): """ - Emulate a ``RIBucketWriter``. + Emulate a ``RIBucketWriter``, but use HTTP protocol underneath. """ client = attr.ib(type=StorageClientImmutables) storage_index = attr.ib(type=bytes) @@ -1069,6 +1069,25 @@ class _HTTPBucketWriter(object): return defer.succeed(None) + +@attr.s +class _HTTPBucketReader(object): + """ + Emulate a ``RIBucketReader``. + """ + client = attr.ib(type=StorageClientImmutables) + storage_index = attr.ib(type=bytes) + share_number = attr.ib(type=int) + + def read(self, offset, length): + return self.client.read_share_chunk( + self.storage_index, self.share_number, offset, length + ) + + def advise_corrupt_share(self, reason): + pass # TODO in later ticket + + # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @implementer(IStorageServer) # type: ignore @attr.s @@ -1117,8 +1136,18 @@ class _HTTPStorageServer(object): }) ) + @defer.inlineCallbacks def get_buckets( self, storage_index, ): - pass + immutable_client = StorageClientImmutables(self._http_client) + share_numbers = yield immutable_client.list_shares( + storage_index + ) + defer.returnValue({ + share_num: _FakeRemoteReference(_HTTPBucketReader( + immutable_client, storage_index, share_num + )) + for share_num in share_numbers + }) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 96d4f687f..dc2aa0efb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1149,8 +1149,6 @@ class HTTPImmutableAPIsTests( "test_get_buckets_skips_unfinished_buckets", "test_matching_overlapping_writes", "test_non_matching_overlapping_writes", - "test_read_bucket_at_offset", - "test_written_shares_are_readable", "test_written_shares_are_allocated", } From 7da506d5d0adbae71fefe87d6981b79540324caf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:26:42 -0500 Subject: [PATCH 427/916] News file. --- newsfragments/3871.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3871.minor diff --git a/newsfragments/3871.minor b/newsfragments/3871.minor new file mode 100644 index 000000000..e69de29bb From 0fbf746e27767c548219b23d25d883ca40a6ab82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:30:27 -0500 Subject: [PATCH 428/916] Skip on Python 2. --- src/allmydata/test/test_istorageserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 96d4f687f..9b21c6480 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1089,6 +1089,11 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + return _SharedMixin.setUp(self) + def _get_istorage_server(self): set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" From 70d0bd0597b46015c45489ba57b5b566abe5e395 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:41:12 -0500 Subject: [PATCH 429/916] Test and document what happens for non-existent storage index. --- docs/proposed/http-storage-node-protocol.rst | 3 +++ src/allmydata/storage/http_server.py | 2 -- src/allmydata/test/test_storage_http.py | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index f47a50d3e..9c09eb362 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -635,6 +635,9 @@ indicated storage index. For example:: [1, 5] +An unknown storage index results in empty list, so that lack of existence of +storage index is not leaked. + ``GET /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f4ba865ca..f885baa22 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -266,8 +266,6 @@ class HTTPServer(object): List shares for the given storage index. """ storage_index = si_a2b(storage_index.encode("ascii")) - - # TODO in future ticket, handle KeyError as 404 share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) return self._cbor(request, share_numbers) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4ddfff62e..b2def5581 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -413,6 +413,14 @@ class ImmutableHTTPAPITests(SyncTestCase): # Now shares 1 and 3 exist: self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + def test_list_shares_unknown_storage_index(self): + """ + Listing unknown storage index's shares results in empty list of shares. + """ + im_client = StorageClientImmutables(self.http.client) + storage_index = b"".join(bytes([i]) for i in range(16)) + self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + def test_multiple_shares_uploaded_to_different_place(self): """ If a storage index has multiple shares, uploads to different shares are From 7a1f8e64f10f75f26b31d95f5f1ff69b73a64901 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 2 Feb 2022 01:33:22 +0100 Subject: [PATCH 430/916] remove code-markup around commands Signed-off-by: fenn-cs --- docs/release-checklist.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index e2002b1fa..9588fd1a5 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -53,16 +53,16 @@ previously generated files. Get into the release directory and install dependencies by running -- ``cd ../tahoe-release-x.x.x`` (assuming you are still in your original clone) -- ``python -m venv venv`` -- ``./venv/bin/pip install --editable .[test]`` +- cd ../tahoe-release-x.x.x (assuming you are still in your original clone) +- python -m venv venv +- ./venv/bin/pip install --editable .[test] Create Branch and Apply Updates ``````````````````````````````` - Create a branch for the release/candidate (e.g. ``XXXX.release-1.16.0``) -- run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) +- run tox -e news to produce a new NEWS.txt file (this does a commit) - create the news for the release - newsfragments/.minor @@ -112,7 +112,7 @@ they will need to evaluate which contributors' signatures they trust. - (all steps above are completed) - sign the release - - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` + - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0 .. note:: - Replace the key-id above with your own, which can simply be your email if it's attached to your fingerprint. @@ -122,11 +122,11 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - ``tox -e py27,codechecks,docs,integration`` + - tox -e py27,codechecks,docs,integration - these can fail (ideally they should not of course): - - ``tox -e deprecations,upcoming-deprecations`` + - tox -e deprecations,upcoming-deprecations - clone to a clean, local checkout (to avoid extra files being included in the release) @@ -138,7 +138,7 @@ they will need to evaluate which contributors' signatures they trust. - tox -e tarballs - Confirm that release tarballs exist by runnig: - - ``ls dist/ | grep 1.16.0rc0`` + - ls dist/ | grep 1.16.0rc0 - inspect and test the tarballs @@ -147,8 +147,8 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl`` - - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz`` + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz Privileged Contributor From aebb5056de20f59f2362431b83514b2a62634f42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 11:00:16 -0500 Subject: [PATCH 431/916] Don't use real reactor in these tests. --- src/allmydata/test/test_storage_http.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b2def5581..982e22859 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -23,6 +23,7 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap +from twisted.internet.task import Clock from .common import SyncTestCase from ..storage.server import StorageServer @@ -230,8 +231,11 @@ class HttpTestFixture(Fixture): """ def _setUp(self): + self.clock = Clock() self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.storage_server = StorageServer( + self.tempdir.path, b"\x00" * 20, clock=self.clock + ) self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), @@ -401,13 +405,15 @@ class ImmutableHTTPAPITests(SyncTestCase): # Upload shares 1 and 3: for share_number in [1, 3]: - progress = result_of(im_client.write_share_chunk( + progress = result_of( + im_client.write_share_chunk( storage_index, share_number, upload_secret, 0, b"0123456789", - )) + ) + ) self.assertTrue(progress.finished) # Now shares 1 and 3 exist: From f0c00fcbe41dcde78790dddbcac599699faa5d9c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 11:04:16 -0500 Subject: [PATCH 432/916] News file. --- newsfragments/3860.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3860.minor diff --git a/newsfragments/3860.minor b/newsfragments/3860.minor new file mode 100644 index 000000000..e69de29bb From bceed6e19984418745c077d0f04993fb994b296c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 11:52:31 -0500 Subject: [PATCH 433/916] More bucket allocation logic. --- src/allmydata/storage/http_server.py | 72 +++++++++++++------------ src/allmydata/test/test_storage_http.py | 50 +++++++++++++++++ 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f885baa22..50aa6ae9e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -128,10 +128,15 @@ class StorageIndexUploads(object): """ # Map share number to BucketWriter - shares = attr.ib() # type: Dict[int,BucketWriter] + shares = attr.ib(factory=dict) # type: Dict[int,BucketWriter] - # The upload key. - upload_secret = attr.ib() # type: bytes + # Mape share number to the upload secret (different shares might have + # different upload secrets). + upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] + + def add_upload(self, share_number, upload_secret, bucket): + self.shares[share_number] = bucket + self.upload_secrets[share_number] = upload_secret class HTTPServer(object): @@ -179,39 +184,40 @@ class HTTPServer(object): def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" storage_index = si_a2b(storage_index.encode("ascii")) - info = loads(request.content.read()) upload_secret = authorization[Secrets.UPLOAD] + info = loads(request.content.read()) if storage_index in self._uploads: - # Pre-existing upload. - in_progress = self._uploads[storage_index] - if timing_safe_compare(in_progress.upload_secret, upload_secret): - # Same session. - # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. - # The backend code may already implement this logic. - pass - else: - # TODO Fail, since the secret doesnt match. - pass - else: - # New upload. - already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( - storage_index, - renew_secret=authorization[Secrets.LEASE_RENEW], - cancel_secret=authorization[Secrets.LEASE_CANCEL], - sharenums=info["share-numbers"], - allocated_size=info["allocated-size"], - ) - self._uploads[storage_index] = StorageIndexUploads( - shares=sharenum_to_bucket, upload_secret=authorization[Secrets.UPLOAD] - ) - return self._cbor( - request, - { - "already-have": set(already_got), - "allocated": set(sharenum_to_bucket), - }, - ) + for share_number in info["share-numbers"]: + in_progress = self._uploads[storage_index] + # For pre-existing upload, make sure password matches. + if ( + share_number in in_progress.upload_secrets + and not timing_safe_compare( + in_progress.upload_secrets[share_number], upload_secret + ) + ): + request.setResponseCode(http.UNAUTHORIZED) + return b"" + + already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( + storage_index, + renew_secret=authorization[Secrets.LEASE_RENEW], + cancel_secret=authorization[Secrets.LEASE_CANCEL], + sharenums=info["share-numbers"], + allocated_size=info["allocated-size"], + ) + uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) + for share_number, bucket in sharenum_to_bucket.items(): + uploads.add_upload(share_number, upload_secret, bucket) + + return self._cbor( + request, + { + "already-have": set(already_got), + "allocated": set(sharenum_to_bucket), + }, + ) @_authorized_route( _app, diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 982e22859..39f07a54a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -24,6 +24,7 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock +from twisted.web import http from .common import SyncTestCase from ..storage.server import StorageServer @@ -386,6 +387,55 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) + def test_allocate_buckets_second_time_wrong_upload_key(self): + """ + If allocate buckets endpoint is called second time with wrong upload + key on the same shares, the result is an error. + """ + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret + ) + ) + with self.assertRaises(ClientException) as e: + result_of( + im_client.create( + storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret + ) + ) + self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) + + def test_allocate_buckets_second_time_different_shares(self): + """ + If allocate buckets endpoint is called second time with different + upload key on different shares, that creates the buckets. + """ + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret + ) + ) + + # Add same shares: + created2 = result_of( + im_client.create( + storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret + ) + ) + self.assertEqual(created2.allocated, {4, 6}) + def test_list_shares(self): """ Once a share is finished uploading, it's possible to list it. From 39fe48b1746d5d854cdd87129112430332265bba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 12:55:41 -0500 Subject: [PATCH 434/916] More passing IStorageServer tests. --- src/allmydata/storage_client.py | 10 ++++++---- src/allmydata/test/test_istorageserver.py | 3 --- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e7dbb27a8..665cfd513 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1094,15 +1094,18 @@ class _HTTPBucketReader(object): class _HTTPStorageServer(object): """ Talk to remote storage server over HTTP. + + The same upload key is used for all communication. """ _http_client = attr.ib(type=StorageClient) + _upload_secret = attr.ib(type=bytes) @staticmethod def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(http_client=http_client) + return _HTTPStorageServer(http_client=http_client, upload_secret=urandom(20)) def get_version(self): return self._http_client.get_version() @@ -1117,10 +1120,9 @@ class _HTTPStorageServer(object): allocated_size, canary, ): - upload_secret = urandom(20) immutable_client = StorageClientImmutables(self._http_client) result = immutable_client.create( - storage_index, sharenums, allocated_size, upload_secret, renew_secret, + storage_index, sharenums, allocated_size, self._upload_secret, renew_secret, cancel_secret ) result = yield result @@ -1130,7 +1132,7 @@ class _HTTPStorageServer(object): client=immutable_client, storage_index=storage_index, share_number=share_num, - upload_secret=upload_secret + upload_secret=self._upload_secret )) for share_num in result.allocated }) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cf1d977d8..679b0f964 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1148,13 +1148,10 @@ class HTTPImmutableAPIsTests( "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", - "test_allocate_buckets_repeat", "test_bucket_advise_corrupt_share", "test_disconnection", - "test_get_buckets_skips_unfinished_buckets", "test_matching_overlapping_writes", "test_non_matching_overlapping_writes", - "test_written_shares_are_allocated", } From 1dfc0bde3679c7eaee8617b2dd9008ef8ae37053 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:43:49 -0500 Subject: [PATCH 435/916] Use better method to listen on random port. --- src/allmydata/test/test_istorageserver.py | 39 +++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9b21c6480..2a875b120 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -25,9 +25,10 @@ else: from random import Random from unittest import SkipTest -from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor +from twisted.internet.endpoints import serverFromString from twisted.web.server import Site from twisted.web.client import HTTPConnectionPool from hyperlink import DecodedURL @@ -37,11 +38,10 @@ from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin -from .common import AsyncTestCase +from .common import AsyncTestCase, SameProcessStreamEndpointAssigner from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer from allmydata.storage.http_client import StorageClient -from allmydata.util.iputil import allocate_tcp_port from allmydata.storage_client import _HTTPStorageServer @@ -1029,6 +1029,11 @@ class _SharedMixin(SystemTestMixin): ) AsyncTestCase.setUp(self) + + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) + self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) @@ -1041,7 +1046,7 @@ class _SharedMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock - self.storage_client = self._get_istorage_server() + self.storage_client = yield self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1073,7 +1078,7 @@ class _FoolscapMixin(_SharedMixin): def _get_istorage_server(self): client = self._get_native_server().get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) - return client + return succeed(client) @inlineCallbacks def disconnect(self): @@ -1094,20 +1099,26 @@ class _HTTPMixin(_SharedMixin): self.skipTest("Not going to bother supporting Python 2") return _SharedMixin.setUp(self) + @inlineCallbacks def _get_istorage_server(self): set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" self._http_storage_server = HTTPServer(self.server, swissnum) - self._port_number = allocate_tcp_port() - self._listening_port = reactor.listenTCP( - self._port_number, - Site(self._http_storage_server.get_resource()), - interface="127.0.0.1", + + # Listen on randomly assigned port: + tcp_address, endpoint_string = self._port_assigner.assign(reactor) + _, host, port = tcp_address.split(":") + port = int(port) + endpoint = serverFromString(reactor, endpoint_string) + self._listening_port = yield endpoint.listen( + Site(self._http_storage_server.get_resource()) ) - return _HTTPStorageServer.from_http_client( - StorageClient( - DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), - swissnum, + returnValue( + _HTTPStorageServer.from_http_client( + StorageClient( + DecodedURL().replace(scheme="http", host=host, port=port), + swissnum, + ) ) ) # Eventually should also: From 23c8bde9d597b83cc3f89ac9a781f3e385961cea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:44:55 -0500 Subject: [PATCH 436/916] Nicer cleanup. --- src/allmydata/test/test_istorageserver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 2a875b120..cfd81feda 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1113,6 +1113,7 @@ class _HTTPMixin(_SharedMixin): self._listening_port = yield endpoint.listen( Site(self._http_storage_server.get_resource()) ) + self.addCleanup(self._listening_port.stopListening) returnValue( _HTTPStorageServer.from_http_client( StorageClient( @@ -1124,11 +1125,6 @@ class _HTTPMixin(_SharedMixin): # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) - @inlineCallbacks - def tearDown(self): - yield _SharedMixin.tearDown(self) - yield self._listening_port.stopListening() - class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase From 6b3722d3f664d9274ce70812a485eb99a8e5f5a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:50:29 -0500 Subject: [PATCH 437/916] Avoid using possibly-private API. --- src/allmydata/test/test_istorageserver.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cfd81feda..7ed8a1b65 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -30,9 +30,9 @@ from twisted.internet.task import Clock from twisted.internet import reactor from twisted.internet.endpoints import serverFromString from twisted.web.server import Site -from twisted.web.client import HTTPConnectionPool +from twisted.web.client import Agent, HTTPConnectionPool from hyperlink import DecodedURL -from treq.api import set_global_pool as set_treq_pool +from treq.client import HTTPClient from foolscap.api import Referenceable, RemoteException @@ -1101,24 +1101,29 @@ class _HTTPMixin(_SharedMixin): @inlineCallbacks def _get_istorage_server(self): - set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" - self._http_storage_server = HTTPServer(self.server, swissnum) + http_storage_server = HTTPServer(self.server, swissnum) # Listen on randomly assigned port: tcp_address, endpoint_string = self._port_assigner.assign(reactor) _, host, port = tcp_address.split(":") port = int(port) endpoint = serverFromString(reactor, endpoint_string) - self._listening_port = yield endpoint.listen( - Site(self._http_storage_server.get_resource()) + listening_port = yield endpoint.listen(Site(http_storage_server.get_resource())) + self.addCleanup(listening_port.stopListening) + + # Create HTTP client with non-persistent connections, so we don't leak + # state across tests: + treq_client = HTTPClient( + Agent(reactor, HTTPConnectionPool(reactor, persistent=False)) ) - self.addCleanup(self._listening_port.stopListening) + returnValue( _HTTPStorageServer.from_http_client( StorageClient( DecodedURL().replace(scheme="http", host=host, port=port), swissnum, + treq=treq_client, ) ) ) From c2c3411dc40155e31743089d99465ad9041cdafc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:57:48 -0500 Subject: [PATCH 438/916] Try to fix Python 2. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ae70a3bb..f65890d37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,7 +217,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -277,7 +277,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 7454929be0d761b87909f04e26a9cf61d85a5702 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:26:25 -0500 Subject: [PATCH 439/916] Less code duplication. --- src/allmydata/storage/http_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8a1d03192..2dc133b52 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -238,7 +238,7 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = loads((yield response.content())) + body = yield _decode_cbor(response) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) @@ -293,8 +293,8 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield response.content() - returnValue(set(loads(body))) + body = yield _decode_cbor(response) + returnValue(set(body)) else: raise ClientException( response.code, From 5e3a31166d7782e8b84094b58fe4bca40faf1995 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:26:58 -0500 Subject: [PATCH 440/916] Better explanation. --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e7dbb27a8..cf5fb65a2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1073,7 +1073,7 @@ class _HTTPBucketWriter(object): @attr.s class _HTTPBucketReader(object): """ - Emulate a ``RIBucketReader``. + Emulate a ``RIBucketReader``, but use HTTP protocol underneath. """ client = attr.ib(type=StorageClientImmutables) storage_index = attr.ib(type=bytes) From 83d8f2eb78f1c9da1eadf97429e9f0eac12deafd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:27:29 -0500 Subject: [PATCH 441/916] Remove incorrect editorial. --- docs/proposed/http-storage-node-protocol.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 9c09eb362..b00b327e3 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -635,8 +635,7 @@ indicated storage index. For example:: [1, 5] -An unknown storage index results in empty list, so that lack of existence of -storage index is not leaked. +An unknown storage index results in an empty list. ``GET /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From ce2468cdffb56705c7f9edde4581b7b675716117 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:53:39 -0500 Subject: [PATCH 442/916] Validate inputs automatically as part of parsing. --- src/allmydata/storage/http_server.py | 27 +++++++++------ src/allmydata/test/test_storage_http.py | 45 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50aa6ae9e..cf8284e3e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,6 +23,7 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header +from werkzeug.routing import BaseConverter # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -32,6 +33,7 @@ from .http_common import swissnum_auth_header, Secrets from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare +from ..util.base32 import rfc3548_alphabet class ClientSecretsException(Exception): @@ -139,12 +141,22 @@ class StorageIndexUploads(object): self.upload_secrets[share_number] = upload_secret +class StorageIndexConverter(BaseConverter): + """Parser/validator for storage index URL path segments.""" + + regex = "[" + str(rfc3548_alphabet, "ascii") + "]{26}" + + def to_python(self, value): + return si_a2b(value.encode("ascii")) + + class HTTPServer(object): """ A HTTP interface to the storage server. """ _app = Klein() + _app.url_map.converters["storage_index"] = StorageIndexConverter def __init__( self, storage_server, swissnum @@ -178,12 +190,11 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, - "/v1/immutable/", + "/v1/immutable/", methods=["POST"], ) def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" - storage_index = si_a2b(storage_index.encode("ascii")) upload_secret = authorization[Secrets.UPLOAD] info = loads(request.content.read()) @@ -222,12 +233,11 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.UPLOAD}, - "/v1/immutable//", + "/v1/immutable//", methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" - storage_index = si_a2b(storage_index.encode("ascii")) content_range = parse_content_range_header(request.getHeader("content-range")) # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 # 1. Malformed header should result in error 416 @@ -236,7 +246,7 @@ class HTTPServer(object): # 4. Impossible range should resul tin error 416 offset = content_range.start - # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. + # TODO basic checks on validity of start, offset, and content-range in general. # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. data = request.content.read() @@ -264,34 +274,31 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable//shares", + "/v1/immutable//shares", methods=["GET"], ) def list_shares(self, request, authorization, storage_index): """ List shares for the given storage index. """ - storage_index = si_a2b(storage_index.encode("ascii")) share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) return self._cbor(request, share_numbers) @_authorized_route( _app, set(), - "/v1/immutable//", + "/v1/immutable//", methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. basic checks on validity on storage index, share number # 2. missing range header should have response code 200 and return whole thing # 3. malformed range header should result in error? or return everything? # 4. non-bytes range results in error # 5. ranges make sense semantically (positive, etc.) # 6. multiple ranges fails with error # 7. missing end of range means "to the end of share" - storage_index = si_a2b(storage_index.encode("ascii")) range_header = parse_range_header(request.getHeader("range")) offset, end = range_header.ranges[0] assert end != None # TODO support this case diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 39f07a54a..f78d83b27 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,6 +25,8 @@ from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock from twisted.web import http +from werkzeug import routing +from werkzeug.exceptions import NotFound as WNotFound from .common import SyncTestCase from ..storage.server import StorageServer @@ -34,6 +36,7 @@ from ..storage.http_server import ( Secrets, ClientSecretsException, _authorized_route, + StorageIndexConverter, ) from ..storage.http_client import ( StorageClient, @@ -42,6 +45,7 @@ from ..storage.http_client import ( ImmutableCreateResult, UploadProgress, ) +from ..storage.common import si_b2a def _post_process(params): @@ -148,6 +152,47 @@ class ExtractSecretsTests(SyncTestCase): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) +class RouteConverterTests(SyncTestCase): + """Tests for custom werkzeug path segment converters.""" + + adapter = routing.Map( + [ + routing.Rule( + "//", endpoint="si", methods=["GET"] + ) + ], + converters={"storage_index": StorageIndexConverter}, + ).bind("example.com", "/") + + @given(storage_index=st.binary(min_size=16, max_size=16)) + def test_good_storage_index_is_parsed(self, storage_index): + """ + A valid storage index is accepted and parsed back out by + StorageIndexConverter. + """ + self.assertEqual( + self.adapter.match( + "/{}/".format(str(si_b2a(storage_index), "ascii")), method="GET" + ), + ("si", {"storage_index": storage_index}), + ) + + def test_long_storage_index_is_not_parsed(self): + """An overly long storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/{}/".format("a" * 27), method="GET") + + def test_short_storage_index_is_not_parsed(self): + """An overly short storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/{}/".format("a" * 25), method="GET") + + def test_bad_characters_storage_index_is_not_parsed(self): + """An overly short storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/{}_/".format("a" * 25), method="GET") + + # TODO should be actual swissnum SWISSNUM_FOR_TEST = b"abcd" From 7107a85fba4bb7c8e6998d48442a1caac987279a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Feb 2022 10:19:37 -0500 Subject: [PATCH 443/916] Refactor client, separating low-level and high-level concerns. --- src/allmydata/storage/http_client.py | 35 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 18 ++++++++----- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2dc133b52..3b1dbfb30 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -68,7 +68,7 @@ class ImmutableCreateResult(object): class StorageClient(object): """ - HTTP client that talks to the HTTP storage server. + Low-level HTTP client that talks to the HTTP storage server. """ def __init__( @@ -78,7 +78,7 @@ class StorageClient(object): self._swissnum = swissnum self._treq = treq - def _url(self, path): + def relative_url(self, path): """Get a URL relative to the base URL.""" return self._base_url.click(path) @@ -92,7 +92,7 @@ class StorageClient(object): ) return headers - def _request( + def request( self, method, url, @@ -120,13 +120,22 @@ class StorageClient(object): ) return self._treq.request(method, url, headers=headers, **kwargs) + +class StorageClientGeneral(object): + """ + High-level HTTP APIs that aren't immutable- or mutable-specific. + """ + + def __init__(self, client): # type: (StorageClient) -> None + self._client = client + @inlineCallbacks def get_version(self): """ Return the version metadata for the server. """ - url = self._url("/v1/version") - response = yield self._request("GET", url) + url = self._client.relative_url("/v1/version") + response = yield self._client.request("GET", url) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) @@ -174,11 +183,11 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client._url("/v1/immutable/" + _encode_si(storage_index)) + url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index)) message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) - response = yield self._client._request( + response = yield self._client.request( "POST", url, lease_renew_secret=lease_renew_secret, @@ -211,10 +220,10 @@ class StorageClientImmutables(object): whether the _complete_ share (i.e. all chunks, not just this one) has been uploaded. """ - url = self._client._url( + url = self._client.relative_url( "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) - response = yield self._client._request( + response = yield self._client.request( "PATCH", url, upload_secret=upload_secret, @@ -262,10 +271,10 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - url = self._client._url( + url = self._client.relative_url( "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) - response = yield self._client._request( + response = yield self._client.request( "GET", url, headers=Headers( @@ -285,10 +294,10 @@ class StorageClientImmutables(object): """ Return the set of shares for a given storage index. """ - url = self._client._url( + url = self._client.relative_url( "/v1/immutable/{}/shares".format(_encode_si(storage_index)) ) - response = yield self._client._request( + response = yield self._client.request( "GET", url, ) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index f78d83b27..41391845f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -44,6 +44,7 @@ from ..storage.http_client import ( StorageClientImmutables, ImmutableCreateResult, UploadProgress, + StorageClientGeneral, ) from ..storage.common import si_b2a @@ -253,7 +254,7 @@ class RoutingTests(SyncTestCase): """ # Without secret, get a 400 error. response = result_of( - self.client._request( + self.client.request( "GET", "http://127.0.0.1/upload_secret", ) @@ -262,7 +263,7 @@ class RoutingTests(SyncTestCase): # With secret, we're good. response = result_of( - self.client._request( + self.client.request( "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" ) ) @@ -307,10 +308,12 @@ class GenericHTTPAPITests(SyncTestCase): If the wrong swissnum is used, an ``Unauthorized`` response code is returned. """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), + client = StorageClientGeneral( + StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) ) with self.assertRaises(ClientException) as e: result_of(client.get_version()) @@ -323,7 +326,8 @@ class GenericHTTPAPITests(SyncTestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = result_of(self.http.client.get_version()) + client = StorageClientGeneral(self.http.client) + version = result_of(client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) From d38183335eb7ca8bbcb3d9b1a7cd57a898061ae6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Feb 2022 10:46:55 -0500 Subject: [PATCH 444/916] Handle bad Content-Range headers. --- src/allmydata/storage/http_client.py | 6 ++- src/allmydata/storage/http_server.py | 12 +++--- src/allmydata/test/test_storage_http.py | 57 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3b1dbfb30..475aa2330 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -48,7 +48,11 @@ def _encode_si(si): # type: (bytes) -> str class ClientException(Exception): - """An unexpected error.""" + """An unexpected response code from the server.""" + + def __init__(self, code, *additional_args): + Exception.__init__(self, code, *additional_args) + self.code = code def _decode_cbor(response): diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index cf8284e3e..ad1b8b2ac 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -239,14 +239,14 @@ class HTTPServer(object): def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" content_range = parse_content_range_header(request.getHeader("content-range")) - # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. Malformed header should result in error 416 - # 2. Non-bytes unit should result in error 416 - # 3. Missing header means full upload in one request - # 4. Impossible range should resul tin error 416 + if content_range is None or content_range.units != "bytes": + # TODO Missing header means full upload in one request + request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) + return b"" + offset = content_range.start - # TODO basic checks on validity of start, offset, and content-range in general. + # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. data = request.content.read() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 41391845f..7e459ee2a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,6 +25,7 @@ from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock from twisted.web import http +from twisted.web.http_headers import Headers from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound @@ -291,6 +292,24 @@ class HttpTestFixture(Fixture): ) +class StorageClientWithHeadersOverride(object): + """Wrap ``StorageClient`` and override sent headers.""" + + def __init__(self, storage_client, add_headers): + self.storage_client = storage_client + self.add_headers = add_headers + + def __getattr__(self, attr): + return getattr(self.storage_client, attr) + + def request(self, *args, headers=None, **kwargs): + if headers is None: + headers = Headers() + for key, value in self.add_headers.items(): + headers.setRawHeaders(key, [value]) + return self.storage_client.request(*args, headers=headers, **kwargs) + + class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API @@ -518,6 +537,44 @@ class ImmutableHTTPAPITests(SyncTestCase): # Now shares 1 and 3 exist: self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + def test_upload_bad_content_range(self): + """ + Malformed or invalid Content-Range headers to the immutable upload + endpoint result in a 416 error. + """ + im_client = StorageClientImmutables(self.http.client) + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"0" * 16 + result_of( + im_client.create( + storage_index, {1}, 10, upload_secret, lease_secret, lease_secret + ) + ) + + def check_invalid(bad_content_range_value): + client = StorageClientImmutables( + StorageClientWithHeadersOverride( + self.http.client, {"content-range": bad_content_range_value} + ) + ) + with self.assertRaises(ClientException) as e: + result_of( + client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"0123456789", + ) + ) + self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) + + check_invalid("not a valid content-range header at all") + check_invalid("bytes -1-9/10") + check_invalid("bytes 0--9/10") + check_invalid("teapots 0-9/10") + def test_list_shares_unknown_storage_index(self): """ Listing unknown storage index's shares results in empty list of shares. From ecb1a3c5a0bd63a6c36be5251b4881771ab65586 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:25:47 -0500 Subject: [PATCH 445/916] Just require content-range for simplicity. --- docs/proposed/http-storage-node-protocol.rst | 2 +- src/allmydata/storage/http_server.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index b00b327e3..4d8d60560 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -540,7 +540,7 @@ Rejected designs for upload secrets: Write data for the indicated share. The share number must belong to the storage index. The request body is the raw share data (i.e., ``application/octet-stream``). -*Content-Range* requests are encouraged for large transfers to allow partially complete uploads to be resumed. +*Content-Range* requests are required; for large transfers this allows partially complete uploads to be resumed. For example, a 1MiB share can be divided in to eight separate 128KiB chunks. Each chunk can be uploaded in a separate request. diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ad1b8b2ac..708b99380 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -240,16 +240,13 @@ class HTTPServer(object): """Write data to an in-progress immutable upload.""" content_range = parse_content_range_header(request.getHeader("content-range")) if content_range is None or content_range.units != "bytes": - # TODO Missing header means full upload in one request request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) return b"" - + offset = content_range.start - - # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. - - data = request.content.read() + # TODO limit memory usage + data = request.content.read(content_range.stop - content_range.start + 1) try: bucket = self._uploads[storage_index].shares[share_number] except (KeyError, IndexError): From 72bac785ee3b21d4d7bd9cb8f71df76ae3fd35a5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:27:08 -0500 Subject: [PATCH 446/916] Done elsewhere. --- src/allmydata/test/test_storage_http.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 7e459ee2a..ae316708c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -615,13 +615,6 @@ class ImmutableHTTPAPITests(SyncTestCase): TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a From 95d7548629e6a96a2a2e6f57dbc8dac2015caeaf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:30:38 -0500 Subject: [PATCH 447/916] Upload to non-existent place. --- src/allmydata/storage/http_server.py | 4 +-- src/allmydata/test/test_storage_http.py | 33 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 708b99380..6f9033380 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -250,8 +250,8 @@ class HTTPServer(object): try: bucket = self._uploads[storage_index].shares[share_number] except (KeyError, IndexError): - # TODO return 404 - raise + request.setResponseCode(http.NOT_FOUND) + return b"" finished = bucket.write(offset, data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index ae316708c..7a5c24905 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -583,6 +583,39 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = b"".join(bytes([i]) for i in range(16)) self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + def test_upload_non_existent_storage_index(self): + """ + Uploading to a non-existent storage index or share number results in + 404. + """ + im_client = StorageClientImmutables(self.http.client) + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1}, 10, upload_secret, lease_secret, lease_secret + ) + ) + + def unknown_check(storage_index, share_number): + with self.assertRaises(ClientException) as e: + result_of( + im_client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + b"0123456789", + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) + + # Wrong share number: + unknown_check(storage_index, 7) + # Wrong storage index: + unknown_check(b"X" * 16, 7) + def test_multiple_shares_uploaded_to_different_place(self): """ If a storage index has multiple shares, uploads to different shares are From 8c739343f3f21d3b8ad112a87eca582e03638967 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:37:49 -0500 Subject: [PATCH 448/916] Reduce duplication. --- src/allmydata/test/test_storage_http.py | 104 +++++++++--------------- 1 file changed, 39 insertions(+), 65 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 7a5c24905..2bad160e4 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -373,6 +373,28 @@ class ImmutableHTTPAPITests(SyncTestCase): self.skipTest("Not going to bother supporting Python 2") super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) + self.im_client = StorageClientImmutables(self.http.client) + + def create_upload(self, share_numbers, length): + """ + Create a write bucket on server, return: + + (upload_secret, lease_secret, storage_index, result) + """ + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + created = result_of( + self.im_client.create( + storage_index, + share_numbers, + length, + upload_secret, + lease_secret, + lease_secret, + ) + ) + return (upload_secret, lease_secret, storage_index, created) def test_upload_can_be_downloaded(self): """ @@ -386,17 +408,8 @@ class ImmutableHTTPAPITests(SyncTestCase): length = 100 expected_data = b"".join(bytes([i]) for i in range(100)) - im_client = StorageClientImmutables(self.http.client) - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = result_of( - im_client.create( - storage_index, {1}, 100, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) ) @@ -407,7 +420,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): remaining.empty(offset, offset + length) - return im_client.write_share_chunk( + return self.im_client.write_share_chunk( storage_index, 1, upload_secret, @@ -451,7 +464,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = result_of( - im_client.read_share_chunk(storage_index, 1, offset, length) + self.im_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) @@ -460,20 +473,13 @@ class ImmutableHTTPAPITests(SyncTestCase): If allocate buckets endpoint is called second time with wrong upload key on the same shares, the result is an error. """ - im_client = StorageClientImmutables(self.http.client) - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret - ) + (upload_secret, lease_secret, storage_index, _) = self.create_upload( + {1, 2, 3}, 100 ) with self.assertRaises(ClientException) as e: result_of( - im_client.create( + self.im_client.create( storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret ) ) @@ -484,21 +490,14 @@ class ImmutableHTTPAPITests(SyncTestCase): If allocate buckets endpoint is called second time with different upload key on different shares, that creates the buckets. """ - im_client = StorageClientImmutables(self.http.client) - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret - ) + (upload_secret, lease_secret, storage_index, created) = self.create_upload( + {1, 2, 3}, 100 ) # Add same shares: created2 = result_of( - im_client.create( + self.im_client.create( storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret ) ) @@ -508,23 +507,15 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Once a share is finished uploading, it's possible to list it. """ - im_client = StorageClientImmutables(self.http.client) - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1, 2, 3}, 10, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10) # Initially there are no shares: - self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) # Upload shares 1 and 3: for share_number in [1, 3]: progress = result_of( - im_client.write_share_chunk( + self.im_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -535,22 +526,14 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertTrue(progress.finished) # Now shares 1 and 3 exist: - self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + self.assertEqual(result_of(self.im_client.list_shares(storage_index)), {1, 3}) def test_upload_bad_content_range(self): """ Malformed or invalid Content-Range headers to the immutable upload endpoint result in a 416 error. """ - im_client = StorageClientImmutables(self.http.client) - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"0" * 16 - result_of( - im_client.create( - storage_index, {1}, 10, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, created) = self.create_upload({1}, 10) def check_invalid(bad_content_range_value): client = StorageClientImmutables( @@ -579,29 +562,20 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Listing unknown storage index's shares results in empty list of shares. """ - im_client = StorageClientImmutables(self.http.client) storage_index = b"".join(bytes([i]) for i in range(16)) - self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) def test_upload_non_existent_storage_index(self): """ Uploading to a non-existent storage index or share number results in 404. """ - im_client = StorageClientImmutables(self.http.client) - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1}, 10, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, _) = self.create_upload({1}, 10) def unknown_check(storage_index, share_number): with self.assertRaises(ClientException) as e: result_of( - im_client.write_share_chunk( + self.im_client.write_share_chunk( storage_index, share_number, upload_secret, From faacde4e32a68543d6e7e92ea80a705e44208afa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:41:32 -0500 Subject: [PATCH 449/916] Conflicting writes. --- src/allmydata/storage/http_server.py | 10 +++++---- src/allmydata/test/test_storage_http.py | 27 +++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6f9033380..4613a73c3 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -31,7 +31,7 @@ from cbor2 import dumps, loads from .server import StorageServer from .http_common import swissnum_auth_header, Secrets from .common import si_a2b -from .immutable import BucketWriter +from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet @@ -253,9 +253,11 @@ class HTTPServer(object): request.setResponseCode(http.NOT_FOUND) return b"" - finished = bucket.write(offset, data) - - # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. + try: + finished = bucket.write(offset, data) + except ConflictingWriteError: + request.setResponseCode(http.CONFLICT) + return b"" if finished: bucket.close() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2bad160e4..225784cb6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -626,9 +626,32 @@ class ImmutableHTTPAPITests(SyncTestCase): """ If an uploaded chunk conflicts with an already uploaded chunk, a CONFLICT error is returned. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) + + # Write: + result_of( + self.im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"0" * 10, + ) + ) + + # Conflicting write: + with self.assertRaises(ClientException) as e: + result_of( + self.im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"0123456789", + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_of_wrong_storage_index_fails(self): """ From bae5d58ab97468e6fdefc8570a1f448ff3c30631 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 13:07:34 -0500 Subject: [PATCH 450/916] Another test. --- src/allmydata/test/test_storage_http.py | 29 +++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 225784cb6..e61a023bd 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -594,9 +594,34 @@ class ImmutableHTTPAPITests(SyncTestCase): """ If a storage index has multiple shares, uploads to different shares are stored separately and can be downloaded separately. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10) + result_of( + self.im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"1" * 10, + ) + ) + result_of( + self.im_client.write_share_chunk( + storage_index, + 2, + upload_secret, + 0, + b"2" * 10, + ) + ) + self.assertEqual( + result_of(self.im_client.read_share_chunk(storage_index, 1, 0, 10)), + b"1" * 10, + ) + self.assertEqual( + result_of(self.im_client.read_share_chunk(storage_index, 2, 0, 10)), + b"2" * 10, + ) def test_bucket_allocated_with_new_shares(self): """ From 45ee5e33460fa06f3c557c17b9697c27f09c4aa4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 13:08:34 -0500 Subject: [PATCH 451/916] Done elsewhere. --- src/allmydata/test/test_storage_http.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e61a023bd..ac52303f6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -623,30 +623,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"2" * 10, ) - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - - def test_bucket_allocation_new_upload_secret(self): - """ - If a bucket was allocated with one upload secret, and a different upload - key is used to allocate the bucket again, the second allocation fails. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - - def test_upload_with_wrong_upload_secret_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a From 5d9e0c9bca14ace590b842250efa69bb092d0b48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 13:14:27 -0500 Subject: [PATCH 452/916] Not found tests and implementation. --- src/allmydata/storage/http_server.py | 9 ++++-- src/allmydata/test/test_storage_http.py | 41 ++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4613a73c3..e7aa12f55 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -302,8 +302,13 @@ class HTTPServer(object): offset, end = range_header.ranges[0] assert end != None # TODO support this case - # TODO if not found, 404 - bucket = self._storage_server.get_buckets(storage_index)[share_number] + try: + bucket = self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + request.setResponseCode(http.NOT_FOUND) + return b"" + + # TODO limit memory usage data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) # TODO set content-range on response. We we need to expand the diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index ac52303f6..b96eebb96 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -654,19 +654,52 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(e.exception.code, http.NOT_FOUND) + def upload(self, share_number): + """ + Create a share, return (storage_index). + """ + (upload_secret, _, storage_index, _) = self.create_upload({share_number}, 26) + result_of( + self.im_client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + b"abcdefghijklmnopqrstuvwxyz", + ) + ) + return storage_index + def test_read_of_wrong_storage_index_fails(self): """ Reading from unknown storage index results in 404. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + with self.assertRaises(ClientException) as e: + result_of( + self.im_client.read_share_chunk( + b"1" * 16, + 1, + 0, + 10, + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + storage_index = self.upload(1) + with self.assertRaises(ClientException) as e: + result_of( + self.im_client.read_share_chunk( + storage_index, + 7, # different share number + 0, + 10, + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_with_negative_offset_fails(self): """ From 7db1ddd8750d429d5a77ef3e3c423f99fe89fbad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:12:17 -0500 Subject: [PATCH 453/916] Implement Range header validation. --- docs/proposed/http-storage-node-protocol.rst | 2 +- src/allmydata/storage/http_server.py | 15 +++++--- src/allmydata/test/test_storage_http.py | 40 ++++++++++++++++---- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 4d8d60560..315546b8a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -644,7 +644,7 @@ Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. -Multiple ranges in a single request are *not* supported. +Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index e7aa12f55..89c9ca6fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -291,16 +291,19 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 # 2. missing range header should have response code 200 and return whole thing - # 3. malformed range header should result in error? or return everything? - # 4. non-bytes range results in error - # 5. ranges make sense semantically (positive, etc.) - # 6. multiple ranges fails with error # 7. missing end of range means "to the end of share" range_header = parse_range_header(request.getHeader("range")) + if ( + range_header is None + or range_header.units != "bytes" + or len(range_header.ranges) > 1 # more than one range + or range_header.ranges[0][1] is None # range without end + ): + request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) + return b"" + offset, end = range_header.ranges[0] - assert end != None # TODO support this case try: bucket = self._storage_server.get_buckets(storage_index)[share_number] diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b96eebb96..f115d632c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -703,14 +703,38 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_read_with_negative_offset_fails(self): """ - The offset for reads cannot be negative. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + Malformed or unsupported Range headers result in 416 (requested range + not satisfiable) error. """ + storage_index = self.upload(1) - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. + def check_bad_range(bad_range_value): + client = StorageClientImmutables( + StorageClientWithHeadersOverride( + self.http.client, {"range": bad_range_value} + ) + ) + + with self.assertRaises(ClientException) as e: + result_of( + client.read_share_chunk( + storage_index, + 1, + 0, + 10, + ) + ) + self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) + + check_bad_range("molluscs=0-9") + check_bad_range("bytes=-2-9") + check_bad_range("bytes=0--10") + check_bad_range("bytes=-300-") + check_bad_range("bytes=") + # Multiple ranges are currently unsupported, even if they're + # semantically valid under HTTP: + check_bad_range("bytes=0-5, 6-7") + # Ranges without an end are currently unsupported, even if they're + # semantically valid under HTTP. + check_bad_range("bytes=0-") - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ From 416af7328cd0a89930c719882ab36ce4f3e74839 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:31:09 -0500 Subject: [PATCH 454/916] Support lack of Range header. --- src/allmydata/storage/http_server.py | 26 ++++++++++++------ src/allmydata/test/test_storage_http.py | 36 ++++++++++++++++++++----- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 89c9ca6fa..88de18f12 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -291,8 +291,24 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # 2. missing range header should have response code 200 and return whole thing - # 7. missing end of range means "to the end of share" + try: + bucket = self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + request.setResponseCode(http.NOT_FOUND) + return b"" + + if request.getHeader("range") is None: + # Return the whole thing. + start = 0 + while True: + # TODO should probably yield to event loop occasionally... + data = bucket.read(start, start + 65536) + if not data: + request.finish() + return + request.write(data) + start += len(data) + range_header = parse_range_header(request.getHeader("range")) if ( range_header is None @@ -305,12 +321,6 @@ class HTTPServer(object): offset, end = range_header.ranges[0] - try: - bucket = self._storage_server.get_buckets(storage_index)[share_number] - except KeyError: - request.setResponseCode(http.NOT_FOUND) - return b"" - # TODO limit memory usage data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index f115d632c..bbda00bf6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -46,6 +46,7 @@ from ..storage.http_client import ( ImmutableCreateResult, UploadProgress, StorageClientGeneral, + _encode_si, ) from ..storage.common import si_b2a @@ -654,21 +655,26 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(e.exception.code, http.NOT_FOUND) - def upload(self, share_number): + def upload(self, share_number, data_length=26): """ - Create a share, return (storage_index). + Create a share, return (storage_index, uploaded_data). """ - (upload_secret, _, storage_index, _) = self.create_upload({share_number}, 26) + uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ + :data_length + ] + (upload_secret, _, storage_index, _) = self.create_upload( + {share_number}, data_length + ) result_of( self.im_client.write_share_chunk( storage_index, share_number, upload_secret, 0, - b"abcdefghijklmnopqrstuvwxyz", + uploaded_data, ) ) - return storage_index + return storage_index, uploaded_data def test_read_of_wrong_storage_index_fails(self): """ @@ -689,7 +695,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Reading from unknown storage index results in 404. """ - storage_index = self.upload(1) + storage_index, _ = self.upload(1) with self.assertRaises(ClientException) as e: result_of( self.im_client.read_share_chunk( @@ -706,7 +712,7 @@ class ImmutableHTTPAPITests(SyncTestCase): Malformed or unsupported Range headers result in 416 (requested range not satisfiable) error. """ - storage_index = self.upload(1) + storage_index, _ = self.upload(1) def check_bad_range(bad_range_value): client = StorageClientImmutables( @@ -738,3 +744,19 @@ class ImmutableHTTPAPITests(SyncTestCase): # semantically valid under HTTP. check_bad_range("bytes=0-") + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole immutable. + """ + storage_index, uploaded_data = self.upload(1, data_length) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/immutable/{}/1".format(_encode_si(storage_index)) + ), + ) + ) + self.assertEqual(response.code, http.OK) + self.assertEqual(result_of(response.content()), uploaded_data) From aa68be645f80dde5a52dc5a32d74304921e97288 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:47:56 -0500 Subject: [PATCH 455/916] Return Content-Range in responses. --- src/allmydata/storage/http_server.py | 12 +++++------ src/allmydata/test/test_storage_http.py | 27 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 88de18f12..bb4ef6c00 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -24,6 +24,7 @@ from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header from werkzeug.routing import BaseConverter +from werkzeug.datastructures import ContentRange # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -323,11 +324,10 @@ class HTTPServer(object): # TODO limit memory usage data = bucket.read(offset, end - offset) + request.setResponseCode(http.PARTIAL_CONTENT) - # TODO set content-range on response. We we need to expand the - # BucketReader interface to return share's length. - # - # request.setHeader( - # "content-range", range_header.make_content_range(share_length).to_header() - # ) + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) return data diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index bbda00bf6..c864f923c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -760,3 +760,30 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(response.code, http.OK) self.assertEqual(result_of(response.content()), uploaded_data) + + def test_validate_content_range_response_to_read(self): + """ + The server responds to ranged reads with an appropriate Content-Range + header. + """ + storage_index, _ = self.upload(1, 26) + + def check_range(requested_range, expected_response): + headers = Headers() + headers.setRawHeaders("range", [requested_range]) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/immutable/{}/1".format(_encode_si(storage_index)) + ), + headers=headers, + ) + ) + self.assertEqual( + response.headers.getRawHeaders("content-range"), [expected_response] + ) + + check_range("bytes=0-10", "bytes 0-10/*") + # Can't go beyond the end of the immutable! + check_range("bytes=10-100", "bytes 10-25/*") From fa2f142bc9ecd507312614ab134e53a74f7c8ce4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:50:09 -0500 Subject: [PATCH 456/916] Another ticket. --- src/allmydata/storage/http_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bb4ef6c00..491fcd39b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -247,6 +247,7 @@ class HTTPServer(object): offset = content_range.start # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = request.content.read(content_range.stop - content_range.start + 1) try: bucket = self._uploads[storage_index].shares[share_number] @@ -303,6 +304,7 @@ class HTTPServer(object): start = 0 while True: # TODO should probably yield to event loop occasionally... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = bucket.read(start, start + 65536) if not data: request.finish() @@ -323,6 +325,7 @@ class HTTPServer(object): offset, end = range_header.ranges[0] # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) From b049d4a792efa69c8f82ef0800380694b41cd5f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:52:47 -0500 Subject: [PATCH 457/916] Fix get_version with new API. --- src/allmydata/storage_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index abced41b3..528711539 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -75,7 +75,9 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict -from allmydata.storage.http_client import StorageClient, StorageClientImmutables +from allmydata.storage.http_client import ( + StorageClient, StorageClientImmutables, StorageClientGeneral, +) # who is responsible for de-duplication? @@ -1108,7 +1110,7 @@ class _HTTPStorageServer(object): return _HTTPStorageServer(http_client=http_client, upload_secret=urandom(20)) def get_version(self): - return self._http_client.get_version() + return StorageClientGeneral(self._http_client).get_version() @defer.inlineCallbacks def allocate_buckets( From 7466ee25a8d94017735a0380fc232fd8ec472421 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:57:57 -0500 Subject: [PATCH 458/916] Don't send header if it makes no sense to do so. --- src/allmydata/storage/http_server.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 491fcd39b..f89a156a3 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -329,8 +329,11 @@ class HTTPServer(object): data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) + if len(data): + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) return data From abf3048ab347ae737344db9bc4701b1452f83ad1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 17:07:21 -0500 Subject: [PATCH 459/916] More passing HTTP IStorageServer tests. --- src/allmydata/storage_client.py | 10 ++++++++-- src/allmydata/test/test_istorageserver.py | 2 -- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 528711539..17a6f79a5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -58,7 +58,7 @@ from twisted.plugin import ( from eliot import ( log_call, ) -from foolscap.api import eventually +from foolscap.api import eventually, RemoteException from foolscap.reconnector import ( ReconnectionInfo, ) @@ -77,6 +77,7 @@ from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, + ClientException as HTTPClientException, ) @@ -1037,8 +1038,13 @@ class _FakeRemoteReference(object): """ local_object = attr.ib(type=object) + @defer.inlineCallbacks def callRemote(self, action, *args, **kwargs): - return getattr(self.local_object, action)(*args, **kwargs) + try: + result = yield getattr(self.local_object, action)(*args, **kwargs) + return result + except HTTPClientException as e: + raise RemoteException(e.args) @attr.s diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index f3083a4bd..95261ddb2 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1162,8 +1162,6 @@ class HTTPImmutableAPIsTests( "test_advise_corrupt_share", "test_bucket_advise_corrupt_share", "test_disconnection", - "test_matching_overlapping_writes", - "test_non_matching_overlapping_writes", } From 5aa00abc3da0624541fe9e74aa61355cd59af8c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 11 Feb 2022 15:02:14 -0500 Subject: [PATCH 460/916] Use the correct API (since direct returns break Python 2 imports) --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 17a6f79a5..c2e26b4b6 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1042,7 +1042,7 @@ class _FakeRemoteReference(object): def callRemote(self, action, *args, **kwargs): try: result = yield getattr(self.local_object, action)(*args, **kwargs) - return result + defer.returnValue(result) except HTTPClientException as e: raise RemoteException(e.args) From 0639f2c16c70b51b5e5a965aedaaa0c220830ec8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:35:43 -0500 Subject: [PATCH 461/916] Try to switch to modern Python 3 world. Temporarily switch image building to always happen. --- ...ckerfile.centos => Dockerfile.oraclelinux} | 2 +- .circleci/config.yml | 205 +++++++----------- 2 files changed, 74 insertions(+), 133 deletions(-) rename .circleci/{Dockerfile.centos => Dockerfile.oraclelinux} (97%) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.oraclelinux similarity index 97% rename from .circleci/Dockerfile.centos rename to .circleci/Dockerfile.oraclelinux index 9070d71d9..ee31643b4 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.oraclelinux @@ -1,5 +1,5 @@ ARG TAG -FROM centos:${TAG} +FROM oraclelinux:${TAG} ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse diff --git a/.circleci/config.yml b/.circleci/config.yml index daf985567..ac62d9ed9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,28 +15,20 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - "debian-9": - {} - "debian-10": + {} + - "debian-11": requires: - - "debian-9" + - "debian-10" - "ubuntu-20-04": {} - "ubuntu-18-04": requires: - "ubuntu-20-04" - - "ubuntu-16-04": - requires: - - "ubuntu-20-04" - - "fedora-29": - {} - - "fedora-28": - requires: - - "fedora-29" - - - "centos-8": + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": {} - "nixos": @@ -47,9 +39,9 @@ workflows: name: "NixOS 21.11" nixpkgs: "21.11" - # Test against PyPy 2.7 - - "pypy27-buster": - {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} # Test against Python 3: - "python37": @@ -74,7 +66,7 @@ workflows: requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - - "debian-9" + - "debian-10" - "typechecks": {} @@ -85,13 +77,13 @@ workflows: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide @@ -104,22 +96,19 @@ workflows: # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "build-image-debian-10": &DOCKERHUB_CONTEXT context: "dockerhub-auth" - - "build-image-debian-9": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-16-04": + - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-18-04": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-28": + - "build-image-fedora-35": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-29": - <<: *DOCKERHUB_CONTEXT - - "build-image-centos-8": - <<: *DOCKERHUB_CONTEXT - - "build-image-pypy27-buster": + - "build-image-oraclelinux-8": <<: *DOCKERHUB_CONTEXT + # Restore later as PyPy38 + #- "build-image-pypy27-buster": + # <<: *DOCKERHUB_CONTEXT - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -150,7 +139,7 @@ jobs: lint: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -168,7 +157,7 @@ jobs: codechecks3: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:3" + image: "cimg/python:3.9" steps: - "checkout" @@ -186,7 +175,7 @@ jobs: pyinstaller: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -209,10 +198,10 @@ jobs: command: | dist/Tahoe-LAFS/tahoe --version - debian-9: &DEBIAN + debian-10: &DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:9-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -226,7 +215,7 @@ jobs: # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "py27" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -299,24 +288,32 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" - pypy27-buster: + debian-11: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/pypy:buster-py2" + image: "tahoelafsci/debian:11-py3.9" user: "nobody" - environment: - <<: *UTF_8_ENVIRONMENT - # We don't do coverage since it makes PyPy far too slow: - TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" - # Since we didn't collect it, don't upload it. - UPLOAD_COVERAGE: "" + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + # Restore later using PyPy3.8 + # pypy27-buster: + # <<: *DEBIAN + # docker: + # - <<: *DOCKERHUB_AUTH + # image: "tahoelafsci/pypy:buster-py2" + # user: "nobody" + # environment: + # <<: *UTF_8_ENVIRONMENT + # # We don't do coverage since it makes PyPy far too slow: + # TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" + # # Since we didn't collect it, don't upload it. + # UPLOAD_COVERAGE: "" c-locale: <<: *DEBIAN @@ -364,23 +361,6 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - - ubuntu-16-04: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:16.04-py2.7" - user: "nobody" - - - ubuntu-18-04: &UBUNTU_18_04 - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py2.7" - user: "nobody" - - python37: <<: *UBUNTU_18_04 docker: @@ -405,10 +385,10 @@ jobs: user: "nobody" - centos-8: &RHEL_DERIV + oracelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/centos:8-py2" + image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -427,20 +407,11 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE - - fedora-28: + fedora-35: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:28-py" - user: "nobody" - - - fedora-29: - <<: *RHEL_DERIV - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:29-py" + image: "tahoelafsci/fedora:35-py3.9" user: "nobody" nixos: @@ -554,7 +525,7 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -566,7 +537,7 @@ jobs: docs: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -589,8 +560,8 @@ jobs: image: "cimg/base:2022.01" environment: - DISTRO: "tahoelafsci/:foo-py2" - TAG: "tahoelafsci/distro:-py2" + DISTRO: "tahoelafsci/:foo-py3.9" + TAG: "tahoelafsci/distro:-py3.9" PYTHON_VERSION: "tahoelafsci/distro:tag-py Date: Mon, 14 Feb 2022 10:40:38 -0500 Subject: [PATCH 462/916] Fix some references. --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac62d9ed9..b178d945b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,10 +43,6 @@ workflows: #- "pypy27-buster": # {} - # Test against Python 3: - - "python37": - {} - # Other assorted tasks and configurations - "lint": {} @@ -361,8 +357,8 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - python37: - <<: *UBUNTU_18_04 + ubuntu-18-04: &UBUNTU_18_04 + <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:18.04-py3.7" From d4810ce5b865ddcf4cfe3138b3a401e4034a3c67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:42:58 -0500 Subject: [PATCH 463/916] Get rid of duplicate. --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b178d945b..c90ce16b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,15 +279,6 @@ jobs: /tmp/venv/bin/codecov fi - - debian-10: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py3.7" - user: "nobody" - - debian-11: <<: *DEBIAN docker: From f0a81e1095a4a1ea90ba819e4e7b99a0b57f3617 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:44:44 -0500 Subject: [PATCH 464/916] Fix typo. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c90ce16b7..0dbcc1314 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,7 +372,7 @@ jobs: user: "nobody" - oracelinux-8: &RHEL_DERIV + oraclelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/oraclelinux:8-py3.8" From 5935d9977697c6ae1e5ffc4f6618084071b33c5b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:45:56 -0500 Subject: [PATCH 465/916] Fix name. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0dbcc1314..511cee87c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,7 +591,7 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-python37-ubuntu: + build-image-ubuntu-18.04: <<: *BUILD_IMAGE environment: From 4e133eb759daafce648fce593248d7930effe235 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:46:55 -0500 Subject: [PATCH 466/916] Fix name. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 511cee87c..31df8d6a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,7 +591,7 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-ubuntu-18.04: + build-image-ubuntu-18-04: <<: *BUILD_IMAGE environment: From 19a3d2acf7b6fc1871d277219724f6cdb23ff016 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:49:17 -0500 Subject: [PATCH 467/916] Fix some more. --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31df8d6a1..2b588fbb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,8 +105,6 @@ workflows: # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT - - "build-image-python37-ubuntu": - <<: *DOCKERHUB_CONTEXT jobs: @@ -609,7 +607,7 @@ jobs: PYTHON_VERSION: "3.9" - build-image-oracelinux: + build-image-oraclelinux-8: <<: *BUILD_IMAGE environment: From 6a17c07158d56c94f2a129509d326daff779be57 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:52:33 -0500 Subject: [PATCH 468/916] Drop unnecessary install. --- .circleci/Dockerfile.oraclelinux | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/Dockerfile.oraclelinux b/.circleci/Dockerfile.oraclelinux index ee31643b4..cf4c009d2 100644 --- a/.circleci/Dockerfile.oraclelinux +++ b/.circleci/Dockerfile.oraclelinux @@ -13,7 +13,6 @@ RUN yum install --assumeyes \ sudo \ make automake gcc gcc-c++ \ python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ From 13d23e3baffceab6d726ddad5a37e579f4217c01 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:57:44 -0500 Subject: [PATCH 469/916] The terminal is a lie. --- .circleci/Dockerfile.debian | 2 +- .circleci/Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 96c54736c..f12f19551 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,7 +1,7 @@ ARG TAG FROM debian:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 2fcc60f5a..22689f0c1 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,7 +1,7 @@ ARG TAG FROM ubuntu:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. From 0928a7993a588298e723c98bbe5d216ad713306f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:02:25 -0500 Subject: [PATCH 470/916] Rip out Python 2. --- .github/workflows/ci.yml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f65890d37..a4d8daca5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,14 +38,11 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 - 3.7 - 3.8 - 3.9 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - - os: macos-10.15 - python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -108,25 +105,6 @@ jobs: # Action for this, as of Jan 2021 it does not support Python coverage # files - only lcov files. Therefore, we use coveralls-python, the # coveralls.io-supplied Python reporter, for this. - # - # It is coveralls-python 1.x that has maintained compatibility - # with Python 2, while coveralls-python 3.x is compatible with - # Python 3. Sadly we can't use them both in the same workflow. - # - # The two versions of coveralls-python are somewhat mutually - # incompatible. Mixing these two different versions when - # reporting coverage to coveralls.io will lead to grief, since - # they get job IDs in different fashion. If we use both - # versions of coveralls in the same workflow, the finalizing - # step will be able to mark only part of the jobs as done, and - # the other part will be left hanging, never marked as done: it - # does not matter if we make an API call or `coveralls --finish` - # to indicate that CI has finished running. - # - # So we try to use the newer coveralls-python that is available - # via Python 3 (which is present in GitHub Actions tool cache, - # even when we're running Python 2.7 tests) throughout this - # workflow. - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 @@ -179,13 +157,10 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 - 3.7 - 3.9 include: # On macOS don't bother with 3.7, just to get faster builds. - - os: macos-10.15 - python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -242,12 +217,7 @@ jobs: - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "Python 2 integration tests" - if: ${{ matrix.python-version == '2.7' }} - run: tox -e integration - - name: Run "Python 3 integration tests" - if: ${{ matrix.python-version != '2.7' }} run: tox -e integration3 - name: Upload eliot.log in case of failure @@ -267,7 +237,7 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 + - 3.9 steps: From 34fe6a41edd3c4468bfd78969b14cfed5d890a8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:05:31 -0500 Subject: [PATCH 471/916] Fix Fedora package name. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b588fbb7..cd26a5de8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -396,7 +396,7 @@ jobs: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:35-py3.9" + image: "tahoelafsci/fedora:35-py3" user: "nobody" nixos: @@ -621,7 +621,7 @@ jobs: environment: DISTRO: "fedora" TAG: "35" - PYTHON_VERSION: "3.9" + PYTHON_VERSION: "3" # build-image-pypy27-buster: # <<: *BUILD_IMAGE From cd33e1cfb384df56cc59a46258c781f708f901c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:10:56 -0500 Subject: [PATCH 472/916] Rip out more Python 2 stuff. --- Makefile | 8 ++++---- docs/release-checklist.rst | 2 +- tox.ini | 15 +-------------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 33a40df02..fa8351535 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ PYTHON=python export PYTHON PYFLAKES=flake8 export PYFLAKES -VIRTUAL_ENV=./.tox/py27 +VIRTUAL_ENV=./.tox/py37 SOURCES=src/allmydata static misc setup.py APPNAME=tahoe-lafs TEST_SUITE=allmydata @@ -33,9 +33,9 @@ default: ## Run all tests and code reports test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. - tox --develop -e codechecks + tox --develop -e codechecks3 # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py37,pypy27' + tox --develop -p auto -e 'py37' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -136,7 +136,7 @@ count-lines: # Here is a list of testing tools that can be run with 'python' from a # virtualenv in which Tahoe has been installed. There used to be Makefile # targets for each, but the exact path to a suitable python is now up to the -# developer. But as a hint, after running 'tox', ./.tox/py27/bin/python will +# developer. But as a hint, after running 'tox', ./.tox/py37/bin/python will # probably work. # src/allmydata/test/bench_dirnode.py diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 9588fd1a5..5697ab95f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py27,codechecks,docs,integration + - tox -e py37,codechecks3,docs,integration - these can fail (ideally they should not of course): diff --git a/tox.ini b/tox.ini index 34d555aa7..3125548f4 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,6 @@ # the tox-gh-actions package. [gh-actions] python = - 2.7: py27-coverage,codechecks 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -17,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks3,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -110,18 +109,6 @@ commands = coverage report -# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2 -# since that can give different results than flake8 on Python 3. -[testenv:codechecks] -basepython = python2.7 -setenv = - # If no positional arguments are given, try to run the checks on the - # entire codebase, including various pieces of supporting code. - DEFAULT_FILES=src integration static misc setup.py -commands = - flake8 {posargs:{env:DEFAULT_FILES}} - - [testenv:codechecks3] basepython = python3 deps = From 9428c5d45b14c432d0f369643c567cdc3a8f5e18 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:29:19 -0500 Subject: [PATCH 473/916] We can use modern PyInstaller. --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 3125548f4..e42958b28 100644 --- a/tox.ini +++ b/tox.ini @@ -218,9 +218,7 @@ extras = deps = {[testenv]deps} packaging - # PyInstaller 4.0 drops Python 2 support. When we finish porting to - # Python 3 we can reconsider this constraint. - pyinstaller < 4.0 + pyinstaller pefile ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build From d54294a6ec039a0bdbb6aeac138eec29996b6e5e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:09 -0500 Subject: [PATCH 474/916] News files. --- newsfragments/3327.minor | 0 newsfragments/3873.incompat | 1 + 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3327.minor create mode 100644 newsfragments/3873.incompat diff --git a/newsfragments/3327.minor b/newsfragments/3327.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat new file mode 100644 index 000000000..7e71b1504 --- /dev/null +++ b/newsfragments/3873.incompat @@ -0,0 +1 @@ +Dropped support for Python 2. \ No newline at end of file From 77e7f80a1a93bec0683588e5ce19eff2df638543 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:19 -0500 Subject: [PATCH 475/916] Try to update to Python 3. --- misc/build_helpers/run-deprecations.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index f99cf90aa..2ad335bd1 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -26,10 +26,10 @@ python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] CO class RunPP(protocol.ProcessProtocol): def outReceived(self, data): self.stdout.write(data) - sys.stdout.write(data) + sys.stdout.write(str(data, sys.stdout.encoding)) def errReceived(self, data): self.stderr.write(data) - sys.stderr.write(data) + sys.stderr.write(str(data, sys.stdout.encoding)) def processEnded(self, reason): signal = reason.value.signal rc = reason.value.exitCode @@ -100,17 +100,19 @@ def run_command(main): pp.stdout.seek(0) for line in pp.stdout.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) # includes newline pp.stderr.seek(0) for line in pp.stderr.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) if warnings: if config["warnings"]: - with open(config["warnings"], "wb") as f: + with open(config["warnings"], "w") as f: print("".join(warnings), file=f) print("ERROR: %d deprecation warnings found" % len(warnings)) sys.exit(1) From 0bcfc58c2279dac8c94293aa8c0ea7fa14d9ce31 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:24 -0500 Subject: [PATCH 476/916] Various version fixes. --- .circleci/config.yml | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd26a5de8..9050c30b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,8 +44,6 @@ workflows: # {} # Other assorted tasks and configurations - - "lint": - {} - "codechecks3": {} - "pyinstaller": @@ -130,24 +128,6 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - lint: - docker: - - <<: *DOCKERHUB_AUTH - image: "cimg/python:3.9" - - steps: - - "checkout" - - - run: - name: "Install tox" - command: | - pip install --user tox - - - run: - name: "Static-ish code checks" - command: | - ~/.local/bin/tox -e codechecks - codechecks3: docker: - <<: *DOCKERHUB_AUTH @@ -368,7 +348,8 @@ jobs: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:20.04" user: "nobody" - + environment: + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" oraclelinux-8: &RHEL_DERIV docker: @@ -376,7 +357,9 @@ jobs: image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" - environment: *UTF_8_ENVIRONMENT + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py38" # pip cannot install packages if the working directory is not readable. # We want to run a lot of steps as nobody instead of as root. From 3ccd051473f364cc37cbc1c94658cb55f2ac4320 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:40:03 -0500 Subject: [PATCH 477/916] Correct image. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9050c30b4..bf0cbd8b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,7 +346,7 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:20.04" + image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" environment: TAHOE_LAFS_TOX_ENVIRONMENT: "py39" From d976524bb09debd3f396da2a7d0b54d3dbcff3f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:41:51 -0500 Subject: [PATCH 478/916] Let's see if this is necessary any more. --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index e42958b28..dcbcc3ab6 100644 --- a/tox.ini +++ b/tox.ini @@ -211,9 +211,6 @@ commands = sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] -# We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) -# is broken when this feature is enabled. -install_command = python -m pip install --no-use-pep517 {opts} {packages} extras = deps = {[testenv]deps} From dfe7de54a25868f8cdb636f03371d991ce9a031c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 09:59:04 -0500 Subject: [PATCH 479/916] Upgrade some versions. --- tox.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index dcbcc3ab6..3916b807c 100644 --- a/tox.ini +++ b/tox.ini @@ -34,12 +34,10 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - # - # For now these are versions that support Python 2. - pip==20.3.4 - setuptools==44.1.1 - wheel==0.36.2 - subunitreporter==19.3.2 + pip==22.0.3 + setuptools==60.9.1 + wheel==0.37.1 + subunitreporter==22.2.0 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is # guaranteed to cause things to break eventually as old certificates From 1fd8603673f6dc89fdd5a25f55fff5dfd82a46eb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:07:04 -0500 Subject: [PATCH 480/916] Use modern Docker version (with bugfixes for modern distributions). --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf0cbd8b4..ac054b7eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -535,6 +535,7 @@ jobs: steps: - "checkout" - "setup_remote_docker" + version: "20.10.11" - run: name: "Log in to Dockerhub" command: | From 4315f6a641893022b8c1c3d1c7e670cb56f9746c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:12:31 -0500 Subject: [PATCH 481/916] Run on Python 3. --- pyinstaller.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 875629c13..eece50757 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -11,7 +11,10 @@ import struct import sys -if not hasattr(sys, 'real_prefix'): +try: + import allmydata + del allmydata +except ImportError: sys.exit("Please run inside a virtualenv with Tahoe-LAFS installed.") From 95c32ef2eec363b369a3439aa1eb39e7a6aefe7d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:13:35 -0500 Subject: [PATCH 482/916] Fix syntax. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac054b7eb..afb82f79a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -534,7 +534,7 @@ jobs: steps: - "checkout" - - "setup_remote_docker" + - setup_remote_docker: version: "20.10.11" - run: name: "Log in to Dockerhub" From 7ea106a018a92f13ba73fce9c8ac9691315b4284 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:19:56 -0500 Subject: [PATCH 483/916] Switch back to building Docker images on a schedule. --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index afb82f79a..7a4be90ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,13 +71,13 @@ workflows: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From be2590f9b8b95fccc795e0df47a89df58e882cc9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:20:29 -0500 Subject: [PATCH 484/916] Python 2 is now unsupported. --- README.rst | 9 ++++----- setup.py | 38 ++++++++++---------------------------- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index 0b73b520e..317378fae 100644 --- a/README.rst +++ b/README.rst @@ -53,12 +53,11 @@ For more detailed instructions, read `Installing Tahoe-LAFS `__ to learn how to set up your first Tahoe-LAFS node. -🐍 Python 3 Support --------------------- +🐍 Python 2 +----------- -Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. -System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. -Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. +Python 3.7 or later is now required. +If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1. 🤖 Issues diff --git a/setup.py b/setup.py index c38c0bfec..da4950d2e 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,7 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 21.7.0 ; python_version > '3.0'", + "foolscap >= 21.7.0", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] @@ -106,16 +105,10 @@ install_requires = [ # for 'tahoe invite' and 'tahoe join' "magic-wormhole >= 0.10.2", - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - "eliot ~= 1.7 ; python_version < '3.0'", - # On Python 3, we want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0 ; python_version > '3.0'", + # We want a new enough version to support custom JSON encoders. + "eliot >= 1.13.0", - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - "pyrsistent < 0.17.0 ; python_version < '3.0'", - "pyrsistent ; python_version > '3.0'", + "pyrsistent", # A great way to define types of values. "attrs >= 18.2.0", @@ -135,14 +128,8 @@ install_requires = [ # Linux distribution detection: "distro >= 1.4.0", - # Backported configparser for Python 2: - "configparser ; python_version < '3.0'", - - # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python - # 2 doesn't actually need this, since HTTP storage protocol isn't supported - # there, so we just pick whatever version so that code imports. - "collections-extended >= 2.0.2 ; python_version > '3.0'", - "collections-extended ; python_version < '3.0'", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. + "collections-extended >= 2.0.2", # HTTP server and client "klein", @@ -201,8 +188,7 @@ trove_classifiers=[ "Natural Language :: English", "Programming Language :: C", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", "Topic :: Utilities", "Topic :: System :: Systems Administration", "Topic :: System :: Filesystems", @@ -229,7 +215,7 @@ def run_command(args, cwd=None): use_shell = sys.platform == "win32" try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+ print("Warning: unable to run %r." % (" ".join(args),)) print(e) return None @@ -380,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and Python 3.7 or later. - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", + # We support Python 3.7 or later. + python_requires=">=3.7", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See @@ -400,10 +386,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "tox", "pytest", "pytest-twisted", - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - "decorator < 5", "hypothesis >= 3.6.1", "towncrier", "testtools", From bd907289468858d1a8e0d0a46a383ffb03516ce2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:26:54 -0500 Subject: [PATCH 485/916] Re-add missing environment. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a4be90ab..b1d73e89f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -264,6 +264,7 @@ jobs: image: "tahoelafsci/debian:11-py3.9" user: "nobody" environment: + <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" # Restore later using PyPy3.8 @@ -349,6 +350,7 @@ jobs: image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" environment: + <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" oraclelinux-8: &RHEL_DERIV From 3255f93a5c1f94af75d05a2d07bf8851d74df369 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:47:22 -0500 Subject: [PATCH 486/916] Try newer version of Chutney. --- integration/conftest.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index ef5c518a8..e284b5cba 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -462,10 +462,8 @@ def chutney(reactor, temp_dir): ) pytest_twisted.blockon(proto.done) - # XXX: Here we reset Chutney to the last revision known to work - # with Python 2, as a workaround for Chutney moving to Python 3. - # When this is no longer necessary, we will have to drop this and - # add '--depth=1' back to the above 'git clone' subprocess. + # XXX: Here we reset Chutney to a specific revision known to work, + # since there are no stability guarantees or releases yet. proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -473,7 +471,7 @@ def chutney(reactor, temp_dir): ( 'git', '-C', chutney_dir, 'reset', '--hard', - '99bd06c7554b9113af8c0877b6eca4ceb95dcbaa' + 'c825cba0bcd813c644c6ac069deeb7347d3200ee' ), env=environ, ) From 6190399aef2f61f318bb5a0c78e5a1e9f8a7e335 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:33:00 -0500 Subject: [PATCH 487/916] Just codechecks. --- .circleci/config.yml | 6 +++--- Makefile | 2 +- docs/release-checklist.rst | 2 +- tox.ini | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1d73e89f..cf0c66aff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ workflows: # {} # Other assorted tasks and configurations - - "codechecks3": + - "codechecks": {} - "pyinstaller": {} @@ -128,7 +128,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - codechecks3: + codechecks: docker: - <<: *DOCKERHUB_AUTH image: "cimg/python:3.9" @@ -144,7 +144,7 @@ jobs: - run: name: "Static-ish code checks" command: | - ~/.local/bin/tox -e codechecks3 + ~/.local/bin/tox -e codechecks pyinstaller: docker: diff --git a/Makefile b/Makefile index fa8351535..5cbd863a3 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ default: ## Run all tests and code reports test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. - tox --develop -e codechecks3 + tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time tox --develop -p auto -e 'py37' .PHONY: test-venv-coverage diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 5697ab95f..aa5531b59 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py37,codechecks3,docs,integration + - tox -e py37,codechecks,docs,integration - these can fail (ideally they should not of course): diff --git a/tox.ini b/tox.ini index 3916b807c..525b5428c 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage,typechecks,codechecks3 + 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage pypy-3.7: pypy3 @@ -16,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks3,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -107,7 +107,7 @@ commands = coverage report -[testenv:codechecks3] +[testenv:codechecks] basepython = python3 deps = # Newer versions of PyLint have buggy configuration From b7b71b2de7a6526564f5d509e29d6168ae1eb776 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:33:37 -0500 Subject: [PATCH 488/916] Clarify. --- newsfragments/3873.incompat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat index 7e71b1504..da8a5fb0e 100644 --- a/newsfragments/3873.incompat +++ b/newsfragments/3873.incompat @@ -1 +1 @@ -Dropped support for Python 2. \ No newline at end of file +Python 3.7 or later is now required; Python 2 is no longer supported. \ No newline at end of file From 21e288a4d0e4b9078edd17589fd72eeda4f5a079 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:35:18 -0500 Subject: [PATCH 489/916] Technically don't support 3.10 yet. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da4950d2e..8bb2b57aa 100644 --- a/setup.py +++ b/setup.py @@ -366,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. - python_requires=">=3.7", + # We support Python 3.7 or later. 3.10 is not supported quite yet. + python_requires=">=3.7, <3.10", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From 510102dab1f6a5b38a14bffd789bfd5188c8b6fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:37:18 -0500 Subject: [PATCH 490/916] Maybe we can use modern tor now. --- .github/workflows/ci.yml | 7 +------ newsfragments/3744.minor | 0 2 files changed, 1 insertion(+), 6 deletions(-) create mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4d8daca5..a0f7889b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,15 +170,10 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. - # We have to use an older version of Tor for running integration - # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.5.8 tor homebrew/cask - brew install tor@0.4.5.8 - brew link --overwrite tor@0.4.5.8 + brew install tor - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor new file mode 100644 index 000000000..e69de29bb From 3a859e3cac580370682c5124744dd486d580600c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:51:55 -0500 Subject: [PATCH 491/916] Try a version that matches Ubuntu's. --- .github/workflows/ci.yml | 7 ++++++- newsfragments/3744.minor | 0 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0f7889b5..8e6b0fd67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,10 +170,15 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor + # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. + # We have to use an older version of Tor for running integration + # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew install tor + brew extract --version 0.4.2.7 tor homebrew/cask + brew install tor@0.4.2.7 + brew link --overwrite tor@0.4.2.7 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor deleted file mode 100644 index e69de29bb..000000000 From 84094b5ca00bd1af8168030f48678748d9ba4faa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Feb 2022 09:31:12 -0500 Subject: [PATCH 492/916] Try version that Windows uses. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e6b0fd67..3a12f51f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,9 +176,9 @@ jobs: - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.2.7 tor homebrew/cask - brew install tor@0.4.2.7 - brew link --overwrite tor@0.4.2.7 + brew extract --version 0.4.6.9 tor homebrew/cask + brew install tor@0.4.6.9 + brew link --overwrite tor@0.4.6.9 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' From 2928a480ff317f4acd50c0572c69d4b0b566e70e Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 16 Feb 2022 21:46:24 -0700 Subject: [PATCH 493/916] RSA key-size is not configurable, it's 2048bits --- src/allmydata/client.py | 32 ++++----------------- src/allmydata/crypto/rsa.py | 8 ++---- src/allmydata/nodemaker.py | 4 +-- src/allmydata/test/common.py | 3 -- src/allmydata/test/common_system.py | 5 ---- src/allmydata/test/mutable/test_problems.py | 3 +- src/allmydata/test/mutable/util.py | 13 ++++----- src/allmydata/test/no_network.py | 2 -- 8 files changed, 16 insertions(+), 54 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 645e157b6..56ecdc6ed 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -168,29 +168,12 @@ class SecretHolder(object): class KeyGenerator(object): """I create RSA keys for mutable files. Each call to generate() returns a - single keypair. The keysize is specified first by the keysize= argument - to generate(), then with a default set by set_default_keysize(), then - with a built-in default of 2048 bits.""" - def __init__(self): - self.default_keysize = 2048 + single keypair.""" - def set_default_keysize(self, keysize): - """Call this to override the size of the RSA keys created for new - mutable files which don't otherwise specify a size. This will affect - all subsequent calls to generate() without a keysize= argument. The - default size is 2048 bits. Test cases should call this method once - during setup, to cause me to create smaller keys, so the unit tests - run faster.""" - self.default_keysize = keysize - - def generate(self, keysize=None): + def generate(self): """I return a Deferred that fires with a (verifyingkey, signingkey) - pair. I accept a keysize in bits (2048 bit keys are standard, smaller - keys are used for testing). If you do not provide a keysize, I will - use my default, which is set by a call to set_default_keysize(). If - set_default_keysize() has never been called, I will create 2048 bit - keys.""" - keysize = keysize or self.default_keysize + pair. The returned key will be 2048 bit""" + keysize = 2048 # RSA key generation for a 2048 bit key takes between 0.8 and 3.2 # secs signer, verifier = rsa.create_signing_keypair(keysize) @@ -993,9 +976,6 @@ class _Client(node.Node, pollmixin.PollMixin): helper_furlfile = self.config.get_private_path("helper.furl").encode(get_filesystem_encoding()) self.tub.registerReference(self.helper, furlFile=helper_furlfile) - def set_default_mutable_keysize(self, keysize): - self._key_generator.set_default_keysize(keysize) - def _get_tempdir(self): """ Determine the path to the directory where temporary files for this node @@ -1096,8 +1076,8 @@ class _Client(node.Node, pollmixin.PollMixin): def create_immutable_dirnode(self, children, convergence=None): return self.nodemaker.create_immutable_directory(children, convergence) - def create_mutable_file(self, contents=None, keysize=None, version=None): - return self.nodemaker.create_mutable_file(contents, keysize, + def create_mutable_file(self, contents=None, version=None): + return self.nodemaker.create_mutable_file(contents, version=version) def upload(self, uploadable, reactor=None): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index d290388da..95cf01413 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -81,13 +81,9 @@ def create_signing_keypair_from_string(private_key_der): raise ValueError( "Private Key did not decode to an RSA key" ) - if priv_key.key_size < 2048: + if priv_key.key_size != 2048: raise ValueError( - "Private Key is smaller than 2048 bits" - ) - if priv_key.key_size > (2048 * 8): - raise ValueError( - "Private Key is unreasonably large" + "Private Key must be 2048 bits" ) return priv_key, priv_key.public_key() diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 6b0b77c5c..23ba4b451 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -126,12 +126,12 @@ class NodeMaker(object): return self._create_dirnode(filenode) return None - def create_mutable_file(self, contents=None, keysize=None, version=None): + def create_mutable_file(self, contents=None, version=None): if version is None: version = self.mutable_file_default n = MutableFileNode(self.storage_broker, self.secret_holder, self.default_encoding_parameters, self.history) - d = self.key_generator.generate(keysize) + d = self.key_generator.generate() d.addCallback(n.create_with_keys, contents, version=version) d.addCallback(lambda res: n) return d diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 4d58fd0e5..b652b2e48 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -133,9 +133,6 @@ from subprocess import ( PIPE, ) -TEST_RSA_KEY_SIZE = 522 -TEST_RSA_KEY_SIZE = 2048 - EMPTY_CLIENT_CONFIG = config_from_string( "/dev/null", "tub.port", diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 0c424136a..9851d2b91 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -34,7 +34,6 @@ from twisted.python.filepath import ( ) from .common import ( - TEST_RSA_KEY_SIZE, SameProcessStreamEndpointAssigner, ) @@ -736,7 +735,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): c = yield client.create_client(basedirs[0]) c.setServiceParent(self.sparent) self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) with open(os.path.join(basedirs[0],"private","helper.furl"), "r") as f: helper_furl = f.read() @@ -754,7 +752,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): c = yield client.create_client(basedirs[i]) c.setServiceParent(self.sparent) self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) log.msg("STARTING") yield self.wait_for_connections() log.msg("CONNECTED") @@ -838,7 +835,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def _stopped(res): new_c = yield client.create_client(self.getdir("client%d" % num)) self.clients[num] = new_c - new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) new_c.setServiceParent(self.sparent) d.addCallback(_stopped) d.addCallback(lambda res: self.wait_for_connections()) @@ -877,7 +873,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): c = yield client.create_client(basedir.path) self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) self.numclients += 1 if add_to_sparent: c.setServiceParent(self.sparent) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 4bcb8161b..d3a779905 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -26,7 +26,6 @@ from allmydata.mutable.common import \ NotEnoughServersError from allmydata.mutable.publish import MutableData from allmydata.storage.common import storage_index_to_dir -from ..common import TEST_RSA_KEY_SIZE from ..no_network import GridTestMixin from .. import common_util as testutil from ..common_util import DevNullDictionary @@ -219,7 +218,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): # use #467 static-server-selection to disable permutation and force # the choice of server for share[0]. - d = nm.key_generator.generate(TEST_RSA_KEY_SIZE) + d = nm.key_generator.generate() def _got_key(keypair): (pubkey, privkey) = keypair nm.key_generator = SameKeyGenerator(pubkey, privkey) diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index dac61a6e3..bed350652 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -25,7 +25,6 @@ from allmydata.storage_client import StorageFarmBroker from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.mutable.publish import MutableData from ..common import ( - TEST_RSA_KEY_SIZE, EMPTY_CLIENT_CONFIG, ) @@ -287,7 +286,7 @@ def make_storagebroker_with_peers(peers): return storage_broker -def make_nodemaker(s=None, num_peers=10, keysize=TEST_RSA_KEY_SIZE): +def make_nodemaker(s=None, num_peers=10): """ Make a ``NodeMaker`` connected to some number of fake storage servers. @@ -298,20 +297,20 @@ def make_nodemaker(s=None, num_peers=10, keysize=TEST_RSA_KEY_SIZE): the node maker. """ storage_broker = make_storagebroker(s, num_peers) - return make_nodemaker_with_storage_broker(storage_broker, keysize) + return make_nodemaker_with_storage_broker(storage_broker) -def make_nodemaker_with_peers(peers, keysize=TEST_RSA_KEY_SIZE): +def make_nodemaker_with_peers(peers): """ Make a ``NodeMaker`` connected to the given storage servers. :param list peers: The storage servers to associate with the node maker. """ storage_broker = make_storagebroker_with_peers(peers) - return make_nodemaker_with_storage_broker(storage_broker, keysize) + return make_nodemaker_with_storage_broker(storage_broker) -def make_nodemaker_with_storage_broker(storage_broker, keysize): +def make_nodemaker_with_storage_broker(storage_broker): """ Make a ``NodeMaker`` using the given storage broker. @@ -319,8 +318,6 @@ def make_nodemaker_with_storage_broker(storage_broker, keysize): """ sh = client.SecretHolder(b"lease secret", b"convergence secret") keygen = client.KeyGenerator() - if keysize: - keygen.set_default_keysize(keysize) nodemaker = NodeMaker(storage_broker, sh, None, None, None, {"k": 3, "n": 10}, SDMF_VERSION, keygen) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 97cb371e6..ed742e624 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -61,7 +61,6 @@ from allmydata.storage_client import ( _StorageServer, ) from .common import ( - TEST_RSA_KEY_SIZE, SameProcessStreamEndpointAssigner, ) @@ -393,7 +392,6 @@ class NoNetworkGrid(service.MultiService): if not c: c = yield create_no_network_client(clientdir) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) c.nodeid = clientid c.short_nodeid = b32encode(clientid).lower()[:8] From 025a3bb455f92ba92191eba5b010e9ace8bd27f1 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 16 Feb 2022 22:00:02 -0700 Subject: [PATCH 494/916] news --- newsfragments/3828.feature | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 newsfragments/3828.feature diff --git a/newsfragments/3828.feature b/newsfragments/3828.feature new file mode 100644 index 000000000..f498421c8 --- /dev/null +++ b/newsfragments/3828.feature @@ -0,0 +1,8 @@ +Mutables' RSA keys are spec'd at 2048 bits + +Some code existed to allow tests to shorten this and it's +conceptually possible a modified client produced mutables +with different key-sizes. However, the spec says that they +must be 2048 bits. If you happen to have a capability with +a key-size different from 2048 you may use 1.17.1 or earlier +to read the content. From a7e4add602b390f2e7f482725182ce8e62475ac2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:25:13 -0500 Subject: [PATCH 495/916] Simplify. --- .github/workflows/ci.yml | 6 ++---- tox.ini | 15 +++------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a12f51f2..5f33dcd96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,9 +176,7 @@ jobs: - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.6.9 tor homebrew/cask - brew install tor@0.4.6.9 - brew link --overwrite tor@0.4.6.9 + brew install tor - name: Install Tor [Windows] if: matrix.os == 'windows-latest' @@ -218,7 +216,7 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" - run: tox -e integration3 + run: tox -e integration - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/tox.ini b/tox.ini index 525b5428c..52c199d41 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration minversion = 2.4 [testenv] @@ -86,21 +86,12 @@ commands = coverage: coverage report [testenv:integration] -setenv = - COVERAGE_PROCESS_START=.coveragerc -commands = - # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration} - coverage combine - coverage report - - -[testenv:integration3] basepython = python3 setenv = COVERAGE_PROCESS_START=.coveragerc + # Without this, temporary file paths are too long on macOS, breaking tor + commands = - python --version # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine From 52f0e18d6bfe2cc7e5e53d1b6cd77ecc2bb78e7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:26:00 -0500 Subject: [PATCH 496/916] Fix for overly-long temporary paths for unix sockets on macOS. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 52c199d41..c0bc159da 100644 --- a/tox.ini +++ b/tox.ini @@ -87,10 +87,13 @@ commands = [testenv:integration] basepython = python3 +platform = mylinux: linux + mymacos: darwin + mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc # Without this, temporary file paths are too long on macOS, breaking tor - + mymacos: TMPDIR=/tmp commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} From 5647b4aee0ce88b8e8761588453dc608f0307d25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:40:16 -0500 Subject: [PATCH 497/916] Try to fix macOS another way. --- .github/workflows/ci.yml | 5 +++++ tox.ini | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f33dcd96..90778c6ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,6 +216,11 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" + env: + # On macOS this is necessary to ensure unix socket paths for tor + # aren't too long. On Windows tox won't pass it through so it has no + # effect. On Linux it doesn't make a difference one way or another. + TMPDIR: "/tmp" run: tox -e integration - name: Upload eliot.log in case of failure diff --git a/tox.ini b/tox.ini index c0bc159da..9a28b7b30 100644 --- a/tox.ini +++ b/tox.ini @@ -92,8 +92,6 @@ platform = mylinux: linux mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc - # Without this, temporary file paths are too long on macOS, breaking tor - mymacos: TMPDIR=/tmp commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} From 82eb0d686e704dbdbb82ade998c6739ef78a6b96 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 Feb 2022 11:18:45 -0700 Subject: [PATCH 498/916] Update newsfragments/3828.feature Co-authored-by: Jean-Paul Calderone --- newsfragments/3828.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3828.feature b/newsfragments/3828.feature index f498421c8..d396439b0 100644 --- a/newsfragments/3828.feature +++ b/newsfragments/3828.feature @@ -1,4 +1,4 @@ -Mutables' RSA keys are spec'd at 2048 bits +The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification. Some code existed to allow tests to shorten this and it's conceptually possible a modified client produced mutables From c6ee41ab5c9413369fc697ce425ba64adc2cb417 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 10:49:39 -0500 Subject: [PATCH 499/916] Add PyPy3. --- .github/workflows/ci.yml | 4 ++++ tox.ini | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90778c6ba..b06a7534e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,10 +41,14 @@ jobs: - 3.7 - 3.8 - 3.9 + - pypy-3.7 + - pypy-3.8 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-latest python-version: 3.9 + - os: macos-latest + python-version: pypy3.8 steps: # See https://github.com/actions/checkout. A fetch-depth of 0 diff --git a/tox.ini b/tox.ini index 9a28b7b30..f628b8568 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,15 @@ python = 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage - pypy-3.7: pypy3 + pypy-3.7: pypy37 + pypy-3.8: pypy38 + pypy-3.9: pypy39 [pytest] twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy37,pypy38,pypy39,integration minversion = 2.4 [testenv] From e5c49c890b651f64a7b92c023f59eca945cc057c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 10:50:41 -0500 Subject: [PATCH 500/916] News file. --- newsfragments/3697.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3697.minor diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor new file mode 100644 index 000000000..356b42748 --- /dev/null +++ b/newsfragments/3697.minor @@ -0,0 +1 @@ +Added support for PyPy3 (3.7 and 3.8). \ No newline at end of file From f81cd6e595ff071a9e8e3d0aa0e491730a1e2dca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 10:57:36 -0500 Subject: [PATCH 501/916] Use an option also available on PyPy3. --- src/allmydata/test/test_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 44c7e1bee..3eb6b8a34 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -203,10 +203,10 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): # but on Windows we parse the whole command line string ourselves so # we have to have our own implementation of skipping these options. - # -t is a harmless option that warns about tabs so we can add it + # -B is a harmless option that prevents writing bytecode so we can add it # without impacting other behavior noticably. - out, err, returncode = run_bintahoe([u"--version"], python_options=[u"-t"]) - self.assertEqual(returncode, 0) + out, err, returncode = run_bintahoe([u"--version"], python_options=[u"-B"]) + self.assertEqual(returncode, 0, f"Out:\n{out}\nErr:\n{err}") self.assertTrue(out.startswith(allmydata.__appname__ + '/')) def test_help_eliot_destinations(self): From 9d9ec698e0b4fc0c67546a2fb6db3737ddfda169 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 11:07:56 -0500 Subject: [PATCH 502/916] Add support for Python 3.10. --- .github/workflows/ci.yml | 3 +++ newsfragments/3697.minor | 2 +- setup.py | 4 ++-- tox.ini | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b06a7534e..2d0290b86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: - 3.7 - 3.8 - 3.9 + - 3.10 - pypy-3.7 - pypy-3.8 include: @@ -49,6 +50,8 @@ jobs: python-version: 3.9 - os: macos-latest python-version: pypy3.8 + - os: macos-latest + python-version: 3.10 steps: # See https://github.com/actions/checkout. A fetch-depth of 0 diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor index 356b42748..8bc959086 100644 --- a/newsfragments/3697.minor +++ b/newsfragments/3697.minor @@ -1 +1 @@ -Added support for PyPy3 (3.7 and 3.8). \ No newline at end of file +Added support for Python 3.10 and PyPy3 (3.7 and 3.8). \ No newline at end of file diff --git a/setup.py b/setup.py index 8bb2b57aa..5285b5d08 100644 --- a/setup.py +++ b/setup.py @@ -366,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. 3.10 is not supported quite yet. - python_requires=">=3.7, <3.10", + # We support Python 3.7 or later. 3.11 is not supported yet. + python_requires=">=3.7, <3.11", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See diff --git a/tox.ini b/tox.ini index f628b8568..57489df89 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ python = 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage + 3.10: py310-coverage pypy-3.7: pypy37 pypy-3.8: pypy38 pypy-3.9: pypy39 @@ -18,7 +19,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy37,pypy38,pypy39,integration +envlist = typechecks,codechecks,py{37,38,39,310}-{coverage},pypy27,pypy37,pypy38,pypy39,integration minversion = 2.4 [testenv] From abe4c6a1f020899892266020f9a0d7f44cb21d00 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 11:09:49 -0500 Subject: [PATCH 503/916] Only support PyPy3 on Linux. --- .github/workflows/ci.yml | 11 ++++++----- newsfragments/3697.minor | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d0290b86..c67bdb007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,17 +42,18 @@ jobs: - 3.8 - 3.9 - 3.10 - - pypy-3.7 - - pypy-3.8 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-latest python-version: 3.9 - - os: macos-latest - python-version: pypy3.8 - os: macos-latest python-version: 3.10 - + # We only support PyPy on Linux at the moment. + - os: ubuntu-latest + python-version: pypy-3.7 + - os: ubuntu-latest + python-version: pypy-3.8 + steps: # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor index 8bc959086..0977d8a6f 100644 --- a/newsfragments/3697.minor +++ b/newsfragments/3697.minor @@ -1 +1 @@ -Added support for Python 3.10 and PyPy3 (3.7 and 3.8). \ No newline at end of file +Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only). \ No newline at end of file From 5ac3cb644f366747537327d2acb1a8728f0025ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 11:11:04 -0500 Subject: [PATCH 504/916] These are not numbers. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c67bdb007..0327014ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,21 +38,21 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 3.7 - - 3.8 - - 3.9 - - 3.10 + - "3.7" + - "3.8" + - "3.9" + - "3.10" include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-latest - python-version: 3.9 + python-version: "3.9" - os: macos-latest - python-version: 3.10 + python-version: "3.10" # We only support PyPy on Linux at the moment. - os: ubuntu-latest - python-version: pypy-3.7 + python-version: "pypy-3.7" - os: ubuntu-latest - python-version: pypy-3.8 + python-version: "pypy-3.8" steps: # See https://github.com/actions/checkout. A fetch-depth of 0 From 32cbc7b9dfe09492f85ed84c77b472448c42861a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Mar 2022 10:35:41 -0500 Subject: [PATCH 505/916] Function for getting SPKI hash. --- src/allmydata/storage/http_common.py | 22 ++++++++----- src/allmydata/test/test_storage_http.py | 42 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index af4224bd0..f570d45d7 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -1,15 +1,12 @@ """ Common HTTP infrastructure for the storge server. """ -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from enum import Enum from base64 import b64encode +from hashlib import sha256 + +from cryptography.x509 import Certificate +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat def swissnum_auth_header(swissnum): # type: (bytes) -> bytes @@ -23,3 +20,14 @@ class Secrets(Enum): LEASE_RENEW = "lease-renew-secret" LEASE_CANCEL = "lease-cancel-secret" UPLOAD = "upload-secret" + + +def get_spki_hash(certificate: Certificate) -> bytes: + """ + Get the public key hash, as per RFC 7469: base64 of sha256 of the public + key encoded in DER + Subject Public Key Info format. + """ + public_key_bytes = certificate.public_key().public_bytes( + Encoding.DER, PublicFormat.SubjectPublicKeyInfo + ) + return b64encode(sha256(public_key_bytes).digest()).strip() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 982e22859..e9c9e83f5 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -24,6 +24,7 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock +from cryptography.x509 import load_pem_x509_certificate from .common import SyncTestCase from ..storage.server import StorageServer @@ -41,6 +42,47 @@ from ..storage.http_client import ( ImmutableCreateResult, UploadProgress, ) +from ..storage.http_common import get_spki_hash + + +class HTTPFurlTests(SyncTestCase): + """Tests for HTTP furls.""" + + def test_spki_hash(self): + """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. + + The expected hash was generated using Appendix A instructions in the + RFC:: + + openssl x509 -noout -in certificate.pem -pubkey | \ + openssl asn1parse -noout -inform pem -out public.key + openssl dgst -sha256 -binary public.key | openssl enc -base64 + """ + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM=" + certificate_text = b"""\ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx +CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl +dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh +bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD +VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x +HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG +q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC +M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj +GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu +YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k +yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk +YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH ++fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C +i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs +2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ +PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr +ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG +-----END CERTIFICATE----- +""" + certificate = load_pem_x509_certificate(certificate_text) + self.assertEqual(get_spki_hash(certificate), expected_hash) def _post_process(params): From afe6b68ede0796bcbe56dc18bc3b771d437b7586 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Mar 2022 10:38:05 -0500 Subject: [PATCH 506/916] News file. --- newsfragments/3875.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3875.minor diff --git a/newsfragments/3875.minor b/newsfragments/3875.minor new file mode 100644 index 000000000..e69de29bb From 7146cff227ae7cf11a25961d90201d5661b65c8f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Mar 2022 10:40:39 -0500 Subject: [PATCH 507/916] Sketch of TLS listening and furl construction for the HTTP storage server. --- src/allmydata/storage/http_server.py | 59 +++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f885baa22..687c9f2c9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -7,28 +7,27 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on -else: - from typing import Dict, List, Set - +from typing import Dict, List, Set, Tuple, Optional +from pathlib import Path from functools import wraps from base64 import b64decode from klein import Klein from twisted.web import http +from twisted.internet.interfaces import IListeningPort +from twisted.internet.defer import Deferred +from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.web.server import Site import attr from werkzeug.http import parse_range_header, parse_content_range_header +from hyperlink import DecodedURL +from cryptography.x509 import load_pem_x509_certificate # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads from .server import StorageServer -from .http_common import swissnum_auth_header, Secrets +from .http_common import swissnum_auth_header, Secrets, get_spki_hash from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -301,3 +300,43 @@ class HTTPServer(object): # "content-range", range_header.make_content_range(share_length).to_header() # ) return data + + +def listen_tls( + server: HTTPServer, + hostname: str, + port: int, + private_key_path: Path, + cert_path: Path, + interface: Optional[str], +) -> Deferred[Tuple[DecodedURL, IListeningPort]]: + """ + Start a HTTPS storage server on the given port, return the fURL and the + listening port. + + The hostname is the external IP or hostname clients will connect to; it + does not modify what interfaces the server listens on. To set the + listening interface, use the ``interface`` argument. + """ + endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( + quoteStringArgument(str(private_key_path)), + quoteStringArgument(str(cert_path)), + port, + ) + if interface is not None: + endpoint_string += ":interface={}".format(quoteStringArgument(interface)) + endpoint = serverFromString(endpoint_string) + + def build_furl(listening_port: IListeningPort) -> DecodedURL: + furl = DecodedURL() + furl.fragment = "v=1" # HTTP-based + furl.host = hostname + furl.port = listening_port.getHost().port + furl.path = (server._swissnum,) + furl.user = get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())) + furl.scheme = "pb" + return furl + + return endpoint.listen(Site(server.get_resource())).addCallback( + lambda listening_port: (build_furl(listening_port), listening_port) + ) From 9f4f6668c0110e153879c950460a0bdfcde27a03 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:21:58 -0500 Subject: [PATCH 508/916] Tweaks. --- src/allmydata/storage/http_client.py | 36 +++++++++++++++------------- src/allmydata/storage/http_server.py | 3 ++- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2dc133b52..b532c292e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -7,22 +7,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from collections import defaultdict - - Optional = Set = defaultdict( - lambda: None - ) # some garbage to just make this module import -else: - # typing module not available in Python 2, and we only do type checking in - # Python 3 anyway. - from typing import Union, Set, Optional - from treq.testing import StubTreq +from typing import Union, Set, Optional from base64 import b64encode @@ -37,6 +22,8 @@ from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq +from treq.client import HTTPClient +from treq.testing import StubTreq from .http_common import swissnum_auth_header, Secrets from .common import si_b2a @@ -73,11 +60,26 @@ class StorageClient(object): def __init__( self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None + ): # type: (DecodedURL, bytes, Union[treq,StubTreq,HTTPClient]) -> None + """ + The URL is a HTTPS URL ("http://..."). To construct from a furl, use + ``StorageClient.from_furl()``. + """ + assert url.to_text().startswith("https://") self._base_url = url self._swissnum = swissnum self._treq = treq + @classmethod + def from_furl(cls, furl: DecodedURL) -> "StorageClient": + """ + Create a ``StorageClient`` for the given furl. + """ + assert furl.fragment == "v=1" + assert furl.scheme == "pb" + swissnum = furl.path[0].encode("ascii") + certificate_hash = furl.user.encode("ascii") + def _url(self, path): """Get a URL relative to the base URL.""" return self._base_url.click(path) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 687c9f2c9..da9eea22f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -303,6 +303,7 @@ class HTTPServer(object): def listen_tls( + reactor, server: HTTPServer, hostname: str, port: int, @@ -325,7 +326,7 @@ def listen_tls( ) if interface is not None: endpoint_string += ":interface={}".format(quoteStringArgument(interface)) - endpoint = serverFromString(endpoint_string) + endpoint = serverFromString(reactor, endpoint_string) def build_furl(listening_port: IListeningPort) -> DecodedURL: furl = DecodedURL() From 60bcd5fe9f41b811cb2630424935977c6e1db674 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:25:12 -0500 Subject: [PATCH 509/916] Address review comments. --- src/allmydata/test/test_storage_http.py | 47 +++++++++++++------------ 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index c864f923c..c15749fc8 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -374,7 +374,7 @@ class ImmutableHTTPAPITests(SyncTestCase): self.skipTest("Not going to bother supporting Python 2") super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - self.im_client = StorageClientImmutables(self.http.client) + self.imm_client = StorageClientImmutables(self.http.client) def create_upload(self, share_numbers, length): """ @@ -386,7 +386,7 @@ class ImmutableHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = urandom(16) created = result_of( - self.im_client.create( + self.imm_client.create( storage_index, share_numbers, length, @@ -407,7 +407,7 @@ class ImmutableHTTPAPITests(SyncTestCase): that's already done in test_storage.py. """ length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) + expected_data = bytes(range(100)) # Create a upload: (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) @@ -421,7 +421,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): remaining.empty(offset, offset + length) - return self.im_client.write_share_chunk( + return self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -465,7 +465,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = result_of( - self.im_client.read_share_chunk(storage_index, 1, offset, length) + self.imm_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) @@ -480,7 +480,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) with self.assertRaises(ClientException) as e: result_of( - self.im_client.create( + self.imm_client.create( storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret ) ) @@ -498,7 +498,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Add same shares: created2 = result_of( - self.im_client.create( + self.imm_client.create( storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret ) ) @@ -511,12 +511,12 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10) # Initially there are no shares: - self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) + self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) # Upload shares 1 and 3: for share_number in [1, 3]: progress = result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -527,7 +527,7 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertTrue(progress.finished) # Now shares 1 and 3 exist: - self.assertEqual(result_of(self.im_client.list_shares(storage_index)), {1, 3}) + self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), {1, 3}) def test_upload_bad_content_range(self): """ @@ -563,8 +563,8 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Listing unknown storage index's shares results in empty list of shares. """ - storage_index = b"".join(bytes([i]) for i in range(16)) - self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) + storage_index = bytes(range(16)) + self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) def test_upload_non_existent_storage_index(self): """ @@ -576,7 +576,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def unknown_check(storage_index, share_number): with self.assertRaises(ClientException) as e: result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -598,7 +598,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10) result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -607,7 +607,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 2, upload_secret, @@ -616,11 +616,11 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) self.assertEqual( - result_of(self.im_client.read_share_chunk(storage_index, 1, 0, 10)), + result_of(self.imm_client.read_share_chunk(storage_index, 1, 0, 10)), b"1" * 10, ) self.assertEqual( - result_of(self.im_client.read_share_chunk(storage_index, 2, 0, 10)), + result_of(self.imm_client.read_share_chunk(storage_index, 2, 0, 10)), b"2" * 10, ) @@ -633,7 +633,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Write: result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -645,7 +645,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Conflicting write: with self.assertRaises(ClientException) as e: result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -666,7 +666,7 @@ class ImmutableHTTPAPITests(SyncTestCase): {share_number}, data_length ) result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -682,7 +682,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ with self.assertRaises(ClientException) as e: result_of( - self.im_client.read_share_chunk( + self.imm_client.read_share_chunk( b"1" * 16, 1, 0, @@ -698,7 +698,7 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index, _ = self.upload(1) with self.assertRaises(ClientException) as e: result_of( - self.im_client.read_share_chunk( + self.imm_client.read_share_chunk( storage_index, 7, # different share number 0, @@ -732,9 +732,12 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) + # Bad unit check_bad_range("molluscs=0-9") + # Negative offsets check_bad_range("bytes=-2-9") check_bad_range("bytes=0--10") + # Negative offset no endpoint check_bad_range("bytes=-300-") check_bad_range("bytes=") # Multiple ranges are currently unsupported, even if they're From 4efa65d3dbfdd844e14537b6dd18683c8ed89092 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:29:26 -0500 Subject: [PATCH 510/916] Typo. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f89a156a3..9048359b1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -133,7 +133,7 @@ class StorageIndexUploads(object): # Map share number to BucketWriter shares = attr.ib(factory=dict) # type: Dict[int,BucketWriter] - # Mape share number to the upload secret (different shares might have + # Map share number to the upload secret (different shares might have # different upload secrets). upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] From 87ab56426ad1634ef97cd2ca3805cc07038cb2a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:38:31 -0500 Subject: [PATCH 511/916] Validate another edge case of bad storage index. --- src/allmydata/storage/http_server.py | 8 ++++++-- src/allmydata/test/test_storage_http.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 9048359b1..0e1969593 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -18,12 +18,13 @@ else: from functools import wraps from base64 import b64decode +import binascii from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header -from werkzeug.routing import BaseConverter +from werkzeug.routing import BaseConverter, ValidationError from werkzeug.datastructures import ContentRange # TODO Make sure to use pure Python versions? @@ -148,7 +149,10 @@ class StorageIndexConverter(BaseConverter): regex = "[" + str(rfc3548_alphabet, "ascii") + "]{26}" def to_python(self, value): - return si_a2b(value.encode("ascii")) + try: + return si_a2b(value.encode("ascii")) + except (AssertionError, binascii.Error, ValueError): + raise ValidationError("Invalid storage index") class HTTPServer(object): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index c15749fc8..14f4437b6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -191,10 +191,15 @@ class RouteConverterTests(SyncTestCase): self.adapter.match("/{}/".format("a" * 25), method="GET") def test_bad_characters_storage_index_is_not_parsed(self): - """An overly short storage_index string is not parsed.""" + """A storage_index string with bad characters is not parsed.""" with self.assertRaises(WNotFound): self.adapter.match("/{}_/".format("a" * 25), method="GET") + def test_invalid_storage_index_is_not_parsed(self): + """An invalid storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/nomd2a65ylxjbqzsw7gcfh4ivr/", method="GET") + # TODO should be actual swissnum SWISSNUM_FOR_TEST = b"abcd" From 8586028af82d4403d8d5e5b94f63bdbed84d758a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 09:13:21 -0500 Subject: [PATCH 512/916] News file. --- newsfragments/3876.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3876.minor diff --git a/newsfragments/3876.minor b/newsfragments/3876.minor new file mode 100644 index 000000000..e69de29bb From 7721c134f2815e50743398d21b95c6eb9c82cc64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 09:28:21 -0500 Subject: [PATCH 513/916] Change the semantics of HTTP bucket creation so that it's possible to have a different upload secret per upload. --- docs/proposed/http-storage-node-protocol.rst | 4 +- src/allmydata/storage/http_server.py | 19 ++--- src/allmydata/storage_client.py | 10 ++- src/allmydata/test/test_storage_http.py | 76 ++++++++++++++------ 4 files changed, 69 insertions(+), 40 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 315546b8a..0f534f0c5 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -493,8 +493,8 @@ Handling repeat calls: * If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. This is necessary to ensure retries work in the face of lost responses from the server. * If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. - In order to prevent storage servers from being able to mess with each other, this API call will fail, because the secret doesn't match. - The use case of restarting upload from scratch if the client dies can be implemented by having the client persist the upload secret. + Or it may happen because the client wants to upload a different share number than a previous client. + New shares will be created, existing shares will be unchanged, regardless of whether the upload secret matches or not. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0e1969593..9b158ecfd 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -203,18 +203,13 @@ class HTTPServer(object): upload_secret = authorization[Secrets.UPLOAD] info = loads(request.content.read()) - if storage_index in self._uploads: - for share_number in info["share-numbers"]: - in_progress = self._uploads[storage_index] - # For pre-existing upload, make sure password matches. - if ( - share_number in in_progress.upload_secrets - and not timing_safe_compare( - in_progress.upload_secrets[share_number], upload_secret - ) - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" + # We do NOT validate the upload secret for existing bucket uploads. + # Another upload may be happening in parallel, with a different upload + # key. That's fine! If a client tries to _write_ to that upload, they + # need to have an upload key. That does mean we leak the existence of + # these parallel uploads, but if you know storage index you can + # download them once upload finishes, so it's not a big deal to leak + # that information. already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( storage_index, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c2e26b4b6..2c7e13890 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1102,18 +1102,15 @@ class _HTTPBucketReader(object): class _HTTPStorageServer(object): """ Talk to remote storage server over HTTP. - - The same upload key is used for all communication. """ _http_client = attr.ib(type=StorageClient) - _upload_secret = attr.ib(type=bytes) @staticmethod def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(http_client=http_client, upload_secret=urandom(20)) + return _HTTPStorageServer(http_client=http_client) def get_version(self): return StorageClientGeneral(self._http_client).get_version() @@ -1128,9 +1125,10 @@ class _HTTPStorageServer(object): allocated_size, canary, ): + upload_secret = urandom(20) immutable_client = StorageClientImmutables(self._http_client) result = immutable_client.create( - storage_index, sharenums, allocated_size, self._upload_secret, renew_secret, + storage_index, sharenums, allocated_size, upload_secret, renew_secret, cancel_secret ) result = yield result @@ -1140,7 +1138,7 @@ class _HTTPStorageServer(object): client=immutable_client, storage_index=storage_index, share_number=share_num, - upload_secret=self._upload_secret + upload_secret=upload_secret )) for share_num in result.allocated }) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 14f4437b6..ae003c65f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -474,41 +474,77 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) - def test_allocate_buckets_second_time_wrong_upload_key(self): - """ - If allocate buckets endpoint is called second time with wrong upload - key on the same shares, the result is an error. - """ - # Create a upload: - (upload_secret, lease_secret, storage_index, _) = self.create_upload( - {1, 2, 3}, 100 - ) - with self.assertRaises(ClientException) as e: - result_of( - self.imm_client.create( - storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret - ) - ) - self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) - def test_allocate_buckets_second_time_different_shares(self): """ If allocate buckets endpoint is called second time with different - upload key on different shares, that creates the buckets. + upload key on potentially different shares, that creates the buckets on + those shares that are different. """ # Create a upload: (upload_secret, lease_secret, storage_index, created) = self.create_upload( {1, 2, 3}, 100 ) - # Add same shares: + # Write half of share 1 + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"a" * 50, + ) + ) + + # Add same shares with a different upload key share 1 overlaps with + # existing shares, this call shouldn't overwrite the existing + # work-in-progress. + upload_secret2 = b"x" * 2 created2 = result_of( self.imm_client.create( - storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret + storage_index, + {1, 4, 6}, + 100, + upload_secret2, + lease_secret, + lease_secret, ) ) self.assertEqual(created2.allocated, {4, 6}) + # Write second half of share 1 + self.assertTrue( + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 50, + b"b" * 50, + ) + ).finished + ) + + # The upload of share 1 succeeded, demonstrating that second create() + # call didn't overwrite work-in-progress. + downloaded = result_of( + self.imm_client.read_share_chunk(storage_index, 1, 0, 100) + ) + self.assertEqual(downloaded, b"a" * 50 + b"b" * 50) + + # We can successfully upload the shares created with the second upload secret. + self.assertTrue( + result_of( + self.imm_client.write_share_chunk( + storage_index, + 4, + upload_secret2, + 0, + b"x" * 100, + ) + ).finished + ) + def test_list_shares(self): """ Once a share is finished uploading, it's possible to list it. From 1d007cc573c6603323deaa6bf4a012ae8ed0fe33 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 13:21:36 -0500 Subject: [PATCH 514/916] News file. --- newsfragments/3877.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3877.minor diff --git a/newsfragments/3877.minor b/newsfragments/3877.minor new file mode 100644 index 000000000..e69de29bb From 52038739950a0c7a7c82dfcdd5e7d2c083d15e6a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Mar 2022 10:13:37 -0500 Subject: [PATCH 515/916] Refactor to unify data structure logic. --- src/allmydata/storage/http_server.py | 114 +++++++++++++++++++++------ 1 file changed, 91 insertions(+), 23 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0e1969593..d0653a97c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -138,9 +138,69 @@ class StorageIndexUploads(object): # different upload secrets). upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] - def add_upload(self, share_number, upload_secret, bucket): - self.shares[share_number] = bucket - self.upload_secrets[share_number] = upload_secret + +@attr.s +class UploadsInProgress(object): + """ + Keep track of uploads for storage indexes. + """ + + # Map storage index to corresponding uploads-in-progress + _uploads = attr.ib(type=Dict[bytes, StorageIndexUploads], factory=dict) + + def add_write_bucket( + self, + storage_index: bytes, + share_number: int, + upload_secret: bytes, + bucket: BucketWriter, + ): + """Add a new ``BucketWriter`` to be tracked. + + TODO 3877 how does a timed-out BucketWriter get removed?! + """ + si_uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) + si_uploads.shares[share_number] = bucket + si_uploads.upload_secrets[share_number] = upload_secret + + def get_write_bucket( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ) -> BucketWriter: + """Get the given in-progress immutable share upload.""" + try: + # TODO 3877 check the upload secret matches given one + return self._uploads[storage_index].shares[share_number] + except (KeyError, IndexError): + raise _HTTPError(http.NOT_FOUND) + + def remove_write_bucket(self, storage_index: bytes, share_number: int): + """Stop tracking the given ``BucketWriter``.""" + uploads_index = self._uploads[storage_index] + uploads_index.shares.pop(share_number) + uploads_index.upload_secrets.pop(share_number) + if not uploads_index.shares: + self._uploads.pop(storage_index) + + def validate_upload_secret( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ): + """ + Raise an unauthorized-HTTP-response exception if the given + storage_index+share_number have a different upload secret than the + given one. + + If the given upload doesn't exist at all, nothing happens. + """ + if storage_index in self._uploads: + try: + in_progress = self._uploads[storage_index] + except KeyError: + return + # For pre-existing upload, make sure password matches. + if share_number in in_progress.upload_secrets and not timing_safe_compare( + in_progress.upload_secrets[share_number], upload_secret + ): + raise _HTTPError(http.UNAUTHORIZED) class StorageIndexConverter(BaseConverter): @@ -155,6 +215,15 @@ class StorageIndexConverter(BaseConverter): raise ValidationError("Invalid storage index") +class _HTTPError(Exception): + """ + Raise from ``HTTPServer`` endpoint to return the given HTTP response code. + """ + + def __init__(self, code: int): + self.code = code + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -163,13 +232,19 @@ class HTTPServer(object): _app = Klein() _app.url_map.converters["storage_index"] = StorageIndexConverter + @_app.handle_errors(_HTTPError) + def _http_error(self, request, failure): + """Handle ``_HTTPError`` exceptions.""" + request.setResponseCode(failure.value.code) + return b"" + def __init__( self, storage_server, swissnum ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum # Maps storage index to StorageIndexUploads: - self._uploads = {} # type: Dict[bytes,StorageIndexUploads] + self._uploads = UploadsInProgress() def get_resource(self): """Return twisted.web ``Resource`` for this object.""" @@ -203,18 +278,10 @@ class HTTPServer(object): upload_secret = authorization[Secrets.UPLOAD] info = loads(request.content.read()) - if storage_index in self._uploads: - for share_number in info["share-numbers"]: - in_progress = self._uploads[storage_index] - # For pre-existing upload, make sure password matches. - if ( - share_number in in_progress.upload_secrets - and not timing_safe_compare( - in_progress.upload_secrets[share_number], upload_secret - ) - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" + for share_number in info["share-numbers"]: + self._uploads.validate_upload_secret( + storage_index, share_number, upload_secret + ) already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( storage_index, @@ -223,9 +290,10 @@ class HTTPServer(object): sharenums=info["share-numbers"], allocated_size=info["allocated-size"], ) - uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) for share_number, bucket in sharenum_to_bucket.items(): - uploads.add_upload(share_number, upload_secret, bucket) + self._uploads.add_write_bucket( + storage_index, share_number, upload_secret, bucket + ) return self._cbor( request, @@ -250,14 +318,14 @@ class HTTPServer(object): offset = content_range.start + # TODO 3877 test for checking upload secret + # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = request.content.read(content_range.stop - content_range.start + 1) - try: - bucket = self._uploads[storage_index].shares[share_number] - except (KeyError, IndexError): - request.setResponseCode(http.NOT_FOUND) - return b"" + bucket = self._uploads.get_write_bucket( + storage_index, share_number, authorization[Secrets.UPLOAD] + ) try: finished = bucket.write(offset, data) From c642218173a969397edda4dc4f1cb45f6b8a7e9f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Mar 2022 10:41:56 -0500 Subject: [PATCH 516/916] Sketch of aborting uploads. --- src/allmydata/storage/http_client.py | 23 ++++++++++++++++++- src/allmydata/storage/http_server.py | 27 +++++++++++++++++++++++ src/allmydata/storage_client.py | 3 ++- src/allmydata/test/test_istorageserver.py | 6 ++--- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 475aa2330..d83ecbdff 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -161,7 +161,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client): # type: (StorageClient) -> None + def __init__(self, client: StorageClient): self._client = client @inlineCallbacks @@ -208,6 +208,27 @@ class StorageClientImmutables(object): ) ) + @inlineCallbacks + def abort_upload( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ) -> Deferred[None]: + """Abort the upload.""" + url = self._client.relative_url( + "/v1/immutable/{}/{}/abort".format(_encode_si(storage_index), share_number) + ) + response = yield self._client.request( + "PUT", + url, + upload_secret=upload_secret, + ) + + if response.code == http.OK: + return + else: + raise ClientException( + response.code, + ) + @inlineCallbacks def write_share_chunk( self, storage_index, share_number, upload_secret, offset, data diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d0653a97c..1acf08a81 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -303,6 +303,33 @@ class HTTPServer(object): }, ) + @_authorized_route( + _app, + {Secrets.UPLOAD}, + "/v1/immutable///abort", + methods=["PUT"], + ) + def abort_share_upload(self, request, authorization, storage_index, share_number): + """Abort an in-progress immutable share upload.""" + try: + bucket = self._uploads.get_write_bucket( + storage_index, share_number, authorization[Secrets.UPLOAD] + ) + except _HTTPError: + # TODO 3877 If 404, check if this was already uploaded, in which case return 405 + # TODO 3877 write tests for 404 cases? + raise + + # TODO 3877 test for checking upload secret + + # Abort the upload: + bucket.abort() + # Stop tracking the bucket, so we can create a new one later if a + # client requests it: + self._uploads.remove_write_bucket(storage_index, share_number) + + return b"" + @_authorized_route( _app, {Secrets.UPLOAD}, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c2e26b4b6..9e47ba3ff 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1059,7 +1059,8 @@ class _HTTPBucketWriter(object): finished = attr.ib(type=bool, default=False) def abort(self): - pass # TODO in later ticket + return self.client.abort_upload(self.storage_index, self.share_number, + self.upload_secret) @defer.inlineCallbacks def write(self, offset, data): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 95261ddb2..668eeecc5 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -176,8 +176,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): canary=Referenceable(), ) - # Bucket 1 is fully written in one go. - yield allocated[0].callRemote("write", 0, b"1" * 1024) + # Bucket 1 get some data written (but not all, or HTTP implicitly + # finishes the upload) + yield allocated[0].callRemote("write", 0, b"1" * 1023) # Disconnect or abort, depending on the test: yield abort_or_disconnect(allocated[0]) @@ -1156,7 +1157,6 @@ class HTTPImmutableAPIsTests( # These will start passing in future PRs as HTTP protocol is implemented. SKIP_TESTS = { - "test_abort", "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", From 92b952a5fe87202358cb133316db40b4a9c8429d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 10:22:00 -0500 Subject: [PATCH 517/916] Authenticate writes! --- src/allmydata/storage/http_server.py | 4 +--- src/allmydata/test/test_storage_http.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1acf08a81..91b492663 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -167,8 +167,8 @@ class UploadsInProgress(object): self, storage_index: bytes, share_number: int, upload_secret: bytes ) -> BucketWriter: """Get the given in-progress immutable share upload.""" + self.validate_upload_secret(storage_index, share_number, upload_secret) try: - # TODO 3877 check the upload secret matches given one return self._uploads[storage_index].shares[share_number] except (KeyError, IndexError): raise _HTTPError(http.NOT_FOUND) @@ -345,8 +345,6 @@ class HTTPServer(object): offset = content_range.start - # TODO 3877 test for checking upload secret - # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = request.content.read(content_range.stop - content_range.start + 1) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 14f4437b6..b3e1ab0ad 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -474,6 +474,21 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) + def test_write_with_wrong_upload_key(self): + """A write with the wrong upload key fails.""" + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + with self.assertRaises(ClientException) as e: + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret + b"X", + 0, + b"123", + ) + ) + self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) + def test_allocate_buckets_second_time_wrong_upload_key(self): """ If allocate buckets endpoint is called second time with wrong upload From f47741afb163fa4680c7914c349ae00d7f6e622f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 10:45:21 -0500 Subject: [PATCH 518/916] Correct behavior on timed out immutable uploads. --- src/allmydata/storage/http_server.py | 38 +++++++++----------- src/allmydata/test/test_storage_http.py | 47 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 91b492663..57e2b3e82 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,19 +2,7 @@ HTTP server for storage. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on -else: - from typing import Dict, List, Set +from typing import Dict, List, Set, Tuple from functools import wraps from base64 import b64decode @@ -148,6 +136,9 @@ class UploadsInProgress(object): # Map storage index to corresponding uploads-in-progress _uploads = attr.ib(type=Dict[bytes, StorageIndexUploads], factory=dict) + # Map BucketWriter to (storage index, share number) + _bucketwriters = attr.ib(type=Dict[BucketWriter, Tuple[bytes, int]], factory=dict) + def add_write_bucket( self, storage_index: bytes, @@ -155,13 +146,11 @@ class UploadsInProgress(object): upload_secret: bytes, bucket: BucketWriter, ): - """Add a new ``BucketWriter`` to be tracked. - - TODO 3877 how does a timed-out BucketWriter get removed?! - """ + """Add a new ``BucketWriter`` to be tracked.""" si_uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) si_uploads.shares[share_number] = bucket si_uploads.upload_secrets[share_number] = upload_secret + self._bucketwriters[bucket] = (storage_index, share_number) def get_write_bucket( self, storage_index: bytes, share_number: int, upload_secret: bytes @@ -173,8 +162,9 @@ class UploadsInProgress(object): except (KeyError, IndexError): raise _HTTPError(http.NOT_FOUND) - def remove_write_bucket(self, storage_index: bytes, share_number: int): + def remove_write_bucket(self, bucket: BucketWriter): """Stop tracking the given ``BucketWriter``.""" + storage_index, share_number = self._bucketwriters.pop(bucket) uploads_index = self._uploads[storage_index] uploads_index.shares.pop(share_number) uploads_index.upload_secrets.pop(share_number) @@ -246,6 +236,12 @@ class HTTPServer(object): # Maps storage index to StorageIndexUploads: self._uploads = UploadsInProgress() + # When an upload finishes successfully, gets aborted, or times out, + # make sure it gets removed from our tracking datastructure: + self._storage_server.register_bucket_writer_close_handler( + self._uploads.remove_write_bucket + ) + def get_resource(self): """Return twisted.web ``Resource`` for this object.""" return self._app.resource() @@ -322,11 +318,9 @@ class HTTPServer(object): # TODO 3877 test for checking upload secret - # Abort the upload: + # Abort the upload; this should close it which will eventually result + # in self._uploads.remove_write_bucket() being called. bucket.abort() - # Stop tracking the bucket, so we can create a new one later if a - # client requests it: - self._uploads.remove_write_bucket(storage_index, share_number) return b"" diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b3e1ab0ad..6788bc657 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -810,3 +810,50 @@ class ImmutableHTTPAPITests(SyncTestCase): check_range("bytes=0-10", "bytes 0-10/*") # Can't go beyond the end of the immutable! check_range("bytes=10-100", "bytes 10-25/*") + + def test_timed_out_upload_allows_reupload(self): + """ + If an in-progress upload times out, it is cancelled, allowing a new + upload to occur. + """ + # Start an upload: + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"123", + ) + ) + + # Now, time passes, the in-progress upload should disappear... + self.http.clock.advance(30 * 60 + 1) + + # Now we can create a new share with the same storage index without + # complaint: + upload_secret = urandom(32) + lease_secret = urandom(32) + created = result_of( + self.imm_client.create( + storage_index, + {1}, + 100, + upload_secret, + lease_secret, + lease_secret, + ) + ) + self.assertEqual(created.allocated, {1}) + + # And write to it, too: + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"ABC", + ) + ) From 4fc7ef75288b935ff1ada9418b4535a7f319aed5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 10:57:05 -0500 Subject: [PATCH 519/916] Basic HTTP test for aborts. --- src/allmydata/test/test_storage_http.py | 32 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6788bc657..117a7037a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -813,8 +813,30 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_timed_out_upload_allows_reupload(self): """ - If an in-progress upload times out, it is cancelled, allowing a new - upload to occur. + If an in-progress upload times out, it is cancelled altogether, + allowing a new upload to occur. + """ + self._test_abort_or_timed_out_upload_to_existing_storage_index( + lambda **kwargs: self.http.clock.advance(30 * 60 + 1) + ) + + def test_abort_upload_allows_reupload(self): + """ + If an in-progress upload is aborted, it is cancelled altogether, + allowing a new upload to occur. + """ + + def abort(storage_index, share_number, upload_secret): + return result_of( + self.imm_client.abort_upload(storage_index, share_number, upload_secret) + ) + + self._test_abort_or_timed_out_upload_to_existing_storage_index(abort) + + def _test_abort_or_timed_out_upload_to_existing_storage_index(self, cancel_upload): + """Start uploading to an existing storage index that then times out or aborts. + + Re-uploading should work. """ # Start an upload: (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) @@ -828,8 +850,10 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) - # Now, time passes, the in-progress upload should disappear... - self.http.clock.advance(30 * 60 + 1) + # Now, the upload is cancelled somehow: + cancel_upload( + storage_index=storage_index, upload_secret=upload_secret, share_number=1 + ) # Now we can create a new share with the same storage index without # complaint: From ef4f912a68912bfe92800eca760058468cf39095 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 11:11:39 -0500 Subject: [PATCH 520/916] Less error-prone testing assertion, and fix a testing bug. --- src/allmydata/test/test_storage_http.py | 46 +++++++++++++++---------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 117a7037a..46914dbf2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from base64 import b64encode +from contextlib import contextmanager from os import urandom from hypothesis import assume, given, strategies as st @@ -316,6 +317,20 @@ class StorageClientWithHeadersOverride(object): return self.storage_client.request(*args, headers=headers, **kwargs) +@contextmanager +def assert_fails_with_http_code(test_case: SyncTestCase, code: int): + """ + Context manager that asserts the code fails with the given HTTP response + code. + """ + with test_case.assertRaises(ClientException) as e: + try: + yield + finally: + pass + test_case.assertEqual(e.exception.code, code) + + class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API @@ -340,9 +355,8 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), ) ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of(client.get_version()) - self.assertEqual(e.exception.args[0], 401) def test_version(self): """ @@ -477,7 +491,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_write_with_wrong_upload_key(self): """A write with the wrong upload key fails.""" (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of( self.imm_client.write_share_chunk( storage_index, @@ -487,7 +501,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"123", ) ) - self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) def test_allocate_buckets_second_time_wrong_upload_key(self): """ @@ -498,13 +511,12 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, lease_secret, storage_index, _) = self.create_upload( {1, 2, 3}, 100 ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of( self.imm_client.create( storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret ) ) - self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) def test_allocate_buckets_second_time_different_shares(self): """ @@ -562,7 +574,9 @@ class ImmutableHTTPAPITests(SyncTestCase): self.http.client, {"content-range": bad_content_range_value} ) ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code( + self, http.REQUESTED_RANGE_NOT_SATISFIABLE + ): result_of( client.write_share_chunk( storage_index, @@ -572,7 +586,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"0123456789", ) ) - self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) check_invalid("not a valid content-range header at all") check_invalid("bytes -1-9/10") @@ -594,7 +607,7 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, _) = self.create_upload({1}, 10) def unknown_check(storage_index, share_number): - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.NOT_FOUND): result_of( self.imm_client.write_share_chunk( storage_index, @@ -604,7 +617,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"0123456789", ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) # Wrong share number: unknown_check(storage_index, 7) @@ -663,7 +675,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) # Conflicting write: - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.CONFLICT): result_of( self.imm_client.write_share_chunk( storage_index, @@ -673,7 +685,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"0123456789", ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) def upload(self, share_number, data_length=26): """ @@ -700,7 +711,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Reading from unknown storage index results in 404. """ - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.NOT_FOUND): result_of( self.imm_client.read_share_chunk( b"1" * 16, @@ -709,14 +720,13 @@ class ImmutableHTTPAPITests(SyncTestCase): 10, ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. """ storage_index, _ = self.upload(1) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.NOT_FOUND): result_of( self.imm_client.read_share_chunk( storage_index, @@ -725,7 +735,6 @@ class ImmutableHTTPAPITests(SyncTestCase): 10, ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_with_negative_offset_fails(self): """ @@ -741,7 +750,9 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code( + self, http.REQUESTED_RANGE_NOT_SATISFIABLE + ): result_of( client.read_share_chunk( storage_index, @@ -750,7 +761,6 @@ class ImmutableHTTPAPITests(SyncTestCase): 10, ) ) - self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) # Bad unit check_bad_range("molluscs=0-9") From 86769c19bf306da3cae583e6d30c39d34e603c33 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 11:19:23 -0500 Subject: [PATCH 521/916] Finish abort logic and tests. --- src/allmydata/storage/http_server.py | 16 ++++-- src/allmydata/test/test_storage_http.py | 68 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 57e2b3e82..7659513e1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -311,13 +311,19 @@ class HTTPServer(object): bucket = self._uploads.get_write_bucket( storage_index, share_number, authorization[Secrets.UPLOAD] ) - except _HTTPError: - # TODO 3877 If 404, check if this was already uploaded, in which case return 405 - # TODO 3877 write tests for 404 cases? + except _HTTPError as e: + if e.code == http.NOT_FOUND: + # It may be we've already uploaded this, in which case error + # should be method not allowed (405). + try: + self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + pass + else: + # Already uploaded, so we can't abort. + raise _HTTPError(http.NOT_ALLOWED) raise - # TODO 3877 test for checking upload secret - # Abort the upload; this should close it which will eventually result # in self._uploads.remove_write_bucket() being called. bucket.abort() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 46914dbf2..97a81e250 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -891,3 +891,71 @@ class ImmutableHTTPAPITests(SyncTestCase): b"ABC", ) ) + + def test_unknown_aborts(self): + """ + Aborting aborts with unknown storage index or share number will 404. + """ + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + + for si, num in [(storage_index, 3), (b"x" * 16, 1)]: + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of(self.imm_client.abort_upload(si, num, upload_secret)) + + def test_unauthorized_abort(self): + """ + An abort with the wrong key will return an unauthorized error, and will + not abort the upload. + """ + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + + # Failed to abort becaues wrong upload secret: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): + result_of( + self.imm_client.abort_upload(storage_index, 1, upload_secret + b"X") + ) + + # We can still write to it: + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"ABC", + ) + ) + + def test_too_late_abort(self): + """ + An abort of an already-fully-uploaded immutable will result in 405 + error and will not affect the immutable. + """ + uploaded_data = b"123" + (upload_secret, _, storage_index, _) = self.create_upload({0}, 3) + result_of( + self.imm_client.write_share_chunk( + storage_index, + 0, + upload_secret, + 0, + uploaded_data, + ) + ) + + # Can't abort, we finished upload: + with assert_fails_with_http_code(self, http.NOT_ALLOWED): + result_of(self.imm_client.abort_upload(storage_index, 0, upload_secret)) + + # Abort didn't prevent reading: + self.assertEqual( + uploaded_data, + result_of( + self.imm_client.read_share_chunk( + storage_index, + 0, + 0, + 3, + ) + ), + ) From edb9eda53b4daac49c8582c9a82513fd809b18cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:41:10 -0500 Subject: [PATCH 522/916] Clarify. --- src/allmydata/test/test_storage_http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2103c5b65..b2da8e9d7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -489,7 +489,10 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertEqual(downloaded, expected_data[offset : offset + length]) def test_write_with_wrong_upload_key(self): - """A write with the wrong upload key fails.""" + """ + A write with an upload key that is different than the original upload + key will fail. + """ (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of( From 5d51aac0d309a83f77f677c41b0da08ca27c991e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:41:40 -0500 Subject: [PATCH 523/916] Clarify. --- src/allmydata/test/test_storage_http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b2da8e9d7..e062864e2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -934,7 +934,8 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_unknown_aborts(self): """ - Aborting aborts with unknown storage index or share number will 404. + Aborting uploads with an unknown storage index or share number will + result 404 HTTP response code. """ (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) From e598fbbc85589be52168d954b47f3c6974be31e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:42:24 -0500 Subject: [PATCH 524/916] Get rid of redundant code. --- src/allmydata/storage/http_server.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 357426000..6a43dec8b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -182,10 +182,7 @@ class UploadsInProgress(object): If the given upload doesn't exist at all, nothing happens. """ if storage_index in self._uploads: - try: - in_progress = self._uploads[storage_index] - except KeyError: - return + in_progress = self._uploads[storage_index] # For pre-existing upload, make sure password matches. if share_number in in_progress.upload_secrets and not timing_safe_compare( in_progress.upload_secrets[share_number], upload_secret From ba604b8231d85e0ec16709cdbeced3e9014910a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:44:24 -0500 Subject: [PATCH 525/916] News file. --- newsfragments/3879.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3879.minor diff --git a/newsfragments/3879.minor b/newsfragments/3879.minor new file mode 100644 index 000000000..e69de29bb From 636ab017d47b7668d9326a064055a2cdfbb78b78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:47:14 -0500 Subject: [PATCH 526/916] Disconnection is purely a Foolscap concern. --- src/allmydata/test/test_istorageserver.py | 57 ++++++++++------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 668eeecc5..fea14df79 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -194,20 +194,6 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) yield allocated[0].callRemote("write", 0, b"2" * 1024) - def test_disconnection(self): - """ - If we disconnect in the middle of writing to a bucket, all data is - wiped, and it's even possible to write different data to the bucket. - - (In the real world one shouldn't do that, but writing different data is - a good way to test that the original data really was wiped.) - - HTTP protocol should skip this test, since disconnection is meaningless - concept; this is more about testing implicit contract the Foolscap - implementation depends on doesn't change as we refactor things. - """ - return self.abort_or_disconnect_half_way(lambda _: self.disconnect()) - @inlineCallbacks def test_written_shares_are_allocated(self): """ @@ -1062,13 +1048,6 @@ class _SharedMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) - @inlineCallbacks - def disconnect(self): - """ - Disconnect and then reconnect with a new ``IStorageServer``. - """ - raise NotImplementedError("implement in subclass") - class _FoolscapMixin(_SharedMixin): """Run tests on Foolscap version of ``IStorageServer``.""" @@ -1081,16 +1060,6 @@ class _FoolscapMixin(_SharedMixin): self.assertTrue(IStorageServer.providedBy(client)) return succeed(client) - @inlineCallbacks - def disconnect(self): - """ - Disconnect and then reconnect with a new ``IStorageServer``. - """ - current = self.storage_client - yield self.bounce_client(0) - self.storage_client = self._get_native_server().get_storage_server() - assert self.storage_client is not current - class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" @@ -1149,6 +1118,31 @@ class FoolscapImmutableAPIsTests( ): """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + def test_disconnection(self): + """ + If we disconnect in the middle of writing to a bucket, all data is + wiped, and it's even possible to write different data to the bucket. + + (In the real world one shouldn't do that, but writing different data is + a good way to test that the original data really was wiped.) + + HTTP protocol doesn't need this test, since disconnection is a + meaningless concept; this is more about testing the implicit contract + the Foolscap implementation depends on doesn't change as we refactor + things. + """ + return self.abort_or_disconnect_half_way(lambda _: self.disconnect()) + + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + current = self.storage_client + yield self.bounce_client(0) + self.storage_client = self._get_native_server().get_storage_server() + assert self.storage_client is not current + class HTTPImmutableAPIsTests( _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase @@ -1161,7 +1155,6 @@ class HTTPImmutableAPIsTests( "test_add_new_lease", "test_advise_corrupt_share", "test_bucket_advise_corrupt_share", - "test_disconnection", } From aee0f7dc69d850c4d971b25bbb3c1db2cb269b8e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 13:10:13 -0500 Subject: [PATCH 527/916] Sketch of lease renewal implementation. --- src/allmydata/storage/http_client.py | 33 ++++++++++++++++++----- src/allmydata/storage/http_server.py | 24 +++++++++++++++++ src/allmydata/storage_client.py | 11 ++++++++ src/allmydata/test/test_istorageserver.py | 2 -- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d83ecbdff..1610f5433 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -310,9 +310,7 @@ class StorageClientImmutables(object): body = yield response.content() returnValue(body) else: - raise ClientException( - response.code, - ) + raise ClientException(response.code) @inlineCallbacks def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] @@ -330,6 +328,29 @@ class StorageClientImmutables(object): body = yield _decode_cbor(response) returnValue(set(body)) else: - raise ClientException( - response.code, - ) + raise ClientException(response.code) + + @inlineCallbacks + def add_or_renew_lease( + self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes + ): + """ + Add or renew a lease. + + If the renewal secret matches an existing lease, it is renewed. + Otherwise a new lease is added. + """ + url = self._client.relative_url( + "/v1/lease/{}".format(_encode_si(storage_index)) + ) + response = yield self._client.request( + "PUT", + url, + lease_renew_secret=renew_secret, + lease_cancel_secret=cancel_secret, + ) + + if response.code == http.NO_CONTENT: + return + else: + raise ClientException(response.code) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6a43dec8b..37bb80d8f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -434,3 +434,27 @@ class HTTPServer(object): ContentRange("bytes", offset, offset + len(data)).to_header(), ) return data + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL}, + "/v1/lease/", + methods=["PUT"], + ) + def add_or_renew_lease(self, request, authorization, storage_index): + """Update the lease for an immutable share.""" + # TODO 3879 write direct test for success case + + # Checking of the renewal secret is done by the backend. + try: + self._storage_server.add_lease( + storage_index, + authorization[Secrets.LEASE_RENEW], + authorization[Secrets.LEASE_CANCEL], + ) + except IndexError: + # TODO 3879 write test for this case + raise + + request.setResponseCode(http.NO_CONTENT) + return b"" diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0d3159b55..ac74f6c67 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1160,3 +1160,14 @@ class _HTTPStorageServer(object): )) for share_num in share_numbers }) + + def add_lease( + self, + storage_index, + renew_secret, + cancel_secret, + ): + immutable_client = StorageClientImmutables(self._http_client) + return immutable_client.add_or_renew_lease( + storage_index, renew_secret, cancel_secret + ) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fea14df79..e85047081 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1151,8 +1151,6 @@ class HTTPImmutableAPIsTests( # These will start passing in future PRs as HTTP protocol is implemented. SKIP_TESTS = { - "test_add_lease_renewal", - "test_add_new_lease", "test_advise_corrupt_share", "test_bucket_advise_corrupt_share", } From f7366833475d1e6432de34b94fd42e546b9ad74b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 13:21:05 -0500 Subject: [PATCH 528/916] Finish testing and implementing lease renewal. --- src/allmydata/storage/http_server.py | 17 ++++---- src/allmydata/test/test_storage_http.py | 55 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 37bb80d8f..2deb03dfd 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -443,18 +443,15 @@ class HTTPServer(object): ) def add_or_renew_lease(self, request, authorization, storage_index): """Update the lease for an immutable share.""" - # TODO 3879 write direct test for success case + if not self._storage_server.get_buckets(storage_index): + raise _HTTPError(http.NOT_FOUND) # Checking of the renewal secret is done by the backend. - try: - self._storage_server.add_lease( - storage_index, - authorization[Secrets.LEASE_RENEW], - authorization[Secrets.LEASE_CANCEL], - ) - except IndexError: - # TODO 3879 write test for this case - raise + self._storage_server.add_lease( + storage_index, + authorization[Secrets.LEASE_RENEW], + authorization[Secrets.LEASE_CANCEL], + ) request.setResponseCode(http.NO_CONTENT) return b"" diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e062864e2..31505a00f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1000,3 +1000,58 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ), ) + + def test_lease_renew_and_add(self): + """ + It's possible the renew the lease on an uploaded immutable, by using + the same renewal secret, or add a new lease by choosing a different + renewal secret. + """ + # Create immutable: + (upload_secret, lease_secret, storage_index, _) = self.create_upload({0}, 100) + result_of( + self.imm_client.write_share_chunk( + storage_index, + 0, + upload_secret, + 0, + b"A" * 100, + ) + ) + + [lease] = self.http.storage_server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.http.clock.advance(167) + + # We renew the lease: + result_of( + self.imm_client.add_or_renew_lease( + storage_index, lease_secret, lease_secret + ) + ) + + # More time passes: + self.http.clock.advance(10) + + # We create a new lease: + lease_secret2 = urandom(32) + result_of( + self.imm_client.add_or_renew_lease( + storage_index, lease_secret2, lease_secret2 + ) + ) + + [lease1, lease2] = self.http.storage_server.get_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) + self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) + + def test_lease_on_unknown_storage_index(self): + """ + An attempt to renew an unknown storage index will result in a HTTP 404. + """ + storage_index = urandom(16) + secret = b"A" * 32 + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of(self.imm_client.add_or_renew_lease(storage_index, secret, secret)) From 922ee4feb117956a1a766c1f3644b669cb75d45f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Mar 2022 11:09:45 -0500 Subject: [PATCH 529/916] Sketch of advise_corrupt_share support for immutables. --- docs/proposed/http-storage-node-protocol.rst | 4 ++- src/allmydata/storage/http_client.py | 28 ++++++++++++++++++++ src/allmydata/storage/http_server.py | 19 +++++++++++++ src/allmydata/storage/server.py | 5 ++-- src/allmydata/storage_client.py | 28 ++++++++++++++++---- src/allmydata/test/test_istorageserver.py | 6 ----- 6 files changed, 76 insertions(+), 14 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 0f534f0c5..33a9c0b0e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -615,7 +615,7 @@ From RFC 7231:: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. -The request body includes an human-meaningful string with details about the corruption. +The request body includes an human-meaningful (Unicode) string with details about the corruption. It also includes potentially important details about the share. For example:: @@ -624,6 +624,8 @@ For example:: .. share-type, storage-index, and share-number are inferred from the URL +The response code is OK, or 404 not found if the share couldn't be found. + Reading ~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1610f5433..d0ae4b584 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -354,3 +354,31 @@ class StorageClientImmutables(object): return else: raise ClientException(response.code) + + @inlineCallbacks + def advise_corrupt_share( + self, + storage_index: bytes, + share_number: int, + reason: str, + ): + """Indicate a share has been corrupted, with a human-readable message.""" + assert isinstance(reason, str) + url = self._client.relative_url( + "/v1/immutable/{}/{}/corrupt".format( + _encode_si(storage_index), share_number + ) + ) + message = dumps({"reason": reason}) + response = yield self._client.request( + "POST", + url, + data=message, + headers=Headers({"content-type": ["application/cbor"]}), + ) + if response.code == http.OK: + return + else: + raise ClientException( + response.code, + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2deb03dfd..5d357b846 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -455,3 +455,22 @@ class HTTPServer(object): request.setResponseCode(http.NO_CONTENT) return b"" + + @_authorized_route( + _app, + set(), + "/v1/immutable///corrupt", + methods=["POST"], + ) + def advise_corrupt_share(self, request, authorization, storage_index, share_number): + """Indicate that given share is corrupt, with a text reason.""" + # TODO 3879 test success path + try: + bucket = self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + # TODO 3879 test this path + raise _HTTPError(http.NOT_FOUND) + + info = loads(request.content.read()) + bucket.advise_corrupt_share(info["reason"].encode("utf-8")) + return b"" diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0add9806b..7ef7b4d37 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -743,8 +743,9 @@ class StorageServer(service.MultiService): def advise_corrupt_share(self, share_type, storage_index, shnum, reason): - # This is a remote API, I believe, so this has to be bytes for legacy - # protocol backwards compatibility reasons. + # Previously this had to be bytes for legacy protocol backwards + # compatibility reasons. Now that Foolscap layer has been abstracted + # out, we can probably refactor this to be unicode... assert isinstance(share_type, bytes) assert isinstance(reason, bytes), "%r is not bytes" % (reason,) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ac74f6c67..55b6cfb05 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -77,7 +77,7 @@ from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, - ClientException as HTTPClientException, + ClientException as HTTPClientException ) @@ -1094,7 +1094,10 @@ class _HTTPBucketReader(object): ) def advise_corrupt_share(self, reason): - pass # TODO in later ticket + return self.client.advise_corrupt_share( + self.storage_index, self.share_number, + str(reason, "utf-8", errors="backslashreplace") + ) # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @@ -1124,7 +1127,7 @@ class _HTTPStorageServer(object): cancel_secret, sharenums, allocated_size, - canary, + canary ): upload_secret = urandom(20) immutable_client = StorageClientImmutables(self._http_client) @@ -1148,7 +1151,7 @@ class _HTTPStorageServer(object): @defer.inlineCallbacks def get_buckets( self, - storage_index, + storage_index ): immutable_client = StorageClientImmutables(self._http_client) share_numbers = yield immutable_client.list_shares( @@ -1165,9 +1168,24 @@ class _HTTPStorageServer(object): self, storage_index, renew_secret, - cancel_secret, + cancel_secret ): immutable_client = StorageClientImmutables(self._http_client) return immutable_client.add_or_renew_lease( storage_index, renew_secret, cancel_secret ) + + def advise_corrupt_share( + self, + share_type, + storage_index, + shnum, + reason: bytes + ): + if share_type == b"immutable": + imm_client = StorageClientImmutables(self._http_client) + return imm_client.advise_corrupt_share( + storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") + ) + else: + raise NotImplementedError() # future tickets diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index e85047081..253ff6046 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1149,12 +1149,6 @@ class HTTPImmutableAPIsTests( ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" - # These will start passing in future PRs as HTTP protocol is implemented. - SKIP_TESTS = { - "test_advise_corrupt_share", - "test_bucket_advise_corrupt_share", - } - class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase From 7e25b43dbaff26449fe5dbef0b2f2d32a3ada883 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Mar 2022 11:28:48 -0500 Subject: [PATCH 530/916] Direct unit tests for advising share is corrupt. --- src/allmydata/storage/http_server.py | 2 -- src/allmydata/test/test_storage_http.py | 32 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 5d357b846..d122b95b4 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -464,11 +464,9 @@ class HTTPServer(object): ) def advise_corrupt_share(self, request, authorization, storage_index, share_number): """Indicate that given share is corrupt, with a text reason.""" - # TODO 3879 test success path try: bucket = self._storage_server.get_buckets(storage_index)[share_number] except KeyError: - # TODO 3879 test this path raise _HTTPError(http.NOT_FOUND) info = loads(request.content.read()) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 31505a00f..70a9f1c16 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1055,3 +1055,35 @@ class ImmutableHTTPAPITests(SyncTestCase): secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): result_of(self.imm_client.add_or_renew_lease(storage_index, secret, secret)) + + def test_advise_corrupt_share(self): + """ + Advising share was corrupted succeeds from HTTP client's perspective, + and calls appropriate method on server. + """ + corrupted = [] + self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append( + args + ) + + storage_index, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) + + self.assertEqual( + corrupted, [(b"immutable", storage_index, 13, reason.encode("utf-8"))] + ) + + def test_advise_corrupt_share_unknown(self): + """ + Advising an unknown share was corrupted results in 404. + """ + storage_index, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) + + for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of( + self.imm_client.advise_corrupt_share(si, share_number, reason) + ) From 5baf63219da4b73270b7f304e0f48e36045d05d8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Mar 2022 17:40:35 -0500 Subject: [PATCH 531/916] Always use UTF-8 for corruption reports. --- newsfragments/3879.minor | 1 + src/allmydata/storage/server.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/newsfragments/3879.minor b/newsfragments/3879.minor index e69de29bb..ca3f24f94 100644 --- a/newsfragments/3879.minor +++ b/newsfragments/3879.minor @@ -0,0 +1 @@ +Share corruption reports stored on disk are now always encoded in UTF-8. \ No newline at end of file diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 7ef7b4d37..9d1a3d6a4 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -778,7 +778,7 @@ class StorageServer(service.MultiService): si_s, shnum, ) - with open(report_path, "w") as f: + with open(report_path, "w", encoding="utf-8") as f: f.write(report) return None From e4b4dc418a0ca2bb27a01f7c748842744262e72a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:15:43 -0400 Subject: [PATCH 532/916] Address review comments. --- docs/proposed/http-storage-node-protocol.rst | 11 ++++++----- newsfragments/{3879.minor => 3879.incompat} | 0 2 files changed, 6 insertions(+), 5 deletions(-) rename newsfragments/{3879.minor => 3879.incompat} (100%) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 33a9c0b0e..2ceb3c03a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -614,17 +614,18 @@ From RFC 7231:: ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Advise the server the data read from the indicated share was corrupt. -The request body includes an human-meaningful (Unicode) string with details about the corruption. -It also includes potentially important details about the share. +Advise the server the data read from the indicated share was corrupt. The +request body includes an human-meaningful text string with details about the +corruption. It also includes potentially important details about the share. For example:: - {"reason": "expected hash abcd, got hash efgh"} + {"reason": u"expected hash abcd, got hash efgh"} .. share-type, storage-index, and share-number are inferred from the URL -The response code is OK, or 404 not found if the share couldn't be found. +The response code is OK (200) by default, or NOT FOUND (404) if the share +couldn't be found. Reading ~~~~~~~ diff --git a/newsfragments/3879.minor b/newsfragments/3879.incompat similarity index 100% rename from newsfragments/3879.minor rename to newsfragments/3879.incompat From f815083b4dfbadf0aab35ca32960b27c0b70c058 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:20:28 -0400 Subject: [PATCH 533/916] News file. --- newsfragments/3881.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3881.minor diff --git a/newsfragments/3881.minor b/newsfragments/3881.minor new file mode 100644 index 000000000..e69de29bb From e55c3e8acf7708f1f548a12d393b9914558fbdbb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:35:39 -0400 Subject: [PATCH 534/916] Check for CBOR content-encoding header in client. --- src/allmydata/storage/http_client.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d0ae4b584..7458f9271 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -58,8 +58,14 @@ class ClientException(Exception): def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: - return treq.content(response).addCallback(loads) - return fail(ClientException(response.code, response.phrase)) + if response.headers.getRawHeaders("content-type") == ["application/cbor"]: + # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + return treq.content(response).addCallback(loads) + else: + raise ClientException(-1, "Server didn't send CBOR") + else: + return fail(ClientException(response.code, response.phrase)) @attr.s From b8ab3dd6a7c7c9bdc90894fd7b0112318464d32c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:53:22 -0400 Subject: [PATCH 535/916] Server handles Accept headers. --- src/allmydata/storage/http_server.py | 35 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 11 ++++++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d122b95b4..79eb35e56 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,7 +11,11 @@ import binascii from klein import Klein from twisted.web import http import attr -from werkzeug.http import parse_range_header, parse_content_range_header +from werkzeug.http import ( + parse_range_header, + parse_content_range_header, + parse_accept_header, +) from werkzeug.routing import BaseConverter, ValidationError from werkzeug.datastructures import ContentRange @@ -243,20 +247,27 @@ class HTTPServer(object): """Return twisted.web ``Resource`` for this object.""" return self._app.resource() - def _cbor(self, request, data): - """Return CBOR-encoded data.""" - # TODO Might want to optionally send JSON someday, based on Accept - # headers, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 - request.setHeader("Content-Type", "application/cbor") - # TODO if data is big, maybe want to use a temporary file eventually... - return dumps(data) + def _send_encoded(self, request, data): + """Return encoded data, by default using CBOR.""" + cbor_mime = "application/cbor" + accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] + accept = parse_accept_header(accept_headers[0]) + if accept.best == cbor_mime: + request.setHeader("Content-Type", cbor_mime) + # TODO if data is big, maybe want to use a temporary file eventually... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + return dumps(data) + else: + # TODO Might want to optionally send JSON someday: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 + raise _HTTPError(http.NOT_ACCEPTABLE) ##### Generic APIs ##### @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) def version(self, request, authorization): """Return version information.""" - return self._cbor(request, self._storage_server.get_version()) + return self._send_encoded(request, self._storage_server.get_version()) ##### Immutable APIs ##### @@ -291,7 +302,7 @@ class HTTPServer(object): storage_index, share_number, upload_secret, bucket ) - return self._cbor( + return self._send_encoded( request, { "already-have": set(already_got), @@ -367,7 +378,7 @@ class HTTPServer(object): required = [] for start, end, _ in bucket.required_ranges().ranges(): required.append({"begin": start, "end": end}) - return self._cbor(request, {"required": required}) + return self._send_encoded(request, {"required": required}) @_authorized_route( _app, @@ -380,7 +391,7 @@ class HTTPServer(object): List shares for the given storage index. """ share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) - return self._cbor(request, share_numbers) + return self._send_encoded(request, share_numbers) @_authorized_route( _app, diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 70a9f1c16..c20012a9b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -358,6 +358,17 @@ class GenericHTTPAPITests(SyncTestCase): with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of(client.get_version()) + def test_unsupported_mime_type(self): + """ + The client can request mime types other than CBOR, and if they are + unsupported a NOT ACCEPTABLE (406) error will be returned. + """ + client = StorageClientGeneral( + StorageClientWithHeadersOverride(self.http.client, {"accept": "image/gif"}) + ) + with assert_fails_with_http_code(self, http.NOT_ACCEPTABLE): + result_of(client.get_version()) + def test_version(self): """ The client can return the version. From 1e108f8445aad556a79c7dbe817de2d51624b112 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:01:09 -0400 Subject: [PATCH 536/916] Don't use a custom parser. --- src/allmydata/storage/http_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7458f9271..5733c1514 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -32,6 +32,7 @@ import attr from cbor2 import loads, dumps from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange +from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred @@ -58,7 +59,10 @@ class ClientException(Exception): def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: - if response.headers.getRawHeaders("content-type") == ["application/cbor"]: + content_type = parse_options_header( + (response.headers.getRawHeaders("content-type") or [None])[0] + )[0] + if content_type == "application/cbor": # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return treq.content(response).addCallback(loads) From 13fd3b3685ab2255931353387bd07139bd1165cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:01:20 -0400 Subject: [PATCH 537/916] Get rid of Python 2 crud. --- src/allmydata/storage/http_client.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5733c1514..2790f1f7e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,27 +2,8 @@ HTTP client that talks to the HTTP storage server. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from collections import defaultdict - - Optional = Set = defaultdict( - lambda: None - ) # some garbage to just make this module import -else: - # typing module not available in Python 2, and we only do type checking in - # Python 3 anyway. - from typing import Union, Set, Optional - from treq.testing import StubTreq +from typing import Union, Set, Optional +from treq.testing import StubTreq from base64 import b64encode From fef332754b28672659cd42ce1b5a47d677b3ef41 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:09:40 -0400 Subject: [PATCH 538/916] Switch to shared utility so server can use it too. --- src/allmydata/storage/http_client.py | 7 ++----- src/allmydata/storage/http_common.py | 21 +++++++++++++++------ src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2790f1f7e..ace97508c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -13,14 +13,13 @@ import attr from cbor2 import loads, dumps from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange -from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq -from .http_common import swissnum_auth_header, Secrets +from .http_common import swissnum_auth_header, Secrets, get_content_type from .common import si_b2a @@ -40,9 +39,7 @@ class ClientException(Exception): def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: - content_type = parse_options_header( - (response.headers.getRawHeaders("content-type") or [None])[0] - )[0] + content_type = get_content_type(response.headers) if content_type == "application/cbor": # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index af4224bd0..fdf637180 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -1,15 +1,24 @@ """ Common HTTP infrastructure for the storge server. """ -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on from enum import Enum from base64 import b64encode +from typing import Optional + +from werkzeug.http import parse_options_header +from twisted.web.http_headers import Headers + + +def get_content_type(headers: Headers) -> Optional[str]: + """ + Get the content type from the HTTP ``Content-Type`` header. + + Returns ``None`` if no content-type was set. + """ + values = headers.getRawHeaders("content-type") or [None] + content_type = parse_options_header(values[0])[0] or None + return content_type def swissnum_auth_header(swissnum): # type: (bytes) -> bytes diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index c20012a9b..af90d58a9 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -49,9 +49,29 @@ from ..storage.http_client import ( StorageClientGeneral, _encode_si, ) +from ..storage.http_common import get_content_type from ..storage.common import si_b2a +class HTTPUtilities(SyncTestCase): + """Tests for HTTP common utilities.""" + + def test_get_content_type(self): + """``get_content_type()`` extracts the content-type from the header.""" + + def assert_header_values_result(values, expected_content_type): + headers = Headers() + if values: + headers.setRawHeaders("Content-Type", values) + content_type = get_content_type(headers) + self.assertEqual(content_type, expected_content_type) + + assert_header_values_result(["text/html"], "text/html") + assert_header_values_result([], None) + assert_header_values_result(["text/plain", "application/json"], "text/plain") + assert_header_values_result(["text/html;encoding=utf-8"], "text/html") + + def _post_process(params): secret_types, secrets = params secrets = {t: s for (t, s) in zip(secret_types, secrets)} From b6073b11c2706a6941754bc5ab36ebe5b7960afa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:16:09 -0400 Subject: [PATCH 539/916] Refactor to check HTTP content-type of request body. --- src/allmydata/storage/http_server.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 79eb35e56..673800321 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,7 +2,7 @@ HTTP server for storage. """ -from typing import Dict, List, Set, Tuple +from typing import Dict, List, Set, Tuple, Any from functools import wraps from base64 import b64decode @@ -23,7 +23,7 @@ from werkzeug.datastructures import ContentRange from cbor2 import dumps, loads from .server import StorageServer -from .http_common import swissnum_auth_header, Secrets +from .http_common import swissnum_auth_header, Secrets, get_content_type from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare @@ -248,7 +248,9 @@ class HTTPServer(object): return self._app.resource() def _send_encoded(self, request, data): - """Return encoded data, by default using CBOR.""" + """ + Return encoded data as the HTTP body response, by default using CBOR. + """ cbor_mime = "application/cbor" accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] accept = parse_accept_header(accept_headers[0]) @@ -262,6 +264,18 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) + def _read_encoded(self, request) -> Any: + """ + Read encoded request body data, decoding it with CBOR by default. + """ + content_type = get_content_type(request.requestHeaders) + if content_type == "application/cbor": + # TODO limit memory usage, client could send arbitrarily large data... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + return loads(request.content.read()) + else: + raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) + ##### Generic APIs ##### @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) @@ -280,7 +294,7 @@ class HTTPServer(object): def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] - info = loads(request.content.read()) + info = self._read_encoded(request) # We do NOT validate the upload secret for existing bucket uploads. # Another upload may be happening in parallel, with a different upload @@ -480,6 +494,6 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - info = loads(request.content.read()) + info = self._read_encoded(request) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" From 722f8e9598a60c870396e2a00164a835e4e485db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:17:06 -0400 Subject: [PATCH 540/916] Expand docs. --- src/allmydata/storage/http_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 673800321..0bd9f5dfc 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -249,7 +249,10 @@ class HTTPServer(object): def _send_encoded(self, request, data): """ - Return encoded data as the HTTP body response, by default using CBOR. + Return encoded data suitable for writing as the HTTP body response, by + default using CBOR. + + Also sets the appropriate ``Content-Type`` header on the response. """ cbor_mime = "application/cbor" accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] From 106cc708a0a1d989c168ef4ad105be5b6a9b954e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:18:53 -0400 Subject: [PATCH 541/916] Use a constant. --- src/allmydata/storage/http_client.py | 8 ++++---- src/allmydata/storage/http_common.py | 2 ++ src/allmydata/storage/http_server.py | 13 +++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ace97508c..f38459cee 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -19,7 +19,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq -from .http_common import swissnum_auth_header, Secrets, get_content_type +from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE from .common import si_b2a @@ -40,7 +40,7 @@ def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) - if content_type == "application/cbor": + if content_type == CBOR_MIME_TYPE: # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return treq.content(response).addCallback(loads) @@ -186,7 +186,7 @@ class StorageClientImmutables(object): lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, data=message, - headers=Headers({"content-type": ["application/cbor"]}), + headers=Headers({"content-type": [CBOR_MIME_TYPE]}), ) decoded_response = yield _decode_cbor(response) returnValue( @@ -362,7 +362,7 @@ class StorageClientImmutables(object): "POST", url, data=message, - headers=Headers({"content-type": ["application/cbor"]}), + headers=Headers({"content-type": [CBOR_MIME_TYPE]}), ) if response.code == http.OK: return diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index fdf637180..8313846c9 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -9,6 +9,8 @@ from typing import Optional from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers +CBOR_MIME_TYPE = "application/cbor" + def get_content_type(headers: Headers) -> Optional[str]: """ diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0bd9f5dfc..f648d8331 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,7 +23,7 @@ from werkzeug.datastructures import ContentRange from cbor2 import dumps, loads from .server import StorageServer -from .http_common import swissnum_auth_header, Secrets, get_content_type +from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare @@ -254,11 +254,12 @@ class HTTPServer(object): Also sets the appropriate ``Content-Type`` header on the response. """ - cbor_mime = "application/cbor" - accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] + accept_headers = request.requestHeaders.getRawHeaders("accept") or [ + CBOR_MIME_TYPE + ] accept = parse_accept_header(accept_headers[0]) - if accept.best == cbor_mime: - request.setHeader("Content-Type", cbor_mime) + if accept.best == CBOR_MIME_TYPE: + request.setHeader("Content-Type", CBOR_MIME_TYPE) # TODO if data is big, maybe want to use a temporary file eventually... # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return dumps(data) @@ -272,7 +273,7 @@ class HTTPServer(object): Read encoded request body data, decoding it with CBOR by default. """ content_type = get_content_type(request.requestHeaders) - if content_type == "application/cbor": + if content_type == CBOR_MIME_TYPE: # TODO limit memory usage, client could send arbitrarily large data... # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return loads(request.content.read()) From 0aa8089d81e29284b865ab5da510b8e1d173de7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:20:23 -0400 Subject: [PATCH 542/916] Explicitly tell the server that the client accepts CBOR. --- src/allmydata/storage/http_client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f38459cee..6b32c13f7 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -99,6 +99,8 @@ class StorageClient(object): into corresponding HTTP headers. """ headers = self._get_headers(headers) + + # Add secrets: for secret, value in [ (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), @@ -110,6 +112,10 @@ class StorageClient(object): "X-Tahoe-Authorization", b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), ) + + # Note we can accept CBOR: + headers.addRawHeader("Accept", CBOR_MIME_TYPE) + return self._treq.request(method, url, headers=headers, **kwargs) From fae9556e3dec59bd4db1bf0ecb7fb9ddd1ca5e71 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:28:54 -0400 Subject: [PATCH 543/916] Centralize client serialization logic too. --- src/allmydata/storage/http_client.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6b32c13f7..41a2dd0b8 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -92,11 +92,15 @@ class StorageClient(object): lease_cancel_secret=None, upload_secret=None, headers=None, + message_to_serialize=None, **kwargs ): """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. + + If ``message_to_serialize`` is set, it will be serialized (by default + with CBOR) and set as the request body. """ headers = self._get_headers(headers) @@ -116,6 +120,13 @@ class StorageClient(object): # Note we can accept CBOR: headers.addRawHeader("Accept", CBOR_MIME_TYPE) + # If there's a request message, serialize it and set the Content-Type + # header: + if message_to_serialize is not None: + assert "data" not in kwargs + kwargs["data"] = dumps(message_to_serialize) + headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) + return self._treq.request(method, url, headers=headers, **kwargs) @@ -182,17 +193,15 @@ class StorageClientImmutables(object): storage index failed the result will fire with an exception. """ url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index)) - message = dumps( - {"share-numbers": share_numbers, "allocated-size": allocated_size} - ) + message = {"share-numbers": share_numbers, "allocated-size": allocated_size} + response = yield self._client.request( "POST", url, lease_renew_secret=lease_renew_secret, lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, - data=message, - headers=Headers({"content-type": [CBOR_MIME_TYPE]}), + message_to_serialize=message, ) decoded_response = yield _decode_cbor(response) returnValue( @@ -363,13 +372,8 @@ class StorageClientImmutables(object): _encode_si(storage_index), share_number ) ) - message = dumps({"reason": reason}) - response = yield self._client.request( - "POST", - url, - data=message, - headers=Headers({"content-type": [CBOR_MIME_TYPE]}), - ) + message = {"reason": reason} + response = yield self._client.request("POST", url, message_to_serialize=message) if response.code == http.OK: return else: From 7de3d93b0eba77e4df5e4947f46727391ba0d6ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 10:12:51 -0400 Subject: [PATCH 544/916] Switch to TypeError. --- src/allmydata/storage/http_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 41a2dd0b8..99275ae24 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -123,7 +123,11 @@ class StorageClient(object): # If there's a request message, serialize it and set the Content-Type # header: if message_to_serialize is not None: - assert "data" not in kwargs + if "data" in kwargs: + raise TypeError( + "Can't use both `message_to_serialize` and `data` " + "as keyword arguments at the same time" + ) kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) From b83b6adfbc46d104151450d76f4b48e428161685 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:41:07 -0400 Subject: [PATCH 545/916] remove py2 compat boilerplate --- src/allmydata/stats.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 13ed8817c..b893be6ff 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -1,17 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from time import clock as process_time -else: - from time import process_time +from time import process_time import time from twisted.application import service From 32e88e580b42f698f5540a44aacd2698093c9bca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:41:33 -0400 Subject: [PATCH 546/916] switch to a deque this makes more of the data structure someone else's responsibility and probably improves performance too (but I didn't measure) --- src/allmydata/stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index b893be6ff..a123ff1d8 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -1,6 +1,8 @@ """ Ported to Python 3. """ + +from collections import deque from time import process_time import time @@ -26,7 +28,7 @@ class CPUUsageMonitor(service.MultiService): # up. self.initial_cpu = 0.0 # just in case eventually(self._set_initial_cpu) - self.samples = [] + self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) @@ -37,8 +39,6 @@ class CPUUsageMonitor(service.MultiService): now_wall = time.time() now_cpu = process_time() self.samples.append( (now_wall, now_cpu) ) - while len(self.samples) > self.HISTORY_LENGTH+1: - self.samples.pop(0) def _average_N_minutes(self, size): if len(self.samples) < size+1: From ce381f3e39db35249bca42f97cff7e6fa74a1aed Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:42:40 -0400 Subject: [PATCH 547/916] pull the default initial_cpu value out to class scope also add python 3 syntax type annotations --- src/allmydata/stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index a123ff1d8..60a24ec02 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -16,8 +16,9 @@ from allmydata.interfaces import IStatsProducer @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): - HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 # type: float + HISTORY_LENGTH: int = 15 + POLL_INTERVAL: float = 60 + initial_cpu: float = 0.0 def __init__(self): service.MultiService.__init__(self) @@ -26,7 +27,6 @@ class CPUUsageMonitor(service.MultiService): # rest of the program will be run by the child process, after twistd # forks. Instead, set self.initial_cpu as soon as the reactor starts # up. - self.initial_cpu = 0.0 # just in case eventually(self._set_initial_cpu) self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) # we provide 1min, 5min, and 15min moving averages From b56f79c7ad52d85cccf0a83abed406d890a5fd83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:43:17 -0400 Subject: [PATCH 548/916] read initial cpu usage value at service start time This removes the Foolscap dependency. --- src/allmydata/stats.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 60a24ec02..a2d208560 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -9,7 +9,6 @@ import time from twisted.application import service from twisted.application.internet import TimerService from zope.interface import implementer -from foolscap.api import eventually from allmydata.util import log, dictutil from allmydata.interfaces import IStatsProducer @@ -22,18 +21,13 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - # we don't use process_time() here, because the constructor is run by - # the twistd parent process (as it loads the .tac file), whereas the - # rest of the program will be run by the child process, after twistd - # forks. Instead, set self.initial_cpu as soon as the reactor starts - # up. - eventually(self._set_initial_cpu) self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) - def _set_initial_cpu(self): + def startService(self): self.initial_cpu = process_time() + return super().startService() def check(self): now_wall = time.time() From e17f4e68048fa08c0a9c9c29016da931d894b143 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:43:54 -0400 Subject: [PATCH 549/916] news fragment --- newsfragments/3883.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3883.minor diff --git a/newsfragments/3883.minor b/newsfragments/3883.minor new file mode 100644 index 000000000..e69de29bb From 9bcf241f10875fb2f5d7d592073d1770a9082408 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 11:40:30 -0400 Subject: [PATCH 550/916] Use environment variables we expect most runners to have. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf0c66aff..75e2ab749 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -398,6 +398,7 @@ jobs: image: "nixos/nix:2.3.16" environment: + <<: *UTF_8_ENVIRONMENT # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and # allows us to push to CACHIX_NAME. We only need this set for # `cachix use` in this step. From 62ac5bd0841ccfdb76341e17944c716d19f1ad09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 11:40:54 -0400 Subject: [PATCH 551/916] News file. --- newsfragments/3882.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3882.minor diff --git a/newsfragments/3882.minor b/newsfragments/3882.minor new file mode 100644 index 000000000..e69de29bb From 211343eca81a126c97df8d43e03995aacfbd644b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 11:51:39 -0400 Subject: [PATCH 552/916] Set the Hypothesis profile in more robust way. --- .circleci/config.yml | 1 - tests.nix | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75e2ab749..cf0c66aff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -398,7 +398,6 @@ jobs: image: "nixos/nix:2.3.16" environment: - <<: *UTF_8_ENVIRONMENT # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and # allows us to push to CACHIX_NAME. We only need this set for # `cachix use` in this step. diff --git a/tests.nix b/tests.nix index 53a8885c0..dd477c273 100644 --- a/tests.nix +++ b/tests.nix @@ -73,6 +73,7 @@ let in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' + export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata # It's not cool to put the whole _trial_temp into $out because it has weird From 98634ae5cb3f706ccf8d7a3abaa1c43588d36793 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 12:36:48 -0400 Subject: [PATCH 553/916] allow the correct number of samples in also speed up the test with some shorter intervals --- src/allmydata/stats.py | 2 +- src/allmydata/test/test_stats.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index a2d208560..6e8de47e9 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -21,7 +21,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) + self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH + 1) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) diff --git a/src/allmydata/test/test_stats.py b/src/allmydata/test/test_stats.py index e56f9d444..6fe690f1f 100644 --- a/src/allmydata/test/test_stats.py +++ b/src/allmydata/test/test_stats.py @@ -17,7 +17,7 @@ from allmydata.util import pollmixin import allmydata.test.common_util as testutil class FasterMonitor(CPUUsageMonitor): - POLL_INTERVAL = 0.1 + POLL_INTERVAL = 0.01 class CPUUsage(unittest.TestCase, pollmixin.PollMixin, testutil.StallMixin): @@ -36,9 +36,9 @@ class CPUUsage(unittest.TestCase, pollmixin.PollMixin, testutil.StallMixin): def _poller(): return bool(len(m.samples) == m.HISTORY_LENGTH+1) d = self.poll(_poller) - # pause one more second, to make sure that the history-trimming code - # is exercised - d.addCallback(self.stall, 1.0) + # pause a couple more intervals, to make sure that the history-trimming + # code is exercised + d.addCallback(self.stall, FasterMonitor.POLL_INTERVAL * 2) def _check(res): s = m.get_stats() self.failUnless("cpu_monitor.1min_avg" in s) From 5310747eaa69d4bedf5e89383de58bc6692a5827 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Mar 2022 16:33:29 -0400 Subject: [PATCH 554/916] Start hooking up end-to-end tests with TLS, fixing bugs along the way. At this point the issue is that the client fails certificate validation (which is expected lacking the pinning validation logic, which should be added next). --- src/allmydata/storage/http_client.py | 6 +++-- src/allmydata/storage/http_common.py | 2 +- src/allmydata/storage/http_server.py | 22 +++++++++++----- src/allmydata/test/certs/domain.crt | 19 ++++++++++++++ src/allmydata/test/certs/private.key | 27 ++++++++++++++++++++ src/allmydata/test/test_istorageserver.py | 31 ++++++++++++----------- 6 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 src/allmydata/test/certs/domain.crt create mode 100644 src/allmydata/test/certs/private.key diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 572da506c..cbb111d18 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -3,7 +3,6 @@ HTTP client that talks to the HTTP storage server. """ from typing import Union, Set, Optional -from treq.testing import StubTreq from base64 import b64encode @@ -77,7 +76,7 @@ class StorageClient(object): self._treq = treq @classmethod - def from_furl(cls, furl: DecodedURL) -> "StorageClient": + def from_furl(cls, furl: DecodedURL, treq=treq) -> "StorageClient": """ Create a ``StorageClient`` for the given furl. """ @@ -86,6 +85,9 @@ class StorageClient(object): swissnum = furl.path[0].encode("ascii") certificate_hash = furl.user.encode("ascii") + https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port) + return cls(https_url, swissnum, treq) + def relative_url(self, path): """Get a URL relative to the base URL.""" return self._base_url.click(path) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index c5e087b5b..11ab880c7 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -48,4 +48,4 @@ def get_spki_hash(certificate: Certificate) -> bytes: public_key_bytes = certificate.public_key().public_bytes( Encoding.DER, PublicFormat.SubjectPublicKeyInfo ) - return b64encode(sha256(public_key_bytes).digest()).strip() + return b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c3228ea04..80b34daa9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -534,6 +534,8 @@ def listen_tls( The hostname is the external IP or hostname clients will connect to; it does not modify what interfaces the server listens on. To set the listening interface, use the ``interface`` argument. + + Port can be 0 to choose a random port. """ endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( quoteStringArgument(str(private_key_path)), @@ -545,13 +547,19 @@ def listen_tls( endpoint = serverFromString(reactor, endpoint_string) def build_furl(listening_port: IListeningPort) -> DecodedURL: - furl = DecodedURL() - furl.fragment = "v=1" # HTTP-based - furl.host = hostname - furl.port = listening_port.getHost().port - furl.path = (server._swissnum,) - furl.user = get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())) - furl.scheme = "pb" + furl = DecodedURL().replace( + fragment="v=1", # HTTP-based + host=hostname, + port=listening_port.getHost().port, + path=(str(server._swissnum, "ascii"),), + userinfo=[ + str( + get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())), + "ascii", + ) + ], + scheme="pb", + ) return furl return endpoint.listen(Site(server.get_resource())).addCallback( diff --git a/src/allmydata/test/certs/domain.crt b/src/allmydata/test/certs/domain.crt new file mode 100644 index 000000000..8932b44e7 --- /dev/null +++ b/src/allmydata/test/certs/domain.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfMCFHrs4pMBs35SlU3ZGMnVY5qp5MfZMA0GCSqGSIb3DQEBCwUAMEIx +CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl +ZmF1bHQgQ29tcGFueSBMdGQwHhcNMjIwMzIzMjAwNTM0WhcNMjIwNDIyMjAwNTM0 +WjBCMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQK +DBNEZWZhdWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiESF/Lvnrc +VcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsxx89u/W5B +VgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4XtqlCKNFGYO +7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVmr7UgatuA +LrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8ACif+ZQJc +AukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCv +BzVSinez5lcSWWCRve/TlMePEJK5d7OFcc90n7kmn+rkEYrel3a7Q+ctJxY9AKYG +A9X1AMDSH9z3243KFNaRJ1xKg0Mg8J/BLN9iphM5AnAuiAkCqs8VbD4hF4hZHLMZ +BVNyuLSdo+lBzbS57/Lz+lUWcxrXR5qgsEWSbjP+SrsDQKODfyoxKuU0XrxmNLd2 +dTswbZKsqXBs80/T1jHwJjJXLp6YUsZqN1TGYtk8hEcE7bGaC3n7WhRjBP1WghNl +OG7FFRPte2w5seQRgkrBodLb9OCkhU4xfdyLnFICqkQHQAqIdXksEvir9uGY/yjC +zxAh60LEEQz6Kz29jYog +-----END CERTIFICATE----- diff --git a/src/allmydata/test/certs/private.key b/src/allmydata/test/certs/private.key new file mode 100644 index 000000000..b4b5ed580 --- /dev/null +++ b/src/allmydata/test/certs/private.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiE +SF/LvnrcVcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsx +x89u/W5BVgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4Xtq +lCKNFGYO7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVm +r7UgatuALrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8A +Cif+ZQJcAukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABAoIBAG71rGDuIazif+Bq +PGDDs/c5q3BQ9LLdF6Zywonp3J0CFqbbc/BsefYVrA4I6mnqECd2TWsO+cfyKxeb +aRrDne75l9YZcaXU2ZKqtIKShHQgqlV2giX6mMCJXWWlenfRMglooLaGslxYZR6e +/GG9iVbXLI0m52EhYjH21W6MVgXUhsrDoI16pB87zk7jFZzyNsjRU5+bwr4L2jed +A1PseE6AI2kIpJCl8IIu6hRhVwjr8MIkaAtI3G8WmSAru99apHNttf6sgB2kcq2r +Qp1uXEXNVFQiJqcwMPOlWZ5X0kMIBxmFe10MkJbmCUoE/jPqO90XN2jyZPSONZU8 +4yqd9GECgYEA5qQ+tfgOJFJ86t103pwkPm0+PxuQUTXry5ETnez+wxqqwbDxrEHi +MQoPZuVXPkbQ6g80KSpdI7AkFvu6BzcNlgpOI3gLZ30PHTF4HJTP01fKfbbVhg8N +WJS0yUh+kQDrGVcZbIbB5Q1vS5hu8ftk5ukns5BdFf/NS7fBfU8b3K0CgYEA5Onm +V2D1D9kdhTjta8f0d0s6+TdHoV86SRbkAEgnqzwlHWV17LlpXQic6iwokfS4TQSl ++1Z23Dt+OhLm/N0N3LgCxBhzTMnWGdy+w9co4GifwqR6T72JAxGOVoqIWk2cVpa5 +8qJx0eAFXqcvpIASEoxYrdoKFUh60mAiQE6JQ08CgYB8wCoLUviTPOrEPrSQE/Sm +r4ATsl0FEB1SJk5uBVpnPW1PBt4xRhGKZN6f0Ty3OqaVc1PLUFbAju12YQHmFSkM +Ftbc6HmCqGocaD2HeBZRQhMMnHAx6sJVP1np5YRP+icvtaTSxrDpq7KfOPwJdujE +3SfUQCmZVJs+cU3+8WMooQKBgQCdvvl2eWAm/a00IxipT2+NzY/kMU3xTFg0Ccww +zYhYnefNrB9pdBPBgq/vR2LlwchHes6OtvTNq0m+50u6MPLeiQeO7nJ2FhiuVco3 +1staaX6+eO24iZojPTPjOy/fWuBDYzbcl0jsIf5RTdCtAXxyv7hUhY6xP/Mzif/Q +ZM5+TQKBgQCF7pB6yLIAp6YJQ4uqOVbC2bMLr6tVWaNdEykiDd9zQkoacw/DPX1Y +FKehY/9EKraJ4t6d2/BBpJQyuIU4/gz8QMvjqQGP3NIfVeqBAPYo/nTYKOK0PSxB +Kl28Axxz6rjEeK4BixOES5PXuq2nNJXT8OSPYZQxQdTHstCWOP4Z6g== +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 253ff6046..7b3810f21 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -24,12 +24,11 @@ else: from random import Random from unittest import SkipTest +from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor -from twisted.internet.endpoints import serverFromString -from twisted.web.server import Site from twisted.web.client import Agent, HTTPConnectionPool from hyperlink import DecodedURL from treq.client import HTTPClient @@ -40,7 +39,7 @@ from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase, SameProcessStreamEndpointAssigner from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage.http_server import HTTPServer +from allmydata.storage.http_server import HTTPServer, listen_tls from allmydata.storage.http_client import StorageClient from allmydata.storage_client import _HTTPStorageServer @@ -1074,27 +1073,29 @@ class _HTTPMixin(_SharedMixin): swissnum = b"1234" http_storage_server = HTTPServer(self.server, swissnum) - # Listen on randomly assigned port: - tcp_address, endpoint_string = self._port_assigner.assign(reactor) - _, host, port = tcp_address.split(":") - port = int(port) - endpoint = serverFromString(reactor, endpoint_string) - listening_port = yield endpoint.listen(Site(http_storage_server.get_resource())) + # Listen on randomly assigned port, using self-signed cert we generated + # manually: + certs_dir = Path(__file__).parent / "certs" + furl, listening_port = yield listen_tls( + reactor, + http_storage_server, + "127.0.0.1", + 0, + certs_dir / "private.key", + certs_dir / "domain.crt", + interface="127.0.0.1", + ) self.addCleanup(listening_port.stopListening) # Create HTTP client with non-persistent connections, so we don't leak # state across tests: treq_client = HTTPClient( - Agent(reactor, HTTPConnectionPool(reactor, persistent=False)) + Agent(reactor, pool=HTTPConnectionPool(reactor, persistent=False)) ) returnValue( _HTTPStorageServer.from_http_client( - StorageClient( - DecodedURL().replace(scheme="http", host=host, port=port), - swissnum, - treq=treq_client, - ) + StorageClient.from_furl(furl, treq_client) ) ) # Eventually should also: From c737bcdb6f59249d09eaa0587dfd76609dd66e8b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Mar 2022 18:40:12 -0400 Subject: [PATCH 555/916] use the generic version of the correct types for `samples` --- src/allmydata/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 6e8de47e9..f6361b074 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -5,6 +5,7 @@ Ported to Python 3. from collections import deque from time import process_time import time +from typing import Deque, Tuple from twisted.application import service from twisted.application.internet import TimerService @@ -21,7 +22,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH + 1) + self.samples: Deque[Tuple[float, float]] = deque([], self.HISTORY_LENGTH + 1) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) From be0ff08275695aad99c8e3cceba6700b52b73be3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Mar 2022 17:18:06 -0400 Subject: [PATCH 556/916] Possibly correct, but communicating, end-to-end TLS with some amount of validation logic. Still untested! --- src/allmydata/storage/http_client.py | 109 +++++++++++++++++++++- src/allmydata/test/test_istorageserver.py | 10 +- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cbb111d18..fc27e1e30 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -14,13 +14,26 @@ from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http +from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.ssl import CertificateOptions +from twisted.internet import reactor +from twisted.web.client import Agent, HTTPConnectionPool +from zope.interface import implementer from hyperlink import DecodedURL import treq from treq.client import HTTPClient from treq.testing import StubTreq +from OpenSSL import SSL -from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE +from .http_common import ( + swissnum_auth_header, + Secrets, + get_content_type, + CBOR_MIME_TYPE, + get_spki_hash, +) from .common import si_b2a @@ -59,6 +72,86 @@ class ImmutableCreateResult(object): allocated = attr.ib(type=Set[int]) +class _TLSContextFactory(CertificateOptions): + """ + Create a context that validates the way Tahoe-LAFS wants to: based on a + pinned certificate hash, rather than a certificate authority. + + Originally implemented as part of Foolscap. + """ + + def getContext(self, expected_spki_hash: bytes) -> SSL.Context: + def always_validate(conn, cert, errno, depth, preverify_ok): + # This function is called to validate the certificate received by + # the other end. OpenSSL calls it multiple times, each time it + # see something funny, to ask if it should proceed. + + # We do not care about certificate authorities or revocation + # lists, we just want to know that the certificate has a valid + # signature and follow the chain back to one which is + # self-signed. We need to protect against forged signatures, but + # not the usual TLS concerns about invalid CAs or revoked + # certificates. + + # these constants are from openssl-0.9.7g/crypto/x509/x509_vfy.h + # and do not appear to be exposed by pyopenssl. Ick. + things_are_ok = ( + 0, # X509_V_OK + 9, # X509_V_ERR_CERT_NOT_YET_VALID + 10, # X509_V_ERR_CERT_HAS_EXPIRED + 18, # X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT + 19, # X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN + ) + # TODO can we do this once instead of multiple times? + if ( + errno in things_are_ok + and get_spki_hash(cert.to_cryptography()) == expected_spki_hash + ): + return 1 + # TODO: log the details of the error, because otherwise they get + # lost in the PyOpenSSL exception that will eventually be raised + # (possibly OpenSSL.SSL.Error: certificate verify failed) + + # I think that X509_V_ERR_CERT_SIGNATURE_FAILURE is the most + # obvious sign of hostile attack. + return 0 + + ctx = CertificateOptions.getContext(self) + + # VERIFY_PEER means we ask the the other end for their certificate. + ctx.set_verify(SSL.VERIFY_PEER, always_validate) + return ctx + + +@implementer(IPolicyForHTTPS) +@implementer(IOpenSSLClientConnectionCreator) +@attr.s +class _StorageClientHTTPSPolicy: + """ + A HTTPS policy that: + + 1. Makes sure the SPKI hash of the certificate matches a known hash (NEEDS TEST). + 2. The certificate hasn't expired. (NEEDS TEST) + 3. The server has a private key that matches the certificate (NEEDS TEST). + + I.e. pinning-based validation. + """ + + expected_spki_hash = attr.ib(type=bytes) + + # IPolicyForHTTPS + def creatorForNetloc(self, hostname, port): + return self + + # IOpenSSLClientConnectionCreator + def clientConnectionForTLS(self, tlsProtocol): + connection = SSL.Connection( + _TLSContextFactory().getContext(self.expected_spki_hash), None + ) + connection.set_app_data(tlsProtocol) + return connection + + class StorageClient(object): """ Low-level HTTP client that talks to the HTTP storage server. @@ -76,17 +169,27 @@ class StorageClient(object): self._treq = treq @classmethod - def from_furl(cls, furl: DecodedURL, treq=treq) -> "StorageClient": + def from_furl(cls, furl: DecodedURL, persistent: bool = True) -> "StorageClient": """ Create a ``StorageClient`` for the given furl. + + ``persistent`` indicates whether to use persistent HTTP connections. """ assert furl.fragment == "v=1" assert furl.scheme == "pb" swissnum = furl.path[0].encode("ascii") certificate_hash = furl.user.encode("ascii") + treq_client = HTTPClient( + Agent( + reactor, + _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), + pool=HTTPConnectionPool(reactor, persistent=persistent), + ) + ) + https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port) - return cls(https_url, swissnum, treq) + return cls(https_url, swissnum, treq_client) def relative_url(self, path): """Get a URL relative to the base URL.""" diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 7b3810f21..272e63764 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -29,9 +29,6 @@ from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor -from twisted.web.client import Agent, HTTPConnectionPool -from hyperlink import DecodedURL -from treq.client import HTTPClient from foolscap.api import Referenceable, RemoteException @@ -1089,15 +1086,12 @@ class _HTTPMixin(_SharedMixin): # Create HTTP client with non-persistent connections, so we don't leak # state across tests: - treq_client = HTTPClient( - Agent(reactor, pool=HTTPConnectionPool(reactor, persistent=False)) - ) - returnValue( _HTTPStorageServer.from_http_client( - StorageClient.from_furl(furl, treq_client) + StorageClient.from_furl(furl, persistent=False) ) ) + # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2d17658ce..0630eeaa0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -67,7 +67,7 @@ class HTTPFurlTests(SyncTestCase): openssl asn1parse -noout -inform pem -out public.key openssl dgst -sha256 -binary public.key | openssl enc -base64 """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM=" + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" certificate_text = b"""\ -----BEGIN CERTIFICATE----- MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx From e50d88f46d07b36926a053265767cb668e1ee8b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 10:45:54 -0400 Subject: [PATCH 557/916] Technically this doesn't matter, because it's client-side, but it's good habit. --- src/allmydata/storage/http_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index fc27e1e30..37bf29901 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -35,6 +35,7 @@ from .http_common import ( get_spki_hash, ) from .common import si_b2a +from ..util.hashutil import timing_safe_compare def _encode_si(si): # type: (bytes) -> str @@ -103,9 +104,8 @@ class _TLSContextFactory(CertificateOptions): 19, # X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN ) # TODO can we do this once instead of multiple times? - if ( - errno in things_are_ok - and get_spki_hash(cert.to_cryptography()) == expected_spki_hash + if errno in things_are_ok and timing_safe_compare( + get_spki_hash(cert.to_cryptography()), expected_spki_hash ): return 1 # TODO: log the details of the error, because otherwise they get From 9240d9d657dd4c25d080fb4a4711ba5810f33c2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 10:46:14 -0400 Subject: [PATCH 558/916] Expand the comment. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 80b34daa9..8cc7972b8 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -548,7 +548,7 @@ def listen_tls( def build_furl(listening_port: IListeningPort) -> DecodedURL: furl = DecodedURL().replace( - fragment="v=1", # HTTP-based + fragment="v=1", # how we know this furl is HTTP-based host=hostname, port=listening_port.getHost().port, path=(str(server._swissnum, "ascii"),), From 6f86675766741f5fe9f8b9762a4434a83421f88d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 10:59:16 -0400 Subject: [PATCH 559/916] Split into its own file. --- src/allmydata/test/test_storage_http.py | 43 +------------------- src/allmydata/test/test_storage_https.py | 52 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 42 deletions(-) create mode 100644 src/allmydata/test/test_storage_https.py diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 0630eeaa0..eb0c61f9a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -27,12 +27,11 @@ from collections_extended import RangeMap from twisted.internet.task import Clock from twisted.web import http from twisted.web.http_headers import Headers -from cryptography.x509 import load_pem_x509_certificate from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound from .common import SyncTestCase -from ..storage.http_common import get_content_type, get_spki_hash +from ..storage.http_common import get_content_type from ..storage.common import si_b2a from ..storage.server import StorageServer from ..storage.http_server import ( @@ -54,46 +53,6 @@ from ..storage.http_client import ( ) -class HTTPFurlTests(SyncTestCase): - """Tests for HTTP furls.""" - - def test_spki_hash(self): - """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. - - The expected hash was generated using Appendix A instructions in the - RFC:: - - openssl x509 -noout -in certificate.pem -pubkey | \ - openssl asn1parse -noout -inform pem -out public.key - openssl dgst -sha256 -binary public.key | openssl enc -base64 - """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" - certificate_text = b"""\ ------BEGIN CERTIFICATE----- -MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx -CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl -dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh -bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD -VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x -HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu -Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG -q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC -M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj -GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu -YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k -yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk -YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH -+fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C -i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs -2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ -PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr -ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG ------END CERTIFICATE----- -""" - certificate = load_pem_x509_certificate(certificate_text) - self.assertEqual(get_spki_hash(certificate), expected_hash) - - class HTTPUtilities(SyncTestCase): """Tests for HTTP common utilities.""" diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py new file mode 100644 index 000000000..ebb605d04 --- /dev/null +++ b/src/allmydata/test/test_storage_https.py @@ -0,0 +1,52 @@ +""" +Tests for the TLS part of the HTTP Storage Protocol. + +More broadly, these are tests for HTTPS usage as replacement for Foolscap's +server authentication logic, which may one day apply outside of HTTP Storage +Protocol. +""" + +from cryptography.x509 import load_pem_x509_certificate + +from .common import SyncTestCase +from ..storage.http_common import get_spki_hash + + +class HTTPFurlTests(SyncTestCase): + """Tests for HTTP furls.""" + + def test_spki_hash(self): + """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. + + The expected hash was generated using Appendix A instructions in the + RFC:: + + openssl x509 -noout -in certificate.pem -pubkey | \ + openssl asn1parse -noout -inform pem -out public.key + openssl dgst -sha256 -binary public.key | openssl enc -base64 + """ + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" + certificate_text = b"""\ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx +CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl +dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh +bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD +VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x +HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG +q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC +M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj +GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu +YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k +yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk +YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH ++fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C +i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs +2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ +PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr +ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG +-----END CERTIFICATE----- +""" + certificate = load_pem_x509_certificate(certificate_text) + self.assertEqual(get_spki_hash(certificate), expected_hash) From 712f4f138546f315640d1e28c4955e01d28f2a2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 13:49:11 -0400 Subject: [PATCH 560/916] Sketch of first HTTPS client logic test. --- src/allmydata/test/test_storage_https.py | 160 ++++++++++++++++++++++- 1 file changed, 155 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index ebb605d04..7f6b4c039 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -6,14 +6,30 @@ server authentication logic, which may one day apply outside of HTTP Storage Protocol. """ -from cryptography.x509 import load_pem_x509_certificate +import datetime +from functools import wraps +from contextlib import asynccontextmanager -from .common import SyncTestCase +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization, hashes + +from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.web.server import Site +from twisted.web.static import Data +from twisted.web.client import Agent, HTTPConnectionPool +from treq.client import HTTPClient + +from .common import SyncTestCase, AsyncTestCase from ..storage.http_common import get_spki_hash +from ..storage.http_client import _StorageClientHTTPSPolicy -class HTTPFurlTests(SyncTestCase): - """Tests for HTTP furls.""" +class HTTPSFurlTests(SyncTestCase): + """Tests for HTTPS furls.""" def test_spki_hash(self): """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. @@ -48,5 +64,139 @@ PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG -----END CERTIFICATE----- """ - certificate = load_pem_x509_certificate(certificate_text) + certificate = x509.load_pem_x509_certificate(certificate_text) self.assertEqual(get_spki_hash(certificate), expected_hash) + + +def async_to_deferred(f): + """ + Wrap an async function to return a Deferred instead. + """ + + @wraps(f) + def not_async(*args, **kwargs): + return Deferred.fromCoroutine(f(*args, **kwargs)) + + return not_async + + +class PinningHTTPSValidation(AsyncTestCase): + """ + Test client-side validation logic of HTTPS certificates that uses + Tahoe-LAFS's pinning-based scheme instead of the traditional certificate + authority scheme. + + https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate + """ + + # NEEDED TESTS + # + # Success case + # + # Failure cases: + # Self-signed cert has wrong hash. Cert+private key match each other. + # Self-signed cert has correct hash, but doesn't match private key, so is invalid cert. + # Self-signed cert has correct hash, is valid, but expired. + # Anonymous server, without certificate. + # Cert has correct hash, but is not self-signed. + # Certificate that isn't valid yet (i.e. from the future)? Or is that silly. + + def to_file(self, key_or_cert) -> str: + """ + Write the given key or cert to a temporary file on disk, return the + path. + """ + path = self.mktemp() + with open(path, "wb") as f: + if isinstance(key_or_cert, x509.Certificate): + data = key_or_cert.public_bytes(serialization.Encoding.PEM) + else: + data = key_or_cert.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + f.write(data) + return path + + def generate_private_key(self): + """Create a RSA private key.""" + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + def generate_certificate(self, private_key, expires_days: int): + """Generate a certificate from a RSA private key.""" + subject = issuer = x509.Name( + [x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Yoyodyne")] + ) + return ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) + ) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost")]), + critical=False, + # Sign our certificate with our private key + ) + .sign(private_key, hashes.SHA256()) + ) + + @asynccontextmanager + async def listen(self, private_key_path, cert_path) -> str: + """ + Context manager that runs a HTTPS server with the given private key + and certificate. + + Returns a URL that will connect to the server. + """ + endpoint = serverFromString( + reactor, + "ssl:privateKey={}:certKey={}:port=0:interface=127.0.0.1".format( + quoteStringArgument(str(private_key_path)), + quoteStringArgument(str(cert_path)), + ), + ) + root = Data(b"YOYODYNE", "text/plain") + root.isLeaf = True + listening_port = await endpoint.listen(Site(root)) + try: + yield f"https://127.0.0.1:{listening_port.getHost().port}/" + finally: + await listening_port.stopListening() + + def request(self, url: str, expected_certificate: x509.Certificate): + """ + Send a HTTPS request to the given URL, ensuring that the given + certificate is the one used via SPKI-hash-based pinning comparison. + """ + # No persistent connections, so we don't have dirty reactor at the end + # of the test. + treq_client = HTTPClient( + Agent( + reactor, + _StorageClientHTTPSPolicy( + expected_spki_hash=get_spki_hash(expected_certificate) + ), + pool=HTTPConnectionPool(reactor, persistent=False), + ) + ) + return treq_client.get(url) + + @async_to_deferred + async def test_success(self): + """ + If all conditions are met, a TLS client using the Tahoe-LAFS policy can + connect to the server. + """ + private_key = self.generate_private_key() + certificate = self.generate_certificate(private_key, 10) + async with self.listen( + self.to_file(private_key), self.to_file(certificate) + ) as url: + response = await self.request(url, certificate) + self.assertEqual(await response.content(), b"YOYODYNE") From bd6e537891c34e9e7b0041238a11177f0b911a25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 13:52:25 -0400 Subject: [PATCH 561/916] Hacky fix. --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 7f6b4c039..d4057760e 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -18,6 +18,7 @@ from cryptography.hazmat.primitives import serialization, hashes from twisted.internet.endpoints import quoteStringArgument, serverFromString from twisted.internet import reactor from twisted.internet.defer import Deferred +from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool @@ -168,6 +169,9 @@ class PinningHTTPSValidation(AsyncTestCase): yield f"https://127.0.0.1:{listening_port.getHost().port}/" finally: await listening_port.stopListening() + # Make sure all server connections are closed :( No idea why this + # is necessary when it's not for IStorageServer HTTPS tests. + await deferLater(reactor, 0.001) def request(self, url: str, expected_certificate: x509.Certificate): """ From 23ce58140573c9bb8c032bac864a76fa417a5274 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 14:01:06 -0400 Subject: [PATCH 562/916] More tests; some still failing. --- src/allmydata/test/test_storage_https.py | 54 ++++++++++++++++++------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index d4057760e..35077d66e 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -21,7 +21,7 @@ from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data -from twisted.web.client import Agent, HTTPConnectionPool +from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived from treq.client import HTTPClient from .common import SyncTestCase, AsyncTestCase @@ -92,11 +92,8 @@ class PinningHTTPSValidation(AsyncTestCase): # NEEDED TESTS # - # Success case - # # Failure cases: - # Self-signed cert has wrong hash. Cert+private key match each other. - # Self-signed cert has correct hash, but doesn't match private key, so is invalid cert. + # DONE Self-signed cert has wrong hash. Cert+private key match each other. # Self-signed cert has correct hash, is valid, but expired. # Anonymous server, without certificate. # Cert has correct hash, but is not self-signed. @@ -124,21 +121,22 @@ class PinningHTTPSValidation(AsyncTestCase): """Create a RSA private key.""" return rsa.generate_private_key(public_exponent=65537, key_size=2048) - def generate_certificate(self, private_key, expires_days: int): + def generate_certificate( + self, private_key, expires_days: int = 10, org_name: str = "Yoyodyne" + ): """Generate a certificate from a RSA private key.""" subject = issuer = x509.Name( - [x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Yoyodyne")] + [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] ) + expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) return ( x509.CertificateBuilder() .subject_name(subject) .issuer_name(issuer) .public_key(private_key.public_key()) .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.datetime.utcnow()) - .not_valid_after( - datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) - ) + .not_valid_before(min(datetime.datetime.utcnow(), expires)) + .not_valid_after(expires) .add_extension( x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False, @@ -198,9 +196,41 @@ class PinningHTTPSValidation(AsyncTestCase): connect to the server. """ private_key = self.generate_private_key() - certificate = self.generate_certificate(private_key, 10) + certificate = self.generate_certificate(private_key) async with self.listen( self.to_file(private_key), self.to_file(certificate) ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + + @async_to_deferred + async def test_server_certificate_has_wrong_hash(self): + """ + If the server's certificate hash doesn't match the hash the client + expects, the request to the server fails. + """ + private_key1 = self.generate_private_key() + certificate1 = self.generate_certificate(private_key1) + private_key2 = self.generate_private_key() + certificate2 = self.generate_certificate(private_key2) + + async with self.listen( + self.to_file(private_key1), self.to_file(certificate1) + ) as url: + with self.assertRaises(ResponseNeverReceived): + await self.request(url, certificate2) + + @async_to_deferred + async def test_server_certificate_expired(self): + """ + If the server's certificate has expired, the request to the server + fails even if the hash matches the one the client expects. + """ + private_key = self.generate_private_key() + certificate = self.generate_certificate(private_key, expires_days=-10) + + async with self.listen( + self.to_file(private_key), self.to_file(certificate) + ) as url: + with self.assertRaises(ResponseNeverReceived): + await self.request(url, certificate) From 638154b2ad2ac5b6eafc57f58303c7c21ac0d21b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 15:46:42 -0400 Subject: [PATCH 563/916] Cleanups. --- src/allmydata/storage/http_client.py | 22 +++++++++------------- src/allmydata/test/test_storage_https.py | 20 ++++++++------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 37bf29901..4c8577e88 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -81,7 +81,11 @@ class _TLSContextFactory(CertificateOptions): Originally implemented as part of Foolscap. """ - def getContext(self, expected_spki_hash: bytes) -> SSL.Context: + def __init__(self, expected_spki_hash: bytes): + self.expected_spki_hash = expected_spki_hash + CertificateOptions.__init__(self) + + def getContext(self) -> SSL.Context: def always_validate(conn, cert, errno, depth, preverify_ok): # This function is called to validate the certificate received by # the other end. OpenSSL calls it multiple times, each time it @@ -105,15 +109,12 @@ class _TLSContextFactory(CertificateOptions): ) # TODO can we do this once instead of multiple times? if errno in things_are_ok and timing_safe_compare( - get_spki_hash(cert.to_cryptography()), expected_spki_hash + get_spki_hash(cert.to_cryptography()), self.expected_spki_hash ): return 1 # TODO: log the details of the error, because otherwise they get # lost in the PyOpenSSL exception that will eventually be raised # (possibly OpenSSL.SSL.Error: certificate verify failed) - - # I think that X509_V_ERR_CERT_SIGNATURE_FAILURE is the most - # obvious sign of hostile attack. return 0 ctx = CertificateOptions.getContext(self) @@ -128,13 +129,8 @@ class _TLSContextFactory(CertificateOptions): @attr.s class _StorageClientHTTPSPolicy: """ - A HTTPS policy that: - - 1. Makes sure the SPKI hash of the certificate matches a known hash (NEEDS TEST). - 2. The certificate hasn't expired. (NEEDS TEST) - 3. The server has a private key that matches the certificate (NEEDS TEST). - - I.e. pinning-based validation. + A HTTPS policy that ensures the SPKI hash of the public key matches a known + hash, i.e. pinning-based validation. """ expected_spki_hash = attr.ib(type=bytes) @@ -146,7 +142,7 @@ class _StorageClientHTTPSPolicy: # IOpenSSLClientConnectionCreator def clientConnectionForTLS(self, tlsProtocol): connection = SSL.Connection( - _TLSContextFactory().getContext(self.expected_spki_hash), None + _TLSContextFactory(self.expected_spki_hash).getContext(), None ) connection.set_app_data(tlsProtocol) return connection diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 35077d66e..801fabc6b 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -90,15 +90,6 @@ class PinningHTTPSValidation(AsyncTestCase): https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate """ - # NEEDED TESTS - # - # Failure cases: - # DONE Self-signed cert has wrong hash. Cert+private key match each other. - # Self-signed cert has correct hash, is valid, but expired. - # Anonymous server, without certificate. - # Cert has correct hash, but is not self-signed. - # Certificate that isn't valid yet (i.e. from the future)? Or is that silly. - def to_file(self, key_or_cert) -> str: """ Write the given key or cert to a temporary file on disk, return the @@ -224,7 +215,8 @@ class PinningHTTPSValidation(AsyncTestCase): async def test_server_certificate_expired(self): """ If the server's certificate has expired, the request to the server - fails even if the hash matches the one the client expects. + succeeds if the hash matches the one the client expects; expiration has + no effect. """ private_key = self.generate_private_key() certificate = self.generate_certificate(private_key, expires_days=-10) @@ -232,5 +224,9 @@ class PinningHTTPSValidation(AsyncTestCase): async with self.listen( self.to_file(private_key), self.to_file(certificate) ) as url: - with self.assertRaises(ResponseNeverReceived): - await self.request(url, certificate) + response = await self.request(url, certificate) + self.assertEqual(await response.content(), b"YOYODYNE") + + # TODO an obvious attack is a private key that doesn't match the + # certificate... but OpenSSL (quite rightly) won't let you listen with that + # so I don't know how to test that! From ae8a7eff438b60c8645aec357c5ba710f9da23c8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 15:52:31 -0400 Subject: [PATCH 564/916] Make mypy happy. --- src/allmydata/storage/http_server.py | 6 +++--- src/allmydata/test/test_storage_https.py | 2 +- tox.ini | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8cc7972b8..59728e1d3 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -552,12 +552,12 @@ def listen_tls( host=hostname, port=listening_port.getHost().port, path=(str(server._swissnum, "ascii"),), - userinfo=[ + userinfo=( str( get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())), "ascii", - ) - ], + ), + ), scheme="pb", ) return furl diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 801fabc6b..19f469990 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -137,7 +137,7 @@ class PinningHTTPSValidation(AsyncTestCase): ) @asynccontextmanager - async def listen(self, private_key_path, cert_path) -> str: + async def listen(self, private_key_path, cert_path): """ Context manager that runs a HTTPS server with the given private key and certificate. diff --git a/tox.ini b/tox.ini index 57489df89..859cf18e0 100644 --- a/tox.ini +++ b/tox.ini @@ -141,6 +141,7 @@ deps = types-six types-PyYAML types-pkg_resources + types-pyOpenSSL git+https://github.com/warner/foolscap # Twisted 21.2.0 introduces some type hints which we are not yet # compatible with. From 4e58748c4a6fb4349ea60a4bd95033ac6f63549b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Mar 2022 11:27:32 -0400 Subject: [PATCH 565/916] Get constants from OpenSSL directly. --- src/allmydata/storage/http_client.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4c8577e88..a4aaad1bc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -26,6 +26,7 @@ import treq from treq.client import HTTPClient from treq.testing import StubTreq from OpenSSL import SSL +from cryptography.hazmat.bindings.openssl.binding import Binding from .http_common import ( swissnum_auth_header, @@ -37,6 +38,8 @@ from .http_common import ( from .common import si_b2a from ..util.hashutil import timing_safe_compare +_OPENSSL = Binding().lib + def _encode_si(si): # type: (bytes) -> str """Encode the storage index into Unicode string.""" @@ -88,8 +91,8 @@ class _TLSContextFactory(CertificateOptions): def getContext(self) -> SSL.Context: def always_validate(conn, cert, errno, depth, preverify_ok): # This function is called to validate the certificate received by - # the other end. OpenSSL calls it multiple times, each time it - # see something funny, to ask if it should proceed. + # the other end. OpenSSL calls it multiple times, for each errno + # for each certificate. # We do not care about certificate authorities or revocation # lists, we just want to know that the certificate has a valid @@ -97,15 +100,12 @@ class _TLSContextFactory(CertificateOptions): # self-signed. We need to protect against forged signatures, but # not the usual TLS concerns about invalid CAs or revoked # certificates. - - # these constants are from openssl-0.9.7g/crypto/x509/x509_vfy.h - # and do not appear to be exposed by pyopenssl. Ick. things_are_ok = ( - 0, # X509_V_OK - 9, # X509_V_ERR_CERT_NOT_YET_VALID - 10, # X509_V_ERR_CERT_HAS_EXPIRED - 18, # X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT - 19, # X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN + _OPENSSL.X509_V_OK, + _OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID, + _OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED, + _OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT, + _OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN, ) # TODO can we do this once instead of multiple times? if errno in things_are_ok and timing_safe_compare( From 119ba9468e57a22cac191743c14c7f9ff50c3790 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Mar 2022 11:28:38 -0400 Subject: [PATCH 566/916] Not needed. --- src/allmydata/storage/http_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a4aaad1bc..c0371ffa5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -141,11 +141,9 @@ class _StorageClientHTTPSPolicy: # IOpenSSLClientConnectionCreator def clientConnectionForTLS(self, tlsProtocol): - connection = SSL.Connection( + return SSL.Connection( _TLSContextFactory(self.expected_spki_hash).getContext(), None ) - connection.set_app_data(tlsProtocol) - return connection class StorageClient(object): From da6838d6f9f65bcb1e7119ba980b773d210a358f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Mar 2022 11:35:45 -0400 Subject: [PATCH 567/916] Stop talking about furl, it's a NURL. --- docs/proposed/http-storage-node-protocol.rst | 20 ++++++++++---------- src/allmydata/storage/http_client.py | 18 +++++++++--------- src/allmydata/storage/http_server.py | 12 ++++++------ src/allmydata/test/test_istorageserver.py | 4 ++-- src/allmydata/test/test_storage_https.py | 4 ++-- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 2ceb3c03a..a6f0e2c36 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -35,10 +35,10 @@ Glossary (the storage service is an example of such an object) NURL - a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap + a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap swissnum - a short random string which is part of a fURL and which acts as a shared secret to authorize clients to use a storage service + a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service lease state associated with a share informing a storage server of the duration of storage desired by a client @@ -211,15 +211,15 @@ To further clarify, consider this example. Alice operates a storage node. Alice generates a key pair and secures it properly. Alice generates a self-signed storage node certificate with the key pair. -Alice's storage node announces (to an introducer) a fURL containing (among other information) the SPKI hash. +Alice's storage node announces (to an introducer) a NURL containing (among other information) the SPKI hash. Imagine the SPKI hash is ``i5xb...``. -This results in a fURL of ``pb://i5xb...@example.com:443/g3m5...#v=1``. +This results in a NURL of ``pb://i5xb...@example.com:443/g3m5...#v=1``. Bob creates a client node pointed at the same introducer. Bob's client node receives the announcement from Alice's storage node (indirected through the introducer). -Bob's client node recognizes the fURL as referring to an HTTP-dialect server due to the ``v=1`` fragment. -Bob's client node can now perform a TLS handshake with a server at the address in the fURL location hints +Bob's client node recognizes the NURL as referring to an HTTP-dialect server due to the ``v=1`` fragment. +Bob's client node can now perform a TLS handshake with a server at the address in the NURL location hints (``example.com:443`` in this example). Following the above described validation procedures, Bob's client node can determine whether it has reached Alice's storage node or not. @@ -230,7 +230,7 @@ Additionally, by continuing to interact using TLS, Bob's client and Alice's storage node are assured of both **message authentication** and **message confidentiality**. -Bob's client further inspects the fURL for the *swissnum*. +Bob's client further inspects the NURL for the *swissnum*. When Bob's client issues HTTP requests to Alice's storage node it includes the *swissnum* in its requests. **Storage authorization** has been achieved. @@ -266,8 +266,8 @@ Generation of a new certificate allows for certain non-optimal conditions to be * The ``commonName`` of ``newpb_thingy`` may be changed to a more descriptive value. * A ``notValidAfter`` field with a timestamp in the past may be updated. -Storage nodes will announce a new fURL for this new HTTP-based server. -This fURL will be announced alongside their existing Foolscap-based server's fURL. +Storage nodes will announce a new NURL for this new HTTP-based server. +This NURL will be announced alongside their existing Foolscap-based server's fURL. Such an announcement will resemble this:: { @@ -312,7 +312,7 @@ The follow sequence of events is likely: #. The client uses the information in its cache to open a Foolscap connection to the storage server. Ideally, -the client would not rely on an update from the introducer to give it the GBS fURL for the updated storage server. +the client would not rely on an update from the introducer to give it the GBS NURL for the updated storage server. Therefore, when an updated client connects to a storage server using Foolscap, it should request the server's version information. diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index c0371ffa5..06b9b1145 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -155,24 +155,24 @@ class StorageClient(object): self, url, swissnum, treq=treq ): # type: (DecodedURL, bytes, Union[treq,StubTreq,HTTPClient]) -> None """ - The URL is a HTTPS URL ("https://..."). To construct from a furl, use - ``StorageClient.from_furl()``. + The URL is a HTTPS URL ("https://..."). To construct from a NURL, use + ``StorageClient.from_nurl()``. """ self._base_url = url self._swissnum = swissnum self._treq = treq @classmethod - def from_furl(cls, furl: DecodedURL, persistent: bool = True) -> "StorageClient": + def from_nurl(cls, nurl: DecodedURL, persistent: bool = True) -> "StorageClient": """ - Create a ``StorageClient`` for the given furl. + Create a ``StorageClient`` for the given NURL. ``persistent`` indicates whether to use persistent HTTP connections. """ - assert furl.fragment == "v=1" - assert furl.scheme == "pb" - swissnum = furl.path[0].encode("ascii") - certificate_hash = furl.user.encode("ascii") + assert nurl.fragment == "v=1" + assert nurl.scheme == "pb" + swissnum = nurl.path[0].encode("ascii") + certificate_hash = nurl.user.encode("ascii") treq_client = HTTPClient( Agent( @@ -182,7 +182,7 @@ class StorageClient(object): ) ) - https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port) + https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) return cls(https_url, swissnum, treq_client) def relative_url(self, path): diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 59728e1d3..0374797c6 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -528,7 +528,7 @@ def listen_tls( interface: Optional[str], ) -> Deferred[Tuple[DecodedURL, IListeningPort]]: """ - Start a HTTPS storage server on the given port, return the fURL and the + Start a HTTPS storage server on the given port, return the NURL and the listening port. The hostname is the external IP or hostname clients will connect to; it @@ -546,9 +546,9 @@ def listen_tls( endpoint_string += ":interface={}".format(quoteStringArgument(interface)) endpoint = serverFromString(reactor, endpoint_string) - def build_furl(listening_port: IListeningPort) -> DecodedURL: - furl = DecodedURL().replace( - fragment="v=1", # how we know this furl is HTTP-based + def build_nurl(listening_port: IListeningPort) -> DecodedURL: + nurl = DecodedURL().replace( + fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) host=hostname, port=listening_port.getHost().port, path=(str(server._swissnum, "ascii"),), @@ -560,8 +560,8 @@ def listen_tls( ), scheme="pb", ) - return furl + return nurl return endpoint.listen(Site(server.get_resource())).addCallback( - lambda listening_port: (build_furl(listening_port), listening_port) + lambda listening_port: (build_nurl(listening_port), listening_port) ) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 272e63764..7c5e64042 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1073,7 +1073,7 @@ class _HTTPMixin(_SharedMixin): # Listen on randomly assigned port, using self-signed cert we generated # manually: certs_dir = Path(__file__).parent / "certs" - furl, listening_port = yield listen_tls( + nurl, listening_port = yield listen_tls( reactor, http_storage_server, "127.0.0.1", @@ -1088,7 +1088,7 @@ class _HTTPMixin(_SharedMixin): # state across tests: returnValue( _HTTPStorageServer.from_http_client( - StorageClient.from_furl(furl, persistent=False) + StorageClient.from_nurl(nurl, persistent=False) ) ) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 19f469990..f4242ae0c 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -29,8 +29,8 @@ from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy -class HTTPSFurlTests(SyncTestCase): - """Tests for HTTPS furls.""" +class HTTPSNurlTests(SyncTestCase): + """Tests for HTTPS NURLs.""" def test_spki_hash(self): """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. From 30a3b006a06c8f850c1008731323922147594200 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 30 Mar 2022 10:26:26 -0400 Subject: [PATCH 568/916] Include self-signed cert in package install. --- setup.py | 1 + src/allmydata/test/test_istorageserver.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 5285b5d08..1883032de 100644 --- a/setup.py +++ b/setup.py @@ -410,6 +410,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "static/css/*.css", ], "allmydata": ["ported-modules.txt"], + "allmydata.test": ["certs/*"] }, include_package_data=True, setup_requires=setup_requires, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 7c5e64042..9a9a1d39a 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1078,6 +1078,10 @@ class _HTTPMixin(_SharedMixin): http_storage_server, "127.0.0.1", 0, + # This is just a self-signed certificate with randomly generated + # private key; nothing at all special about it. You can regenerate + # with code in allmydata.test.test_storage_https or with openssl + # CLI, with no meaningful change to the test. certs_dir / "private.key", certs_dir / "domain.crt", interface="127.0.0.1", From 5972a13457f9b0104f77551681f81a3525992965 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 09:34:17 -0400 Subject: [PATCH 569/916] Add reactor argument. --- src/allmydata/storage/http_client.py | 3 +-- src/allmydata/test/test_istorageserver.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 06b9b1145..766036427 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -18,7 +18,6 @@ from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.internet.ssl import CertificateOptions -from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -163,7 +162,7 @@ class StorageClient(object): self._treq = treq @classmethod - def from_nurl(cls, nurl: DecodedURL, persistent: bool = True) -> "StorageClient": + def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> "StorageClient": """ Create a ``StorageClient`` for the given NURL. diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9a9a1d39a..c5c515434 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1092,7 +1092,7 @@ class _HTTPMixin(_SharedMixin): # state across tests: returnValue( _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, persistent=False) + StorageClient.from_nurl(nurl, reactor, persistent=False) ) ) From 2e934574f0bf169db493a17b7abf7e3c554d4a2e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 09:37:18 -0400 Subject: [PATCH 570/916] Switch to URL-safe base64 for SPKI hash, for nicer usage in NURLs. --- src/allmydata/storage/http_common.py | 6 ++++-- src/allmydata/test/test_storage_https.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index 11ab880c7..bd88f9fae 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -3,7 +3,7 @@ Common HTTP infrastructure for the storge server. """ from enum import Enum -from base64 import b64encode +from base64 import urlsafe_b64encode, b64encode from hashlib import sha256 from typing import Optional @@ -44,8 +44,10 @@ def get_spki_hash(certificate: Certificate) -> bytes: """ Get the public key hash, as per RFC 7469: base64 of sha256 of the public key encoded in DER + Subject Public Key Info format. + + We use the URL-safe base64 variant, since this is typically found in NURLs. """ public_key_bytes = certificate.public_key().public_bytes( Encoding.DER, PublicFormat.SubjectPublicKeyInfo ) - return b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") + return urlsafe_b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index f4242ae0c..82e907f46 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -42,7 +42,7 @@ class HTTPSNurlTests(SyncTestCase): openssl asn1parse -noout -inform pem -out public.key openssl dgst -sha256 -binary public.key | openssl enc -base64 """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC_mrVQHy4KAFyL-8ZNPGPM" certificate_text = b"""\ -----BEGIN CERTIFICATE----- MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx From 710fad4f8ac3d9cfa0f81fc391f88ac6532ec5e2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:10:42 -0400 Subject: [PATCH 571/916] Support broader range of server endpoints, and switch to more robust random port assignment. --- src/allmydata/storage/http_server.py | 51 +++++++++++++++-------- src/allmydata/test/test_istorageserver.py | 30 ++++--------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0374797c6..a2cb58545 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,19 +2,21 @@ HTTP server for storage. """ -from typing import Dict, List, Set, Tuple, Any, Optional +from typing import Dict, List, Set, Tuple, Any from pathlib import Path from functools import wraps from base64 import b64decode import binascii +from zope.interface import implementer from klein import Klein from twisted.web import http -from twisted.internet.interfaces import IListeningPort +from twisted.internet.interfaces import IListeningPort, IStreamServerEndpoint from twisted.internet.defer import Deferred -from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.web.server import Site +from twisted.protocols.tls import TLSMemoryBIOFactory import attr from werkzeug.http import ( parse_range_header, @@ -518,33 +520,48 @@ class HTTPServer(object): return b"" +@implementer(IStreamServerEndpoint) +@attr.s +class _TLSEndpointWrapper(object): + """ + Wrap an existing endpoint with the storage TLS policy. This is useful + because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for example + there's Tor and i2p. + """ + + endpoint = attr.ib(type=IStreamServerEndpoint) + context_factory = attr.ib(type=CertificateOptions) + + def listen(self, factory): + return self.endpoint.listen( + TLSMemoryBIOFactory(self.context_factory, False, factory) + ) + + def listen_tls( - reactor, server: HTTPServer, hostname: str, - port: int, + endpoint: IStreamServerEndpoint, private_key_path: Path, cert_path: Path, - interface: Optional[str], ) -> Deferred[Tuple[DecodedURL, IListeningPort]]: """ Start a HTTPS storage server on the given port, return the NURL and the listening port. - The hostname is the external IP or hostname clients will connect to; it - does not modify what interfaces the server listens on. To set the - listening interface, use the ``interface`` argument. + The hostname is the external IP or hostname clients will connect to, used + to constrtuct the NURL; it does not modify what interfaces the server + listens on. - Port can be 0 to choose a random port. + This will likely need to be updated eventually to handle Tor/i2p. """ - endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( - quoteStringArgument(str(private_key_path)), - quoteStringArgument(str(cert_path)), - port, + certificate = Certificate.loadPEM(cert_path.read_bytes()).original + private_key = PrivateCertificate.loadPEM( + cert_path.read_bytes() + b"\n" + private_key_path.read_bytes() + ).privateKey.original + endpoint = _TLSEndpointWrapper( + endpoint, CertificateOptions(privateKey=private_key, certificate=certificate) ) - if interface is not None: - endpoint_string += ":interface={}".format(quoteStringArgument(interface)) - endpoint = serverFromString(reactor, endpoint_string) def build_nurl(listening_port: IListeningPort) -> DecodedURL: nurl = DecodedURL().replace( diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index c5c515434..495115231 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -8,19 +8,9 @@ reused across tests, so each test should be careful to generate unique storage indexes. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from future.utils import bchr -from future.utils import PY2, bchr - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on -else: - from typing import Set +from typing import Set from random import Random from unittest import SkipTest @@ -29,7 +19,7 @@ from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor - +from twisted.internet.endpoints import serverFromString from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient @@ -1013,10 +1003,6 @@ class _SharedMixin(SystemTestMixin): AsyncTestCase.setUp(self) - self._port_assigner = SameProcessStreamEndpointAssigner() - self._port_assigner.setUp() - self.addCleanup(self._port_assigner.tearDown) - self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) @@ -1061,8 +1047,9 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) return _SharedMixin.setUp(self) @inlineCallbacks @@ -1073,18 +1060,17 @@ class _HTTPMixin(_SharedMixin): # Listen on randomly assigned port, using self-signed cert we generated # manually: certs_dir = Path(__file__).parent / "certs" + _, endpoint_string = self._port_assigner.assign(reactor) nurl, listening_port = yield listen_tls( - reactor, http_storage_server, "127.0.0.1", - 0, + serverFromString(reactor, endpoint_string), # This is just a self-signed certificate with randomly generated # private key; nothing at all special about it. You can regenerate # with code in allmydata.test.test_storage_https or with openssl # CLI, with no meaningful change to the test. certs_dir / "private.key", certs_dir / "domain.crt", - interface="127.0.0.1", ) self.addCleanup(listening_port.stopListening) From eda5925548518c3ab30cf787e20504defef56750 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:25:37 -0400 Subject: [PATCH 572/916] Get rid of another place where listen on port 0, and switch to FilePath only for now. --- src/allmydata/storage/http_server.py | 40 +++++++++++++++-------- src/allmydata/test/test_istorageserver.py | 8 ++--- src/allmydata/test/test_storage_https.py | 28 +++++++++------- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a2cb58545..d22a67995 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,6 @@ HTTP server for storage. """ from typing import Dict, List, Set, Tuple, Any -from pathlib import Path from functools import wraps from base64 import b64decode @@ -17,6 +16,8 @@ from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory +from twisted.python.filepath import FilePath + import attr from werkzeug.http import ( parse_range_header, @@ -524,14 +525,31 @@ class HTTPServer(object): @attr.s class _TLSEndpointWrapper(object): """ - Wrap an existing endpoint with the storage TLS policy. This is useful - because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for example - there's Tor and i2p. + Wrap an existing endpoint with the server-side storage TLS policy. This is + useful because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for + example there's Tor and i2p. """ endpoint = attr.ib(type=IStreamServerEndpoint) context_factory = attr.ib(type=CertificateOptions) + @classmethod + def from_paths( + cls, endpoint, private_key_path: FilePath, cert_path: FilePath + ) -> "_TLSEndpointWrapper": + """ + Create an endpoint with the given private key and certificate paths on + the filesystem. + """ + certificate = Certificate.loadPEM(cert_path.getContent()).original + private_key = PrivateCertificate.loadPEM( + cert_path.getContent() + b"\n" + private_key_path.getContent() + ).privateKey.original + certificate_options = CertificateOptions( + privateKey=private_key, certificate=certificate + ) + return cls(endpoint=endpoint, context_factory=certificate_options) + def listen(self, factory): return self.endpoint.listen( TLSMemoryBIOFactory(self.context_factory, False, factory) @@ -542,8 +560,8 @@ def listen_tls( server: HTTPServer, hostname: str, endpoint: IStreamServerEndpoint, - private_key_path: Path, - cert_path: Path, + private_key_path: FilePath, + cert_path: FilePath, ) -> Deferred[Tuple[DecodedURL, IListeningPort]]: """ Start a HTTPS storage server on the given port, return the NURL and the @@ -555,13 +573,7 @@ def listen_tls( This will likely need to be updated eventually to handle Tor/i2p. """ - certificate = Certificate.loadPEM(cert_path.read_bytes()).original - private_key = PrivateCertificate.loadPEM( - cert_path.read_bytes() + b"\n" + private_key_path.read_bytes() - ).privateKey.original - endpoint = _TLSEndpointWrapper( - endpoint, CertificateOptions(privateKey=private_key, certificate=certificate) - ) + endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path) def build_nurl(listening_port: IListeningPort) -> DecodedURL: nurl = DecodedURL().replace( @@ -571,7 +583,7 @@ def listen_tls( path=(str(server._swissnum, "ascii"),), userinfo=( str( - get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())), + get_spki_hash(load_pem_x509_certificate(cert_path.getContent())), "ascii", ), ), diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 495115231..bc7d5b853 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -14,12 +14,12 @@ from typing import Set from random import Random from unittest import SkipTest -from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor from twisted.internet.endpoints import serverFromString +from twisted.python.filepath import FilePath from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient @@ -1059,7 +1059,7 @@ class _HTTPMixin(_SharedMixin): # Listen on randomly assigned port, using self-signed cert we generated # manually: - certs_dir = Path(__file__).parent / "certs" + certs_dir = FilePath(__file__).parent().child("certs") _, endpoint_string = self._port_assigner.assign(reactor) nurl, listening_port = yield listen_tls( http_storage_server, @@ -1069,8 +1069,8 @@ class _HTTPMixin(_SharedMixin): # private key; nothing at all special about it. You can regenerate # with code in allmydata.test.test_storage_https or with openssl # CLI, with no meaningful change to the test. - certs_dir / "private.key", - certs_dir / "domain.crt", + certs_dir.child("private.key"), + certs_dir.child("domain.crt"), ) self.addCleanup(listening_port.stopListening) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 82e907f46..0a5b73a96 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -15,18 +15,20 @@ from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization, hashes -from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.internet.endpoints import serverFromString from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived +from twisted.python.filepath import FilePath from treq.client import HTTPClient -from .common import SyncTestCase, AsyncTestCase +from .common import SyncTestCase, AsyncTestCase, SameProcessStreamEndpointAssigner from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy +from ..storage.http_server import _TLSEndpointWrapper class HTTPSNurlTests(SyncTestCase): @@ -90,7 +92,13 @@ class PinningHTTPSValidation(AsyncTestCase): https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate """ - def to_file(self, key_or_cert) -> str: + def setUp(self): + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) + return AsyncTestCase.setUp(self) + + def to_file(self, key_or_cert) -> FilePath: """ Write the given key or cert to a temporary file on disk, return the path. @@ -106,7 +114,7 @@ class PinningHTTPSValidation(AsyncTestCase): encryption_algorithm=serialization.NoEncryption(), ) f.write(data) - return path + return FilePath(path) def generate_private_key(self): """Create a RSA private key.""" @@ -137,19 +145,17 @@ class PinningHTTPSValidation(AsyncTestCase): ) @asynccontextmanager - async def listen(self, private_key_path, cert_path): + async def listen(self, private_key_path: FilePath, cert_path: FilePath): """ Context manager that runs a HTTPS server with the given private key and certificate. Returns a URL that will connect to the server. """ - endpoint = serverFromString( - reactor, - "ssl:privateKey={}:certKey={}:port=0:interface=127.0.0.1".format( - quoteStringArgument(str(private_key_path)), - quoteStringArgument(str(cert_path)), - ), + location_hint, endpoint_string = self._port_assigner.assign(reactor) + underlying_endpoint = serverFromString(reactor, endpoint_string) + endpoint = _TLSEndpointWrapper.from_paths( + underlying_endpoint, private_key_path, cert_path ) root = Data(b"YOYODYNE", "text/plain") root.isLeaf = True From ab1297cdd6b7144aa974a41ee0a1b565f2f5202f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:27:42 -0400 Subject: [PATCH 573/916] Link to ticket. --- src/allmydata/test/test_storage_https.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 0a5b73a96..35aebf3f2 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -74,6 +74,8 @@ ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG def async_to_deferred(f): """ Wrap an async function to return a Deferred instead. + + Maybe solution to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3886 """ @wraps(f) From 5a82ea880baa84670cf9d7d3525fa128fb835a51 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:31:26 -0400 Subject: [PATCH 574/916] More specific methods. --- src/allmydata/test/test_storage_https.py | 39 +++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 35aebf3f2..1d95c037d 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -100,24 +100,35 @@ class PinningHTTPSValidation(AsyncTestCase): self.addCleanup(self._port_assigner.tearDown) return AsyncTestCase.setUp(self) - def to_file(self, key_or_cert) -> FilePath: + def _temp_file_with_data(self, data: bytes) -> FilePath: """ - Write the given key or cert to a temporary file on disk, return the - path. + Write data to temporary file, return its path. """ path = self.mktemp() with open(path, "wb") as f: - if isinstance(key_or_cert, x509.Certificate): - data = key_or_cert.public_bytes(serialization.Encoding.PEM) - else: - data = key_or_cert.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) f.write(data) return FilePath(path) + def cert_to_file(self, cert) -> FilePath: + """ + Write the given certificate to a temporary file on disk, return the + path. + """ + return self._temp_file_with_data(cert.public_bytes(serialization.Encoding.PEM)) + + def private_key_to_file(self, private_key) -> FilePath: + """ + Write the given key to a temporary file on disk, return the + path. + """ + return self._temp_file_with_data( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + def generate_private_key(self): """Create a RSA private key.""" return rsa.generate_private_key(public_exponent=65537, key_size=2048) @@ -197,7 +208,7 @@ class PinningHTTPSValidation(AsyncTestCase): private_key = self.generate_private_key() certificate = self.generate_certificate(private_key) async with self.listen( - self.to_file(private_key), self.to_file(certificate) + self.private_key_to_file(private_key), self.cert_to_file(certificate) ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") @@ -214,7 +225,7 @@ class PinningHTTPSValidation(AsyncTestCase): certificate2 = self.generate_certificate(private_key2) async with self.listen( - self.to_file(private_key1), self.to_file(certificate1) + self.private_key_to_file(private_key1), self.cert_to_file(certificate1) ) as url: with self.assertRaises(ResponseNeverReceived): await self.request(url, certificate2) @@ -230,7 +241,7 @@ class PinningHTTPSValidation(AsyncTestCase): certificate = self.generate_certificate(private_key, expires_days=-10) async with self.listen( - self.to_file(private_key), self.to_file(certificate) + self.private_key_to_file(private_key), self.cert_to_file(certificate) ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") From 22ebbba5acc80c542df36c4334f4a6b7ee03a081 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:35:05 -0400 Subject: [PATCH 575/916] Extend testing. --- src/allmydata/test/test_storage_https.py | 32 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 1d95c037d..4d08e28b9 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -134,12 +134,17 @@ class PinningHTTPSValidation(AsyncTestCase): return rsa.generate_private_key(public_exponent=65537, key_size=2048) def generate_certificate( - self, private_key, expires_days: int = 10, org_name: str = "Yoyodyne" + self, + private_key, + expires_days: int = 10, + valid_in_days: int = 0, + org_name: str = "Yoyodyne", ): """Generate a certificate from a RSA private key.""" subject = issuer = x509.Name( [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] ) + starts = datetime.datetime.utcnow() + datetime.timedelta(days=valid_in_days) expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) return ( x509.CertificateBuilder() @@ -147,7 +152,7 @@ class PinningHTTPSValidation(AsyncTestCase): .issuer_name(issuer) .public_key(private_key.public_key()) .serial_number(x509.random_serial_number()) - .not_valid_before(min(datetime.datetime.utcnow(), expires)) + .not_valid_before(min(starts, expires)) .not_valid_after(expires) .add_extension( x509.SubjectAlternativeName([x509.DNSName("localhost")]), @@ -246,6 +251,25 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # TODO an obvious attack is a private key that doesn't match the + @async_to_deferred + async def test_server_certificate_not_valid_yet(self): + """ + If the server's certificate is only valid starting in The Future, the + request to the server succeeds if the hash matches the one the client + expects; start time has no effect. + """ + private_key = self.generate_private_key() + certificate = self.generate_certificate( + private_key, expires_days=10, valid_in_days=5 + ) + + async with self.listen( + self.private_key_to_file(private_key), self.cert_to_file(certificate) + ) as url: + response = await self.request(url, certificate) + self.assertEqual(await response.content(), b"YOYODYNE") + + # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that - # so I don't know how to test that! + # so I don't know how to test that! See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3884 From 423512ad0019f155c012b8223e07fa8dde8169f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:48:17 -0400 Subject: [PATCH 576/916] Wait harder. --- src/allmydata/test/test_storage_https.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 4d08e28b9..6877dae2a 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -184,7 +184,7 @@ class PinningHTTPSValidation(AsyncTestCase): await listening_port.stopListening() # Make sure all server connections are closed :( No idea why this # is necessary when it's not for IStorageServer HTTPS tests. - await deferLater(reactor, 0.001) + await deferLater(reactor, 0.01) def request(self, url: str, expected_certificate: x509.Certificate): """ From bdcf054de61d66b25a8a12aff4658b11d29a3a7b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 8 Apr 2022 13:37:18 -0400 Subject: [PATCH 577/916] Switch to generating certs on the fly since Python packaging was being a pain. --- setup.py | 1 - src/allmydata/test/certs.py | 66 ++++++++++++++ src/allmydata/test/certs/domain.crt | 19 ---- src/allmydata/test/certs/private.key | 27 ------ src/allmydata/test/test_istorageserver.py | 20 +++-- src/allmydata/test/test_storage_https.py | 104 +++++----------------- 6 files changed, 101 insertions(+), 136 deletions(-) create mode 100644 src/allmydata/test/certs.py delete mode 100644 src/allmydata/test/certs/domain.crt delete mode 100644 src/allmydata/test/certs/private.key diff --git a/setup.py b/setup.py index 1883032de..5285b5d08 100644 --- a/setup.py +++ b/setup.py @@ -410,7 +410,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "static/css/*.css", ], "allmydata": ["ported-modules.txt"], - "allmydata.test": ["certs/*"] }, include_package_data=True, setup_requires=setup_requires, diff --git a/src/allmydata/test/certs.py b/src/allmydata/test/certs.py new file mode 100644 index 000000000..9e6640386 --- /dev/null +++ b/src/allmydata/test/certs.py @@ -0,0 +1,66 @@ +"""Utilities for generating TLS certificates.""" + +import datetime + +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization, hashes + +from twisted.python.filepath import FilePath + + +def cert_to_file(path: FilePath, cert) -> FilePath: + """ + Write the given certificate to a file on disk. Returns the path. + """ + path.setContent(cert.public_bytes(serialization.Encoding.PEM)) + return path + + +def private_key_to_file(path: FilePath, private_key) -> FilePath: + """ + Write the given key to a file on disk. Returns the path. + """ + path.setContent( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + return path + + +def generate_private_key(): + """Create a RSA private key.""" + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + +def generate_certificate( + private_key, + expires_days: int = 10, + valid_in_days: int = 0, + org_name: str = "Yoyodyne", +): + """Generate a certificate from a RSA private key.""" + subject = issuer = x509.Name( + [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] + ) + starts = datetime.datetime.utcnow() + datetime.timedelta(days=valid_in_days) + expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) + return ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(min(starts, expires)) + .not_valid_after(expires) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost")]), + critical=False, + # Sign our certificate with our private key + ) + .sign(private_key, hashes.SHA256()) + ) diff --git a/src/allmydata/test/certs/domain.crt b/src/allmydata/test/certs/domain.crt deleted file mode 100644 index 8932b44e7..000000000 --- a/src/allmydata/test/certs/domain.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDCzCCAfMCFHrs4pMBs35SlU3ZGMnVY5qp5MfZMA0GCSqGSIb3DQEBCwUAMEIx -CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl -ZmF1bHQgQ29tcGFueSBMdGQwHhcNMjIwMzIzMjAwNTM0WhcNMjIwNDIyMjAwNTM0 -WjBCMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQK -DBNEZWZhdWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiESF/Lvnrc -VcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsxx89u/W5B -VgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4XtqlCKNFGYO -7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVmr7UgatuA -LrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8ACif+ZQJc -AukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCv -BzVSinez5lcSWWCRve/TlMePEJK5d7OFcc90n7kmn+rkEYrel3a7Q+ctJxY9AKYG -A9X1AMDSH9z3243KFNaRJ1xKg0Mg8J/BLN9iphM5AnAuiAkCqs8VbD4hF4hZHLMZ -BVNyuLSdo+lBzbS57/Lz+lUWcxrXR5qgsEWSbjP+SrsDQKODfyoxKuU0XrxmNLd2 -dTswbZKsqXBs80/T1jHwJjJXLp6YUsZqN1TGYtk8hEcE7bGaC3n7WhRjBP1WghNl -OG7FFRPte2w5seQRgkrBodLb9OCkhU4xfdyLnFICqkQHQAqIdXksEvir9uGY/yjC -zxAh60LEEQz6Kz29jYog ------END CERTIFICATE----- diff --git a/src/allmydata/test/certs/private.key b/src/allmydata/test/certs/private.key deleted file mode 100644 index b4b5ed580..000000000 --- a/src/allmydata/test/certs/private.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiE -SF/LvnrcVcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsx -x89u/W5BVgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4Xtq -lCKNFGYO7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVm -r7UgatuALrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8A -Cif+ZQJcAukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABAoIBAG71rGDuIazif+Bq -PGDDs/c5q3BQ9LLdF6Zywonp3J0CFqbbc/BsefYVrA4I6mnqECd2TWsO+cfyKxeb -aRrDne75l9YZcaXU2ZKqtIKShHQgqlV2giX6mMCJXWWlenfRMglooLaGslxYZR6e -/GG9iVbXLI0m52EhYjH21W6MVgXUhsrDoI16pB87zk7jFZzyNsjRU5+bwr4L2jed -A1PseE6AI2kIpJCl8IIu6hRhVwjr8MIkaAtI3G8WmSAru99apHNttf6sgB2kcq2r -Qp1uXEXNVFQiJqcwMPOlWZ5X0kMIBxmFe10MkJbmCUoE/jPqO90XN2jyZPSONZU8 -4yqd9GECgYEA5qQ+tfgOJFJ86t103pwkPm0+PxuQUTXry5ETnez+wxqqwbDxrEHi -MQoPZuVXPkbQ6g80KSpdI7AkFvu6BzcNlgpOI3gLZ30PHTF4HJTP01fKfbbVhg8N -WJS0yUh+kQDrGVcZbIbB5Q1vS5hu8ftk5ukns5BdFf/NS7fBfU8b3K0CgYEA5Onm -V2D1D9kdhTjta8f0d0s6+TdHoV86SRbkAEgnqzwlHWV17LlpXQic6iwokfS4TQSl -+1Z23Dt+OhLm/N0N3LgCxBhzTMnWGdy+w9co4GifwqR6T72JAxGOVoqIWk2cVpa5 -8qJx0eAFXqcvpIASEoxYrdoKFUh60mAiQE6JQ08CgYB8wCoLUviTPOrEPrSQE/Sm -r4ATsl0FEB1SJk5uBVpnPW1PBt4xRhGKZN6f0Ty3OqaVc1PLUFbAju12YQHmFSkM -Ftbc6HmCqGocaD2HeBZRQhMMnHAx6sJVP1np5YRP+icvtaTSxrDpq7KfOPwJdujE -3SfUQCmZVJs+cU3+8WMooQKBgQCdvvl2eWAm/a00IxipT2+NzY/kMU3xTFg0Ccww -zYhYnefNrB9pdBPBgq/vR2LlwchHes6OtvTNq0m+50u6MPLeiQeO7nJ2FhiuVco3 -1staaX6+eO24iZojPTPjOy/fWuBDYzbcl0jsIf5RTdCtAXxyv7hUhY6xP/Mzif/Q -ZM5+TQKBgQCF7pB6yLIAp6YJQ4uqOVbC2bMLr6tVWaNdEykiDd9zQkoacw/DPX1Y -FKehY/9EKraJ4t6d2/BBpJQyuIU4/gz8QMvjqQGP3NIfVeqBAPYo/nTYKOK0PSxB -Kl28Axxz6rjEeK4BixOES5PXuq2nNJXT8OSPYZQxQdTHstCWOP4Z6g== ------END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index bc7d5b853..3d6f610be 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -25,6 +25,12 @@ from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase, SameProcessStreamEndpointAssigner +from .certs import ( + generate_certificate, + generate_private_key, + private_key_to_file, + cert_to_file, +) from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer, listen_tls from allmydata.storage.http_client import StorageClient @@ -1057,20 +1063,16 @@ class _HTTPMixin(_SharedMixin): swissnum = b"1234" http_storage_server = HTTPServer(self.server, swissnum) - # Listen on randomly assigned port, using self-signed cert we generated - # manually: - certs_dir = FilePath(__file__).parent().child("certs") + # Listen on randomly assigned port, using self-signed cert: + private_key = generate_private_key() + certificate = generate_certificate(private_key) _, endpoint_string = self._port_assigner.assign(reactor) nurl, listening_port = yield listen_tls( http_storage_server, "127.0.0.1", serverFromString(reactor, endpoint_string), - # This is just a self-signed certificate with randomly generated - # private key; nothing at all special about it. You can regenerate - # with code in allmydata.test.test_storage_https or with openssl - # CLI, with no meaningful change to the test. - certs_dir.child("private.key"), - certs_dir.child("domain.crt"), + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) self.addCleanup(listening_port.stopListening) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 6877dae2a..73c99725a 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -6,14 +6,10 @@ server authentication logic, which may one day apply outside of HTTP Storage Protocol. """ -import datetime from functools import wraps from contextlib import asynccontextmanager from cryptography import x509 -from cryptography.x509.oid import NameOID -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives import serialization, hashes from twisted.internet.endpoints import serverFromString from twisted.internet import reactor @@ -26,6 +22,12 @@ from twisted.python.filepath import FilePath from treq.client import HTTPClient from .common import SyncTestCase, AsyncTestCase, SameProcessStreamEndpointAssigner +from .certs import ( + generate_certificate, + generate_private_key, + private_key_to_file, + cert_to_file, +) from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper @@ -100,68 +102,6 @@ class PinningHTTPSValidation(AsyncTestCase): self.addCleanup(self._port_assigner.tearDown) return AsyncTestCase.setUp(self) - def _temp_file_with_data(self, data: bytes) -> FilePath: - """ - Write data to temporary file, return its path. - """ - path = self.mktemp() - with open(path, "wb") as f: - f.write(data) - return FilePath(path) - - def cert_to_file(self, cert) -> FilePath: - """ - Write the given certificate to a temporary file on disk, return the - path. - """ - return self._temp_file_with_data(cert.public_bytes(serialization.Encoding.PEM)) - - def private_key_to_file(self, private_key) -> FilePath: - """ - Write the given key to a temporary file on disk, return the - path. - """ - return self._temp_file_with_data( - private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - ) - - def generate_private_key(self): - """Create a RSA private key.""" - return rsa.generate_private_key(public_exponent=65537, key_size=2048) - - def generate_certificate( - self, - private_key, - expires_days: int = 10, - valid_in_days: int = 0, - org_name: str = "Yoyodyne", - ): - """Generate a certificate from a RSA private key.""" - subject = issuer = x509.Name( - [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] - ) - starts = datetime.datetime.utcnow() + datetime.timedelta(days=valid_in_days) - expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) - return ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(issuer) - .public_key(private_key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(min(starts, expires)) - .not_valid_after(expires) - .add_extension( - x509.SubjectAlternativeName([x509.DNSName("localhost")]), - critical=False, - # Sign our certificate with our private key - ) - .sign(private_key, hashes.SHA256()) - ) - @asynccontextmanager async def listen(self, private_key_path: FilePath, cert_path: FilePath): """ @@ -210,10 +150,11 @@ class PinningHTTPSValidation(AsyncTestCase): If all conditions are met, a TLS client using the Tahoe-LAFS policy can connect to the server. """ - private_key = self.generate_private_key() - certificate = self.generate_certificate(private_key) + private_key = generate_private_key() + certificate = generate_certificate(private_key) async with self.listen( - self.private_key_to_file(private_key), self.cert_to_file(certificate) + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") @@ -224,13 +165,14 @@ class PinningHTTPSValidation(AsyncTestCase): If the server's certificate hash doesn't match the hash the client expects, the request to the server fails. """ - private_key1 = self.generate_private_key() - certificate1 = self.generate_certificate(private_key1) - private_key2 = self.generate_private_key() - certificate2 = self.generate_certificate(private_key2) + private_key1 = generate_private_key() + certificate1 = generate_certificate(private_key1) + private_key2 = generate_private_key() + certificate2 = generate_certificate(private_key2) async with self.listen( - self.private_key_to_file(private_key1), self.cert_to_file(certificate1) + private_key_to_file(FilePath(self.mktemp()), private_key1), + cert_to_file(FilePath(self.mktemp()), certificate1), ) as url: with self.assertRaises(ResponseNeverReceived): await self.request(url, certificate2) @@ -242,11 +184,12 @@ class PinningHTTPSValidation(AsyncTestCase): succeeds if the hash matches the one the client expects; expiration has no effect. """ - private_key = self.generate_private_key() - certificate = self.generate_certificate(private_key, expires_days=-10) + private_key = generate_private_key() + certificate = generate_certificate(private_key, expires_days=-10) async with self.listen( - self.private_key_to_file(private_key), self.cert_to_file(certificate) + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") @@ -258,13 +201,14 @@ class PinningHTTPSValidation(AsyncTestCase): request to the server succeeds if the hash matches the one the client expects; start time has no effect. """ - private_key = self.generate_private_key() - certificate = self.generate_certificate( + private_key = generate_private_key() + certificate = generate_certificate( private_key, expires_days=10, valid_in_days=5 ) async with self.listen( - self.private_key_to_file(private_key), self.cert_to_file(certificate) + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") From e5b0e51f72cdb110179ec4b4a2527a8af55afb4b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 13:11:45 -0400 Subject: [PATCH 578/916] Server-side schema validation of CBOR. --- setup.py | 3 +- src/allmydata/storage/http_client.py | 4 ++- src/allmydata/storage/http_server.py | 38 +++++++++++++++++++++---- src/allmydata/test/test_storage_http.py | 30 +++++++++++++++++++ 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 5285b5d08..c84d0ecde 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,8 @@ install_requires = [ "klein", "werkzeug", "treq", - "cbor2" + "cbor2", + "pycddl", ] setup_requires = [ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 99275ae24..e9a593a3e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -47,7 +47,9 @@ def _decode_cbor(response): else: raise ClientException(-1, "Server didn't send CBOR") else: - return fail(ClientException(response.code, response.phrase)) + return treq.content(response).addCallback( + lambda data: fail(ClientException(response.code, response.phrase, data)) + ) @attr.s diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f648d8331..4bf552fc5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -21,7 +21,7 @@ from werkzeug.datastructures import ContentRange # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads - +from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE from .common import si_a2b @@ -215,6 +215,25 @@ class _HTTPError(Exception): self.code = code +# CDDL schemas. +# +# Tags are of the form #6.nnn, where the number is documented at +# https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 +# indicates a set. +_SCHEMAS = { + "allocate_buckets": Schema(""" + message = { + share-numbers: #6.258([* uint]) + allocated-size: uint + } + """), + "advise_corrupt_share": Schema(""" + message = { + reason: tstr + } + """) +} + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -229,6 +248,12 @@ class HTTPServer(object): request.setResponseCode(failure.value.code) return b"" + @_app.handle_errors(CDDLValidationError) + def _cddl_validation_error(self, request, failure): + """Handle CDDL validation errors.""" + request.setResponseCode(http.BAD_REQUEST) + return str(failure.value).encode("utf-8") + def __init__( self, storage_server, swissnum ): # type: (StorageServer, bytes) -> None @@ -268,7 +293,7 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - def _read_encoded(self, request) -> Any: + def _read_encoded(self, request, schema: Schema) -> Any: """ Read encoded request body data, decoding it with CBOR by default. """ @@ -276,7 +301,10 @@ class HTTPServer(object): if content_type == CBOR_MIME_TYPE: # TODO limit memory usage, client could send arbitrarily large data... # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return loads(request.content.read()) + message = request.content.read() + schema.validate_cbor(message) + result = loads(message) + return result else: raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) @@ -298,7 +326,7 @@ class HTTPServer(object): def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] - info = self._read_encoded(request) + info = self._read_encoded(request, _SCHEMAS["allocate_buckets"]) # We do NOT validate the upload secret for existing bucket uploads. # Another upload may be happening in parallel, with a different upload @@ -498,6 +526,6 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - info = self._read_encoded(request) + info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af90d58a9..af868ddce 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -413,6 +413,36 @@ class GenericHTTPAPITests(SyncTestCase): ) self.assertEqual(version, expected_version) + def test_schema_validation(self): + """ + Ensure that schema validation is happening: invalid CBOR should result + in bad request response code (error 400). + + We don't bother checking every single request, the API on the + server-side is designed to require a schema, so it validates + everywhere. But we check at least one to ensure we get correct + response code on bad input, so we know validation happened. + """ + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + url = self.http.client.relative_url( + "/v1/immutable/" + _encode_si(storage_index) + ) + message = {"bad-message": "missing expected keys"} + + response = result_of( + self.http.client.request( + "POST", + url, + lease_renew_secret=lease_secret, + lease_cancel_secret=lease_secret, + upload_secret=upload_secret, + message_to_serialize=message, + ) + ) + self.assertEqual(response.code, http.BAD_REQUEST) + class ImmutableHTTPAPITests(SyncTestCase): """ From 07049c2ac8f7eddc54393015f285f72231db8502 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 13:13:14 -0400 Subject: [PATCH 579/916] News file. --- newsfragments/3802.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3802.minor diff --git a/newsfragments/3802.minor b/newsfragments/3802.minor new file mode 100644 index 000000000..e69de29bb From dfad50b1c236aeac0c7944b636e81ef2c9855f67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 14:03:30 -0400 Subject: [PATCH 580/916] Better error. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4bf552fc5..3876409b0 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -86,8 +86,8 @@ def _authorization_decorator(required_secrets): try: secrets = _extract_secrets(authorization, required_secrets) except ClientSecretsException: - request.setResponseCode(400) - return b"" + request.setResponseCode(http.BAD_REQUEST) + return b"Missing required secrets" return f(self, request, secrets, *args, **kwargs) return route From 4b20b67ce60834cf81499b8eff2e2b6abfd4eb86 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 14:03:48 -0400 Subject: [PATCH 581/916] Client-side schema validation. --- src/allmydata/storage/http_client.py | 61 ++++++++++++++++++++++--- src/allmydata/test/test_storage_http.py | 30 +++++++++--- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e9a593a3e..3a758e592 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -11,6 +11,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps +from pycddl import Schema from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers @@ -36,14 +37,62 @@ class ClientException(Exception): self.code = code -def _decode_cbor(response): +# Schemas for server responses. +# +# TODO usage of sets is inconsistent. Either use everywhere (and document in +# spec document) or use nowhere. +_SCHEMAS = { + "get_version": Schema( + """ + message = {'http://allmydata.org/tahoe/protocols/storage/v1' => { + 'maximum-immutable-share-size' => uint + 'maximum-mutable-share-size' => uint + 'available-space' => uint + 'tolerates-immutable-read-overrun' => bool + 'delete-mutable-shares-with-zero-length-writev' => bool + 'fills-holes-with-zero-bytes' => bool + 'prevents-read-past-end-of-share-data' => bool + } + 'application-version' => bstr + } + """ + ), + "allocate_buckets": Schema( + """ + message = { + already-have: #6.258([* uint]) + allocated: #6.258([* uint]) + } + """ + ), + "immutable_write_share_chunk": Schema( + """ + message = { + required: [* {begin: uint, end: uint}] + } + """ + ), + "list_shares": Schema( + """ + message = [* uint] + """ + ), +} + + +def _decode_cbor(response, schema: Schema): """Given HTTP response, return decoded CBOR body.""" + + def got_content(data): + schema.validate_cbor(data) + return loads(data) + if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) if content_type == CBOR_MIME_TYPE: # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return treq.content(response).addCallback(loads) + return treq.content(response).addCallback(got_content) else: raise ClientException(-1, "Server didn't send CBOR") else: @@ -151,7 +200,7 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/v1/version") response = yield self._client.request("GET", url) - decoded_response = yield _decode_cbor(response) + decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) @@ -209,7 +258,7 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield _decode_cbor(response) + decoded_response = yield _decode_cbor(response, _SCHEMAS["allocate_buckets"]) returnValue( ImmutableCreateResult( already_have=decoded_response["already-have"], @@ -281,7 +330,7 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield _decode_cbor(response) + body = yield _decode_cbor(response, _SCHEMAS["immutable_write_share_chunk"]) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) @@ -334,7 +383,7 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield _decode_cbor(response) + body = yield _decode_cbor(response, _SCHEMAS["list_shares"]) returnValue(set(body)) else: raise ClientException(response.code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af868ddce..4679880a0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -18,6 +18,8 @@ from base64 import b64encode from contextlib import contextmanager from os import urandom +from cbor2 import dumps +from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq @@ -49,7 +51,7 @@ from ..storage.http_client import ( StorageClientGeneral, _encode_si, ) -from ..storage.http_common import get_content_type +from ..storage.http_common import get_content_type, CBOR_MIME_TYPE from ..storage.common import si_b2a @@ -239,6 +241,12 @@ class TestApp(object): else: return "BAD: {}".format(authorization) + @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) + def bad_version(self, request, authorization): + """Return version result that violates the expected schema.""" + request.setHeader("content-type", CBOR_MIME_TYPE) + return dumps({"garbage": 123}) + def result_of(d): """ @@ -257,15 +265,15 @@ def result_of(d): ) -class RoutingTests(SyncTestCase): +class CustomHTTPServerTests(SyncTestCase): """ - Tests for the HTTP routing infrastructure. + Tests that use a custom HTTP server. """ def setUp(self): if PY2: self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() + super(CustomHTTPServerTests, self).setUp() # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -277,8 +285,8 @@ class RoutingTests(SyncTestCase): def test_authorization_enforcement(self): """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. + The requirement for secrets is enforced by the ``_authorized_route`` + decorator; if they are not given, a 400 response code is returned. """ # Without secret, get a 400 error. response = result_of( @@ -298,6 +306,14 @@ class RoutingTests(SyncTestCase): self.assertEqual(response.code, 200) self.assertEqual(result_of(response.content()), b"GOOD SECRET") + def test_client_side_schema_validation(self): + """ + The client validates returned CBOR message against a schema. + """ + client = StorageClientGeneral(self.client) + with self.assertRaises(CDDLValidationError): + result_of(client.get_version()) + class HttpTestFixture(Fixture): """ @@ -413,7 +429,7 @@ class GenericHTTPAPITests(SyncTestCase): ) self.assertEqual(version, expected_version) - def test_schema_validation(self): + def test_server_side_schema_validation(self): """ Ensure that schema validation is happening: invalid CBOR should result in bad request response code (error 400). From f19bf8cf86d58457be4e7b4726626f3de5f29dc3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 11 Apr 2022 15:04:55 -0400 Subject: [PATCH 582/916] Parameterize the options object to the `run_cli` helper --- src/allmydata/test/common_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index d2d20916d..e63c3eef8 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -69,6 +69,9 @@ def run_cli_native(verb, *args, **kwargs): Most code should prefer ``run_cli_unicode`` which deals with all the necessary encoding considerations. + :param runner.Options options: The options instance to use to parse the + given arguments. + :param native_str verb: The command to run. For example, ``"create-node"``. @@ -88,6 +91,7 @@ def run_cli_native(verb, *args, **kwargs): matching native behavior. If True, stdout/stderr are returned as bytes. """ + options = kwargs.pop("options", runner.Options()) nodeargs = kwargs.pop("nodeargs", []) encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8" return_bytes = kwargs.pop("return_bytes", False) @@ -134,7 +138,7 @@ def run_cli_native(verb, *args, **kwargs): d.addCallback( partial( runner.parse_or_exit, - runner.Options(), + options, ), stdout=stdout, stderr=stderr, From dffcdf28543ced3bd214370bccbb5b78354d587d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 11 Apr 2022 15:05:32 -0400 Subject: [PATCH 583/916] Clean up the Py2/Py3 boilerplate --- src/allmydata/test/cli/test_invite.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 20d012995..749898b77 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -1,24 +1,12 @@ """ -Ported to Pythn 3. +Tests for ``tahoe invite``. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import mock import json from os.path import join - -try: - from typing import Optional, Sequence -except ImportError: - pass +from typing import Optional, Sequence from twisted.trial import unittest from twisted.internet import defer From bc6dafa999fa9b7d2af8dbab36ed532b74919e6b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 11:01:04 -0400 Subject: [PATCH 584/916] Replace monkey-patching of wormhole with a parameter to run_cli --- src/allmydata/scripts/create_node.py | 5 +- src/allmydata/scripts/runner.py | 6 + src/allmydata/scripts/tahoe_invite.py | 6 +- src/allmydata/test/cli/test_invite.py | 508 +++++++++++++--------- src/allmydata/test/cli/wormholetesting.py | 304 +++++++++++++ 5 files changed, 614 insertions(+), 215 deletions(-) create mode 100644 src/allmydata/test/cli/wormholetesting.py diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 4959ed391..5d9da518b 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -37,9 +37,6 @@ from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json -from wormhole import wormhole - - dummy_tac = """ import sys print("Nodes created by Tahoe-LAFS v1.11.0 or later cannot be run by") @@ -377,7 +374,7 @@ def _get_config_via_wormhole(config): relay_url = config.parent['wormhole-server'] print("Connecting to '{}'".format(relay_url), file=out) - wh = wormhole.create( + wh = config.parent.wormhole.create( appid=config.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 145ee6464..a0d8a752b 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -58,11 +58,17 @@ process_control_commands = [ class Options(usage.Options): + """ + :ivar wormhole: An object exposing the magic-wormhole API (mainly a test + hook). + """ # unit tests can override these to point at StringIO instances stdin = sys.stdin stdout = sys.stdout stderr = sys.stderr + from wormhole import wormhole + subCommands = ( create_node.subCommands + admin.subCommands + process_control_commands diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 09d4cbd59..c5f08f588 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -18,8 +18,6 @@ except ImportError: from twisted.python import usage from twisted.internet import defer, reactor -from wormhole import wormhole - from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import jsonbytes as json from allmydata.scripts.common import get_default_nodedir, get_introducer_furl @@ -44,13 +42,15 @@ class InviteOptions(usage.Options): self['nick'] = args[0].strip() +wormhole = None + @defer.inlineCallbacks def _send_config_via_wormhole(options, config): out = options.stdout err = options.stderr relay_url = options.parent['wormhole-server'] print("Connecting to '{}'...".format(relay_url), file=out) - wh = wormhole.create( + wh = options.parent.wormhole.create( appid=options.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 749898b77..c4bb6fd7e 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -2,62 +2,21 @@ Tests for ``tahoe invite``. """ -import os -import mock import json +import os from os.path import join from typing import Optional, Sequence -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest + +from ...client import read_config +from ...scripts import runner +from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from ...client import ( - read_config, -) - -class _FakeWormhole(object): - - def __init__(self, outgoing_messages): - self.messages = [] - for o in outgoing_messages: - assert isinstance(o, bytes) - self._outgoing = outgoing_messages - - def get_code(self): - return defer.succeed(u"6-alarmist-tuba") - - def set_code(self, code): - self._code = code - - def get_welcome(self): - return defer.succeed( - { - u"welcome": {}, - } - ) - - def allocate_code(self): - return None - - def send_message(self, msg): - assert isinstance(msg, bytes) - self.messages.append(msg) - - def get_message(self): - return defer.succeed(self._outgoing.pop(0)) - - def close(self): - return defer.succeed(None) - - -def _create_fake_wormhole(outgoing_messages): - outgoing_messages = [ - m.encode("utf-8") if isinstance(m, str) else m - for m in outgoing_messages - ] - return _FakeWormhole(outgoing_messages) +from .wormholetesting import MemoryWormholeServer, memory_server class Join(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -74,41 +33,52 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): successfully join after an invite """ node_dir = self.mktemp() + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v1": {}}}), - json.dumps({ - u"shares-needed": 1, - u"shares-happy": 1, - u"shares-total": 1, - u"nickname": u"somethinghopefullyunique", - u"introducer": u"pb://foo", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v1": {}}}, + { + u"shares-needed": 1, + u"shares-happy": 1, + u"shares-total": 1, + u"nickname": u"somethinghopefullyunique", + u"introducer": u"pb://foo", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) - rc, out, err = yield run_cli( - "create-client", - "--join", "1-abysmal-ant", - node_dir, - ) + rc, out, err = yield run_cli( + "create-client", + "--join", code, + node_dir, + options=options, + ) - self.assertEqual(0, rc) + self.assertEqual(0, rc) - config = read_config(node_dir, u"") - self.assertIn( - "pb://foo", - set( - furl - for (furl, cache) - in config.get_introducer_configuration().values() - ), - ) + config = read_config(node_dir, u"") + self.assertIn( + "pb://foo", + set( + furl + for (furl, cache) + in config.get_introducer_configuration().values() + ), + ) - with open(join(node_dir, 'tahoe.cfg'), 'r') as f: - config = f.read() - self.assertIn(u"somethinghopefullyunique", config) + with open(join(node_dir, 'tahoe.cfg'), 'r') as f: + config = f.read() + self.assertIn(u"somethinghopefullyunique", config) @defer.inlineCallbacks def test_create_node_illegal_option(self): @@ -116,30 +86,41 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): Server sends JSON with unknown/illegal key """ node_dir = self.mktemp() + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v1": {}}}), - json.dumps({ - u"shares-needed": 1, - u"shares-happy": 1, - u"shares-total": 1, - u"nickname": u"somethinghopefullyunique", - u"introducer": u"pb://foo", - u"something-else": u"not allowed", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v1": {}}}, + { + u"shares-needed": 1, + u"shares-happy": 1, + u"shares-total": 1, + u"nickname": u"somethinghopefullyunique", + u"introducer": u"pb://foo", + u"something-else": u"not allowed", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) - rc, out, err = yield run_cli( - "create-client", - "--join", "1-abysmal-ant", - node_dir, - ) + rc, out, err = yield run_cli( + "create-client", + "--join", code, + node_dir, + options=options, + ) - # should still succeed -- just ignores the not-whitelisted - # "something-else" option - self.assertEqual(0, rc) + # should still succeed -- just ignores the not-whitelisted + # "something-else" option + self.assertEqual(0, rc) class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -156,7 +137,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - def _invite_success(self, extra_args=(), tahoe_config=None): + async def _invite_success(self, extra_args=(), tahoe_config=None): # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred """ Exercise an expected-success case of ``tahoe invite``. @@ -178,53 +159,82 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): with open(join(intro_dir, "tahoe.cfg"), "wb") as fobj_cfg: fobj_cfg.write(tahoe_config) - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v1": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - extra_args = tuple(extra_args) - - d = run_cli( + async def server(): + # Run the server side of the invitation process using the CLI. + rc, out, err = await run_cli( "-d", intro_dir, "invite", - *(extra_args + ("foo",)) + *tuple(extra_args) + ("foo",), + options=options, ) - def done(result): - rc, out, err = result - self.assertEqual(2, len(fake_wh.messages)) - self.assertEqual( - json.loads(fake_wh.messages[0]), + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send a proper client abilities message. + other_end.send_message(dumps_bytes({u"abilities": {u"client-v1": {}}})) + + # Check the server's messages. First, it should announce its + # abilities correctly. + server_abilities = json.loads(await other_end.when_received()) + self.assertEqual( + server_abilities, + { + "abilities": { - "abilities": - { - "server-v1": {} - }, + "server-v1": {} }, - ) - invite = json.loads(fake_wh.messages[1]) - self.assertEqual( - invite["nickname"], "foo", - ) - self.assertEqual( - invite["introducer"], "pb://fooblam", - ) - return invite - d.addCallback(done) - return d + }, + ) + + # Second, it should have an invitation with a nickname and + # introducer furl. + invite = json.loads(await other_end.when_received()) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) + return invite + + invite, _ = await defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + return invite + @defer.inlineCallbacks def test_invite_success(self): """ successfully send an invite """ - invite = yield self._invite_success(( + invite = yield defer.Deferred.fromCoroutine(self._invite_success(( "--shares-needed", "1", "--shares-happy", "2", "--shares-total", "3", - )) + ))) self.assertEqual( invite["shares-needed"], "1", ) @@ -241,12 +251,12 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): If ``--shares-{needed,happy,total}`` are not given on the command line then the invitation is generated using the configured values. """ - invite = yield self._invite_success(tahoe_config=b""" + invite = yield defer.Deferred.fromCoroutine(self._invite_success(tahoe_config=b""" [client] shares.needed = 2 shares.happy = 4 shares.total = 6 -""") +""")) self.assertEqual( invite["shares-needed"], "2", ) @@ -265,22 +275,20 @@ shares.total = 6 """ intro_dir = os.path.join(self.basedir, "introducer") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v1": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + options = runner.Options() + options.wormhole = None - rc, out, err = yield run_cli( - "-d", intro_dir, - "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - "foo", - ) - self.assertNotEqual(rc, 0) - self.assertIn(u"Can't find introducer FURL", out + err) + rc, out, err = yield run_cli( + "-d", intro_dir, + "invite", + "--shares-needed", "1", + "--shares-happy", "1", + "--shares-total", "1", + "foo", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn(u"Can't find introducer FURL", out + err) @defer.inlineCallbacks def test_invite_wrong_client_abilities(self): @@ -294,23 +302,51 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v9000": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( + async def server(): + rc, out, err = await run_cli( "-d", intro_dir, "invite", "--shares-needed", "1", "--shares-happy", "1", "--shares-total", "1", "foo", + options=options, ) self.assertNotEqual(rc, 0) self.assertIn(u"No 'client-v1' in abilities", out + err) + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send some surprising client abilities. + other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) + + yield defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + + @defer.inlineCallbacks def test_invite_no_client_abilities(self): """ @@ -323,23 +359,52 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( + async def server(): + # Run the server side of the invitation process using the CLI. + rc, out, err = await run_cli( "-d", intro_dir, "invite", "--shares-needed", "1", "--shares-happy", "1", "--shares-total", "1", "foo", + options=options, ) self.assertNotEqual(rc, 0) self.assertIn(u"No 'abilities' from client", out + err) + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send a no-abilities message through to the server. + other_end.send_message(dumps_bytes({})) + + yield defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + + @defer.inlineCallbacks def test_invite_wrong_server_abilities(self): """ @@ -352,26 +417,38 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v9000": {}}}), - json.dumps({ - "shares-needed": "1", - "shares-total": "1", - "shares-happy": "1", - "nickname": "foo", - "introducer": "pb://fooblam", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( - "create-client", - "--join", "1-alarmist-tuba", - "foo", - ) - self.assertNotEqual(rc, 0) - self.assertIn("Expected 'server-v1' in server abilities", out + err) + wormhole = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v9000": {}}}, + { + "shares-needed": "1", + "shares-total": "1", + "shares-happy": "1", + "nickname": "foo", + "introducer": "pb://fooblam", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + rc, out, err = yield run_cli( + "create-client", + "--join", code, + "foo", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn("Expected 'server-v1' in server abilities", out + err) @defer.inlineCallbacks def test_invite_no_server_abilities(self): @@ -385,26 +462,38 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({}), - json.dumps({ - "shares-needed": "1", - "shares-total": "1", - "shares-happy": "1", - "nickname": "bar", - "introducer": "pb://fooblam", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - rc, out, err = yield run_cli( - "create-client", - "--join", "1-alarmist-tuba", - "bar", - ) - self.assertNotEqual(rc, 0) - self.assertIn("Expected 'abilities' in server introduction", out + err) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {}, + { + "shares-needed": "1", + "shares-total": "1", + "shares-happy": "1", + "nickname": "bar", + "introducer": "pb://fooblam", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + rc, out, err = yield run_cli( + "create-client", + "--join", code, + "bar", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn("Expected 'abilities' in server introduction", out + err) @defer.inlineCallbacks def test_invite_no_nick(self): @@ -413,13 +502,16 @@ shares.total = 6 """ intro_dir = os.path.join(self.basedir, "introducer") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole'): - rc, out, err = yield run_cli( - "-d", intro_dir, - "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - ) - self.assertTrue(rc) - self.assertIn(u"Provide a single argument", out + err) + options = runner.Options() + options.wormhole = None + + rc, out, err = yield run_cli( + "-d", intro_dir, + "invite", + "--shares-needed", "1", + "--shares-happy", "1", + "--shares-total", "1", + options=options, + ) + self.assertTrue(rc) + self.assertIn(u"Provide a single argument", out + err) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py new file mode 100644 index 000000000..b60980bff --- /dev/null +++ b/src/allmydata/test/cli/wormholetesting.py @@ -0,0 +1,304 @@ +""" +An in-memory implementation of some of the magic-wormhole interfaces for +use by automated tests. + +For example:: + + async def peerA(mw): + wormhole = mw.create("myapp", "wss://myserver", reactor) + code = await wormhole.get_code() + print(f"I have a code: {code}") + message = await wormhole.when_received() + print(f"I have a message: {message}") + + async def local_peerB(helper, mw): + peerA_wormhole = await helper.wait_for_wormhole("myapp", "wss://myserver") + code = await peerA_wormhole.when_code() + + peerB_wormhole = mw.create("myapp", "wss://myserver") + peerB_wormhole.set_code(code) + + peerB_wormhole.send_message("Hello, peer A") + + # Run peerA against local_peerB with pure in-memory message passing. + server, helper = memory_server() + run(gather(peerA(server), local_peerB(helper, server))) + + # Run peerA against a peerB somewhere out in the world, using a real + # wormhole relay server somewhere. + import wormhole + run(peerA(wormhole)) +""" + +from __future__ import annotations + +from typing import Iterator +from collections.abc import Awaitable +from inspect import getargspec +from itertools import count +from sys import stderr + +from attrs import frozen, define, field, Factory +from twisted.internet.defer import Deferred, DeferredQueue, succeed +from wormhole._interfaces import IWormhole +from wormhole.wormhole import create +from zope.interface import implementer + + +@define +class MemoryWormholeServer(object): + """ + A factory for in-memory wormholes. + + :ivar _apps: Wormhole state arranged by the application id and relay URL + it belongs to. + + :ivar _waiters: Observers waiting for a wormhole to be created for a + specific application id and relay URL combination. + """ + _apps: dict[tuple[str, str], _WormholeApp] = field(default=Factory(dict)) + _waiters: dict[tuple[str, str], Deferred] = field(default=Factory(dict)) + + def create( + self, + appid, + relay_url, + reactor, + versions={}, + delegate=None, + journal=None, + tor=None, + timing=None, + stderr=stderr, + _eventual_queue=None, + _enable_dilate=False, + ): + """ + Create a wormhole. It will be able to connect to other wormholes created + by this instance (and constrained by the normal appid/relay_url + rules). + """ + if tor is not None: + raise ValueError("Cannot deal with Tor right now.") + if _enable_dilate: + raise ValueError("Cannot deal with dilation right now.") + + key = (relay_url, appid) + wormhole = _MemoryWormhole(self._view(key)) + if key in self._waiters: + self._waiters.pop(key).callback(wormhole) + return wormhole + + def _view(self, key: tuple[str, str]) -> _WormholeServerView: + """ + Created a view onto this server's state that is limited by a certain + appid/relay_url pair. + """ + return _WormholeServerView(self, key) + + +@frozen +class TestingHelper(object): + """ + Provide extra functionality for interacting with an in-memory wormhole + implementation. + + This is intentionally a separate API so that it is not confused with + proper public interface of the real wormhole implementation. + """ + _server: MemoryWormholeServer + + async def wait_for_wormhole(self, appid: str, relay_url: str) -> IWormhole: + """ + Wait for a wormhole to appear at a specific location. + + :param appid: The appid that the resulting wormhole will have. + + :param relay_url: The URL of the relay at which the resulting wormhole + will presume to be created. + + :return: The first wormhole to be created which matches the given + parameters. + """ + key = relay_url, appid + if key in self._server._waiters: + raise ValueError(f"There is already a waiter for {key}") + d = Deferred() + self._server._waiters[key] = d + wormhole = await d + return wormhole + + +def _verify(): + """ + Roughly confirm that the in-memory wormhole creation function matches the + interface of the real implementation. + """ + # Poor man's interface verification. + + a = getargspec(create) + b = getargspec(MemoryWormholeServer.create) + # I know it has a `self` argument at the beginning. That's okay. + b = b._replace(args=b.args[1:]) + assert a == b, "{} != {}".format(a, b) + + +_verify() + + +@define +class _WormholeApp(object): + """ + Represent a collection of wormholes that belong to the same + appid/relay_url scope. + """ + wormholes: dict = field(default=Factory(dict)) + _waiting: dict = field(default=Factory(dict)) + _counter: Iterator[int] = field(default=Factory(count)) + + def allocate_code(self, wormhole, code): + """ + Allocate a new code for the given wormhole. + + This also associates the given wormhole with the code for future + lookup. + + Code generation logic is trivial and certainly not good enough for any + real use. It is sufficient for automated testing, though. + """ + if code is None: + code = "{}-persnickety-tardigrade".format(next(self._counter)) + self.wormholes.setdefault(code, []).append(wormhole) + try: + waiters = self._waiting.pop(code) + except KeyError: + pass + else: + for w in waiters: + w.callback(wormhole) + + return code + + def wait_for_wormhole(self, code: str) -> Awaitable[_MemoryWormhole]: + """ + Return a ``Deferred`` which fires with the next wormhole to be associated + with the given code. This is used to let the first end of a wormhole + rendezvous with the second end. + """ + d = Deferred() + self._waiting.setdefault(code, []).append(d) + return d + + +@frozen +class _WormholeServerView(object): + """ + Present an interface onto the server to be consumed by individual + wormholes. + """ + _server: MemoryWormholeServer + _key: tuple[str, str] + + def allocate_code(self, wormhole: _MemoryWormhole, code: str) -> str: + """ + Allocate a new code for the given wormhole in the scope associated with + this view. + """ + app = self._server._apps.setdefault(self._key, _WormholeApp()) + return app.allocate_code(wormhole, code) + + def wormhole_by_code(self, code, exclude): + """ + Retrieve all wormholes previously associated with a code. + """ + app = self._server._apps[self._key] + wormholes = app.wormholes[code] + try: + [wormhole] = list(wormhole for wormhole in wormholes if wormhole != exclude) + except ValueError: + return app.wait_for_wormhole(code) + return succeed(wormhole) + + +@implementer(IWormhole) +@define +class _MemoryWormhole(object): + """ + Represent one side of a wormhole as conceived by ``MemoryWormholeServer``. + """ + + _view: _WormholeServerView + _code: str = None + _payload: DeferredQueue = field(default=Factory(DeferredQueue)) + _waiting_for_code: list[Deferred] = field(default=Factory(list)) + _allocated: bool = False + + def allocate_code(self): + if self._code is not None: + raise ValueError( + "allocate_code used with a wormhole which already has a code" + ) + self._allocated = True + self._code = self._view.allocate_code(self, None) + waiters = self._waiting_for_code + self._waiting_for_code = None + for d in waiters: + d.callback(self._code) + + def set_code(self, code): + if self._code is None: + self._code = code + self._view.allocate_code(self, code) + else: + raise ValueError("set_code used with a wormhole which already has a code") + + def when_code(self): + if self._code is None: + d = Deferred() + self._waiting_for_code.append(d) + return d + return succeed(self._code) + + get_code = when_code + + def get_welcome(self): + return succeed("welcome") + + def send_message(self, payload): + self._payload.put(payload) + + def when_received(self): + if self._code is None: + raise ValueError( + "This implementation requires set_code or allocate_code " + "before when_received." + ) + d = self._view.wormhole_by_code(self._code, exclude=self) + + def got_wormhole(wormhole): + msg = wormhole._payload.get() + return msg + + d.addCallback(got_wormhole) + return d + + get_message = when_received + + def close(self): + pass + + # 0.9.2 compatibility + def get_code(self): + if self._code is None: + self.allocate_code() + return self.when_code() + + get = when_received + + +def memory_server() -> tuple[MemoryWormholeServer, TestingHelper]: + """ + Create a paired in-memory wormhole server and testing helper. + """ + server = MemoryWormholeServer() + return server, TestingHelper(server) From e35bab966351a7a24238430f25efb3dd24cdd89c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 11:01:35 -0400 Subject: [PATCH 585/916] news fragment --- newsfragments/3526.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3526.minor diff --git a/newsfragments/3526.minor b/newsfragments/3526.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3526.minor @@ -0,0 +1 @@ + From 1634f137be90eed8e7da7ce3964d45b7f0cc0651 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Apr 2022 12:54:16 -0400 Subject: [PATCH 586/916] Use sets more widely in the schema. --- docs/proposed/http-storage-node-protocol.rst | 3 +++ src/allmydata/storage/http_client.py | 7 ++++--- src/allmydata/storage/http_server.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 2ceb3c03a..be543abb2 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -350,6 +350,9 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. +For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and is hashable in Python) should be sent as a set. +Tag 6.258 is used to indicate sets in CBOR. + HTTP Design ~~~~~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3a758e592..e735a6369 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -39,8 +39,9 @@ class ClientException(Exception): # Schemas for server responses. # -# TODO usage of sets is inconsistent. Either use everywhere (and document in -# spec document) or use nowhere. +# Tags are of the form #6.nnn, where the number is documented at +# https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 +# indicates a set. _SCHEMAS = { "get_version": Schema( """ @@ -74,7 +75,7 @@ _SCHEMAS = { ), "list_shares": Schema( """ - message = [* uint] + message = #6.258([* uint]) """ ), } diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3876409b0..54e60f913 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -436,7 +436,7 @@ class HTTPServer(object): """ List shares for the given storage index. """ - share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) + share_numbers = set(self._storage_server.get_buckets(storage_index).keys()) return self._send_encoded(request, share_numbers) @_authorized_route( From b0fffabed0aa525610714864ed74f3dd6c7445e8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:10:02 -0400 Subject: [PATCH 587/916] remove unnecessary module-scope wormhole used this during testing so the other mock() calls wouldn't explode in a boring way --- src/allmydata/scripts/tahoe_invite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index c5f08f588..b62d6a463 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -42,8 +42,6 @@ class InviteOptions(usage.Options): self['nick'] = args[0].strip() -wormhole = None - @defer.inlineCallbacks def _send_config_via_wormhole(options, config): out = options.stdout From 71b5cd9e0d64643a907d365c8d9580384cb92d4b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:13:48 -0400 Subject: [PATCH 588/916] rewrite comment annotations with syntax --- src/allmydata/test/cli/test_invite.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index c4bb6fd7e..50f446ae2 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -137,8 +137,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - async def _invite_success(self, extra_args=(), tahoe_config=None): - # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred + async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[byte] = None) -> str: """ Exercise an expected-success case of ``tahoe invite``. From 0f61a1dab9e72152e938ea3e5279ed1a1ac65d9f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:33:11 -0400 Subject: [PATCH 589/916] Factor some duplication out of the test methods --- src/allmydata/test/cli/test_invite.py | 149 ++++++++++++-------------- 1 file changed, 70 insertions(+), 79 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 50f446ae2..5b4871944 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -4,8 +4,9 @@ Tests for ``tahoe invite``. import json import os +from functools import partial from os.path import join -from typing import Optional, Sequence +from typing import Awaitable, Callable, Optional, Sequence, TypeVar from twisted.internet import defer from twisted.trial import unittest @@ -16,7 +17,58 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import MemoryWormholeServer, memory_server +from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server + + +async def open_wormhole() -> tuple[Callable, IWormhole, str]: + """ + Create a new in-memory wormhole server, open one end of a wormhole, and + return it and related info. + + :return: A three-tuple allowing use of the wormhole. The first element is + a callable like ``run_cli`` but which will run commands so that they + use the in-memory wormhole server instead of a real one. The second + element is the open wormhole. The third element is the wormhole's + code. + """ + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() + + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = await wormhole.get_code() + + return (partial(run_cli, options=options), wormhole, code) + + +def send_messages(wormhole: IWormhole, messages: list[dict]) -> None: + """ + Send a list of message through a wormhole. + """ + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + +A = TypeVar("A") +B = TypeVar("B") + +def concurrently( + client: Callable[[], Awaitable[A]], + server: Callable[[], Awaitable[B]], +) -> defer.Deferred[tuple[A, B]]: + """ + Run two asynchronous functions concurrently and asynchronously return a + tuple of both their results. + """ + return defer.gatherResults([ + defer.Deferred.fromCoroutine(client()), + defer.Deferred.fromCoroutine(server()), + ]) class Join(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -33,18 +85,8 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): successfully join after an invite """ node_dir = self.mktemp() - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v1": {}}}, { u"shares-needed": 1, @@ -53,15 +95,12 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): u"nickname": u"somethinghopefullyunique", u"introducer": u"pb://foo", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, node_dir, - options=options, ) self.assertEqual(0, rc) @@ -86,18 +125,8 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): Server sends JSON with unknown/illegal key """ node_dir = self.mktemp() - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v1": {}}}, { u"shares-needed": 1, @@ -107,15 +136,12 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): u"introducer": u"pb://foo", u"something-else": u"not allowed", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, node_dir, - options=options, ) # should still succeed -- just ignores the not-whitelisted @@ -137,7 +163,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[byte] = None) -> str: + async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[bytes] = None) -> str: """ Exercise an expected-success case of ``tahoe invite``. @@ -217,10 +243,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): ) return invite - invite, _ = await defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + invite, _ = await concurrently(client, server) return invite @@ -340,10 +363,7 @@ shares.total = 6 # Send some surprising client abilities. other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) - yield defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + yield concurrently(client, server) @defer.inlineCallbacks @@ -398,10 +418,7 @@ shares.total = 6 # Send a no-abilities message through to the server. other_end.send_message(dumps_bytes({})) - yield defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + yield concurrently(client, server) @defer.inlineCallbacks @@ -416,18 +433,8 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - wormhole_server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = wormhole_server - reactor = object() - - wormhole = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v9000": {}}}, { "shares-needed": "1", @@ -436,15 +443,12 @@ shares.total = 6 "nickname": "foo", "introducer": "pb://fooblam", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, "foo", - options=options, ) self.assertNotEqual(rc, 0) self.assertIn("Expected 'server-v1' in server abilities", out + err) @@ -461,18 +465,8 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {}, { "shares-needed": "1", @@ -481,15 +475,12 @@ shares.total = 6 "nickname": "bar", "introducer": "pb://fooblam", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, "bar", - options=options, ) self.assertNotEqual(rc, 0) self.assertIn("Expected 'abilities' in server introduction", out + err) From 2e8c51ac4e9736134d40dcb2d55d117836504041 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 08:24:53 -0400 Subject: [PATCH 590/916] bump nixpkgs-21.11 and drop the special zfec handling the latest zfec release works fine without help --- default.nix | 14 -------------- nix/sources.json | 6 +++--- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/default.nix b/default.nix index 095c54578..5f4db2c78 100644 --- a/default.nix +++ b/default.nix @@ -86,24 +86,10 @@ mach-nix.buildPythonPackage rec { # There are some reasonable defaults so we only need to specify certain # packages where the default configuration runs into some issue. providers = { - # Through zfec 1.5.5 the wheel has an incorrect runtime dependency - # declared on argparse, not available for recent versions of Python 3. - # Force mach-nix to use the sdist instead. This allows us to apply a - # patch that removes the offending declaration. - zfec = "sdist"; }; # Define certain overrides to the way Python dependencies are built. _ = { - # Apply the argparse declaration fix to zfec sdist. - zfec.patches = with pkgs; [ - (fetchpatch { - name = "fix-argparse.patch"; - url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch"; - sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; - }) - ]; - # Remove a click-default-group patch for a test suite problem which no # longer applies because the project apparently no longer has a test suite # in its source distribution. diff --git a/nix/sources.json b/nix/sources.json index e0235a3fb..ced730ab7 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -41,10 +41,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", - "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", + "rev": "838eefb4f93f2306d4614aafb9b2375f315d917f", + "sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg", "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "pypi-deps-db": { From cb91012f982a6a2602d5e51737f855ab4394b796 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 08:25:11 -0400 Subject: [PATCH 591/916] bump the pypi db to a version including upcoming pycddl dependency --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index ced730ab7..79eabe7a1 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -53,10 +53,10 @@ "homepage": "", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "0f6de8bf1f186c275af862ec9667abb95aae8542", - "sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y", + "rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a", + "sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz", + "url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From 1c49b2375e6cd5f3d19e5d68b069f3e0bdbba135 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 08:31:49 -0400 Subject: [PATCH 592/916] news fragment --- newsfragments/3889.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3889.minor diff --git a/newsfragments/3889.minor b/newsfragments/3889.minor new file mode 100644 index 000000000..e69de29bb From 7aab039a00154885a6cd554131797e19092060fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Apr 2022 13:33:51 -0400 Subject: [PATCH 593/916] Improve docs. --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index be543abb2..b57e9056a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -350,8 +350,8 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. -For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and is hashable in Python) should be sent as a set. -Tag 6.258 is used to indicate sets in CBOR. +For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. +Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry `_ for more details. HTTP Design ~~~~~~~~~~~ From 10f79ce8aa8ac07637ffcfc0af3a9003ff2736ba Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 14:39:52 -0400 Subject: [PATCH 594/916] Use __future__.annotations in test_invite for generic builtins too --- src/allmydata/test/cli/test_invite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 5b4871944..7d2250bac 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -2,6 +2,8 @@ Tests for ``tahoe invite``. """ +from __future__ import annotations + import json import os from functools import partial From ec5be01f38ffdca34930b2301a206d1de1ecb047 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 14:50:38 -0400 Subject: [PATCH 595/916] more completely annotate types in the wormholetesting module --- src/allmydata/test/cli/wormholetesting.py | 51 ++++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index b60980bff..715e82236 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator +from typing import Iterator, Optional, Sequence from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -44,6 +44,11 @@ from wormhole._interfaces import IWormhole from wormhole.wormhole import create from zope.interface import implementer +WormholeCode = str +WormholeMessage = bytes +AppId = str +RelayURL = str +ApplicationKey = tuple[RelayURL, AppId] @define class MemoryWormholeServer(object): @@ -56,8 +61,8 @@ class MemoryWormholeServer(object): :ivar _waiters: Observers waiting for a wormhole to be created for a specific application id and relay URL combination. """ - _apps: dict[tuple[str, str], _WormholeApp] = field(default=Factory(dict)) - _waiters: dict[tuple[str, str], Deferred] = field(default=Factory(dict)) + _apps: dict[ApplicationKey, _WormholeApp] = field(default=Factory(dict)) + _waiters: dict[ApplicationKey, Deferred] = field(default=Factory(dict)) def create( self, @@ -89,7 +94,7 @@ class MemoryWormholeServer(object): self._waiters.pop(key).callback(wormhole) return wormhole - def _view(self, key: tuple[str, str]) -> _WormholeServerView: + def _view(self, key: ApplicationKey) -> _WormholeServerView: """ Created a view onto this server's state that is limited by a certain appid/relay_url pair. @@ -108,7 +113,7 @@ class TestingHelper(object): """ _server: MemoryWormholeServer - async def wait_for_wormhole(self, appid: str, relay_url: str) -> IWormhole: + async def wait_for_wormhole(self, appid: AppId, relay_url: RelayURL) -> IWormhole: """ Wait for a wormhole to appear at a specific location. @@ -120,7 +125,7 @@ class TestingHelper(object): :return: The first wormhole to be created which matches the given parameters. """ - key = relay_url, appid + key = (relay_url, appid) if key in self._server._waiters: raise ValueError(f"There is already a waiter for {key}") d = Deferred() @@ -152,11 +157,11 @@ class _WormholeApp(object): Represent a collection of wormholes that belong to the same appid/relay_url scope. """ - wormholes: dict = field(default=Factory(dict)) - _waiting: dict = field(default=Factory(dict)) + wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, Sequence[Deferred]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) - def allocate_code(self, wormhole, code): + def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: """ Allocate a new code for the given wormhole. @@ -179,7 +184,7 @@ class _WormholeApp(object): return code - def wait_for_wormhole(self, code: str) -> Awaitable[_MemoryWormhole]: + def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]: """ Return a ``Deferred`` which fires with the next wormhole to be associated with the given code. This is used to let the first end of a wormhole @@ -197,9 +202,9 @@ class _WormholeServerView(object): wormholes. """ _server: MemoryWormholeServer - _key: tuple[str, str] + _key: ApplicationKey - def allocate_code(self, wormhole: _MemoryWormhole, code: str) -> str: + def allocate_code(self, wormhole: _MemoryWormhole, code: Optional[WormholeCode]) -> WormholeCode: """ Allocate a new code for the given wormhole in the scope associated with this view. @@ -207,7 +212,7 @@ class _WormholeServerView(object): app = self._server._apps.setdefault(self._key, _WormholeApp()) return app.allocate_code(wormhole, code) - def wormhole_by_code(self, code, exclude): + def wormhole_by_code(self, code: WormholeCode, exclude: object) -> Deferred[IWormhole]: """ Retrieve all wormholes previously associated with a code. """ @@ -228,46 +233,42 @@ class _MemoryWormhole(object): """ _view: _WormholeServerView - _code: str = None + _code: Optional[WormholeCode] = None _payload: DeferredQueue = field(default=Factory(DeferredQueue)) _waiting_for_code: list[Deferred] = field(default=Factory(list)) - _allocated: bool = False - def allocate_code(self): + def allocate_code(self) -> None: if self._code is not None: raise ValueError( "allocate_code used with a wormhole which already has a code" ) - self._allocated = True self._code = self._view.allocate_code(self, None) waiters = self._waiting_for_code self._waiting_for_code = None for d in waiters: d.callback(self._code) - def set_code(self, code): + def set_code(self, code: WormholeCode) -> None: if self._code is None: self._code = code self._view.allocate_code(self, code) else: raise ValueError("set_code used with a wormhole which already has a code") - def when_code(self): + def when_code(self) -> Deferred[WormholeCode]: if self._code is None: d = Deferred() self._waiting_for_code.append(d) return d return succeed(self._code) - get_code = when_code - def get_welcome(self): return succeed("welcome") - def send_message(self, payload): + def send_message(self, payload: WormholeMessage) -> None: self._payload.put(payload) - def when_received(self): + def when_received(self) -> Deferred[WormholeMessage]: if self._code is None: raise ValueError( "This implementation requires set_code or allocate_code " @@ -284,11 +285,11 @@ class _MemoryWormhole(object): get_message = when_received - def close(self): + def close(self) -> None: pass # 0.9.2 compatibility - def get_code(self): + def get_code(self) -> Deferred[WormholeCode]: if self._code is None: self.allocate_code() return self.when_code() From 38e1e93a75356a9275f7fc6b23bd998bc55d012e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 15:42:10 -0400 Subject: [PATCH 596/916] factor the duplicate client logic out --- src/allmydata/test/cli/test_invite.py | 157 +++++++++++--------------- 1 file changed, 69 insertions(+), 88 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 7d2250bac..9f2607433 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -8,7 +8,7 @@ import json import os from functools import partial from os.path import join -from typing import Awaitable, Callable, Optional, Sequence, TypeVar +from typing import Awaitable, Callable, Optional, Sequence, TypeVar, Union from twisted.internet import defer from twisted.trial import unittest @@ -21,6 +21,12 @@ from ..no_network import GridTestMixin from .common import CLITestMixin from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server +# Logically: +# JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] +# +# But practically: +JSONable = Union[dict, None, int, float, str, list] + async def open_wormhole() -> tuple[Callable, IWormhole, str]: """ @@ -48,7 +54,42 @@ async def open_wormhole() -> tuple[Callable, IWormhole, str]: return (partial(run_cli, options=options), wormhole, code) -def send_messages(wormhole: IWormhole, messages: list[dict]) -> None: +def make_simple_peer( + reactor, + server: MemoryWormholeServer, + helper: TestingHelper, + messages: Sequence[JSONable], +) -> Callable[[], Awaitable[IWormhole]]: + """ + Make a wormhole peer that just sends the given messages. + + The returned function returns an awaitable that fires with the peer's end + of the wormhole. + """ + async def peer() -> IWormhole: + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + send_messages(other_end, messages) + return other_end + + return peer + + +def send_messages(wormhole: IWormhole, messages: Sequence[JSONable]) -> None: """ Send a list of message through a wormhole. """ @@ -200,55 +241,34 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): options=options, ) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. + # Send a proper client abilities message. + client = make_simple_peer(reactor, wormhole_server, helper, [{u"abilities": {u"client-v1": {}}}]) + other_end, _ = await concurrently(client, server) - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send a proper client abilities message. - other_end.send_message(dumps_bytes({u"abilities": {u"client-v1": {}}})) - - # Check the server's messages. First, it should announce its - # abilities correctly. - server_abilities = json.loads(await other_end.when_received()) - self.assertEqual( - server_abilities, + # Check the server's messages. First, it should announce its + # abilities correctly. + server_abilities = json.loads(await other_end.when_received()) + self.assertEqual( + server_abilities, + { + "abilities": { - "abilities": - { - "server-v1": {} - }, + "server-v1": {} }, - ) + }, + ) - # Second, it should have an invitation with a nickname and - # introducer furl. - invite = json.loads(await other_end.when_received()) - self.assertEqual( - invite["nickname"], "foo", - ) - self.assertEqual( - invite["introducer"], "pb://fooblam", - ) - return invite - - invite, _ = await concurrently(client, server) + # Second, it should have an invitation with a nickname and introducer + # furl. + invite = json.loads(await other_end.when_received()) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) return invite - @defer.inlineCallbacks def test_invite_success(self): """ @@ -344,30 +364,10 @@ shares.total = 6 self.assertNotEqual(rc, 0) self.assertIn(u"No 'client-v1' in abilities", out + err) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. - - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send some surprising client abilities. - other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) - + # Send some surprising client abilities. + client = make_simple_peer(reactor, wormhole_server, helper, [{u"abilities": {u"client-v9000": {}}}]) yield concurrently(client, server) - @defer.inlineCallbacks def test_invite_no_client_abilities(self): """ @@ -399,27 +399,8 @@ shares.total = 6 self.assertNotEqual(rc, 0) self.assertIn(u"No 'abilities' from client", out + err) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. - - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send a no-abilities message through to the server. - other_end.send_message(dumps_bytes({})) - + # Send a no-abilities message through to the server. + client = make_simple_peer(reactor, wormhole_server, helper, [{}]) yield concurrently(client, server) From 03674bd4526ce2374f5077d50cd6da603e78f48a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 16:01:32 -0400 Subject: [PATCH 597/916] use Tuple for type alias __future__.annotations only fixes py37/generic builtins in annotations syntax, not arbitrary expressions --- src/allmydata/test/cli/wormholetesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 715e82236..0cee78a5a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, Sequence +from typing import Iterator, Optional, Sequence, Tuple from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -48,7 +48,7 @@ WormholeCode = str WormholeMessage = bytes AppId = str RelayURL = str -ApplicationKey = tuple[RelayURL, AppId] +ApplicationKey = Tuple[RelayURL, AppId] @define class MemoryWormholeServer(object): From f34e01649df46753cf4f129293b7b2fb2506a757 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 18:35:18 -0400 Subject: [PATCH 598/916] some more fixes for mypy --- src/allmydata/test/cli/test_invite.py | 2 +- src/allmydata/test/cli/wormholetesting.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 9f2607433..07756eeed 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -19,7 +19,7 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server +from .wormholetesting import IWormhole, MemoryWormholeServer, TestingHelper, memory_server # Logically: # JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 0cee78a5a..744f9d75a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, Sequence, Tuple +from typing import Iterator, Optional, List, Tuple from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -158,7 +158,7 @@ class _WormholeApp(object): appid/relay_url scope. """ wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) - _waiting: dict[WormholeCode, Sequence[Deferred]] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, List[Deferred]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: @@ -244,7 +244,7 @@ class _MemoryWormhole(object): ) self._code = self._view.allocate_code(self, None) waiters = self._waiting_for_code - self._waiting_for_code = None + self._waiting_for_code = [] for d in waiters: d.callback(self._code) From 2bc8cdf852f821ab264e442ec857b2426ff87724 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Apr 2022 11:40:19 -0400 Subject: [PATCH 599/916] Drop Python 2. --- src/allmydata/test/test_storage_http.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index cf1504f58..df781012e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -2,18 +2,6 @@ Tests for HTTP storage client + server. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from base64 import b64encode from contextlib import contextmanager from os import urandom @@ -108,11 +96,6 @@ class ExtractSecretsTests(SyncTestCase): Tests for ``_extract_secrets``. """ - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ExtractSecretsTests, self).setUp() - @given(secrets_to_send=SECRETS_STRATEGY) def test_extract_secrets(self, secrets_to_send): """ @@ -271,8 +254,6 @@ class CustomHTTPServerTests(SyncTestCase): """ def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") super(CustomHTTPServerTests, self).setUp() # Could be a fixture, but will only be used in this test class so not # going to bother: @@ -374,8 +355,6 @@ class GenericHTTPAPITests(SyncTestCase): """ def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) @@ -466,8 +445,6 @@ class ImmutableHTTPAPITests(SyncTestCase): """ def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) self.imm_client = StorageClientImmutables(self.http.client) From 9db5a397e1d88929b775c464042fa7ffcb736501 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Apr 2022 11:45:47 -0400 Subject: [PATCH 600/916] Minor type annotation improvements. --- src/allmydata/storage/http_client.py | 4 +++- src/allmydata/storage/http_common.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6ff462d73..9a2774aef 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,6 +2,8 @@ HTTP client that talks to the HTTP storage server. """ +from __future__ import annotations + from typing import Union, Set, Optional from base64 import b64encode @@ -214,7 +216,7 @@ class StorageClient(object): self._treq = treq @classmethod - def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> "StorageClient": + def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index bd88f9fae..addd926d1 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -27,7 +27,7 @@ def get_content_type(headers: Headers) -> Optional[str]: return content_type -def swissnum_auth_header(swissnum): # type: (bytes) -> bytes +def swissnum_auth_header(swissnum: bytes) -> bytes: """Return value for ``Authentication`` header.""" return b"Tahoe-LAFS " + b64encode(swissnum).strip() From 4e0f912a100bbee6e684fd830bb4d0510a2545e6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Apr 2022 11:52:20 -0400 Subject: [PATCH 601/916] Comply with license. --- src/allmydata/storage/http_client.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9a2774aef..65bcd1c4b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -134,7 +134,28 @@ class _TLSContextFactory(CertificateOptions): Create a context that validates the way Tahoe-LAFS wants to: based on a pinned certificate hash, rather than a certificate authority. - Originally implemented as part of Foolscap. + Originally implemented as part of Foolscap. To comply with the license, + here's the original licensing terms: + + Copyright (c) 2006-2008 Brian Warner + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. """ def __init__(self, expected_spki_hash: bytes): From fc2807cccc773386b83e53d7eb02ee02ef326ba9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:08:16 -0400 Subject: [PATCH 602/916] Sketch of server-side read-test-write endpoint. --- src/allmydata/storage/http_common.py | 1 + src/allmydata/storage/http_server.py | 60 ++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index addd926d1..123ce403b 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -38,6 +38,7 @@ class Secrets(Enum): LEASE_RENEW = "lease-renew-secret" LEASE_CANCEL = "lease-cancel-secret" UPLOAD = "upload-secret" + WRITE_ENABLER = "write-enabler" def get_spki_hash(certificate: Certificate) -> bytes: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7c4860d57..bcb4b22c9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -239,19 +239,39 @@ class _HTTPError(Exception): # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. _SCHEMAS = { - "allocate_buckets": Schema(""" - message = { + "allocate_buckets": Schema( + """ + request = { share-numbers: #6.258([* uint]) allocated-size: uint } - """), - "advise_corrupt_share": Schema(""" - message = { + """ + ), + "advise_corrupt_share": Schema( + """ + request = { reason: tstr } - """) + """ + ), + "mutable_read_test_write": Schema( + """ + request = { + "test-write-vectors": { + * share_number: { + "test": [* {"offset": uint, "size": uint, "specimen": bstr}] + "write": [* {"offset": uint, "data": bstr}] + "new-length": uint + } + } + "read-vector": [* {"offset": uint, "size": uint}] + } + share_number = uint + """ + ), } + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -537,7 +557,9 @@ class HTTPServer(object): "/v1/immutable///corrupt", methods=["POST"], ) - def advise_corrupt_share(self, request, authorization, storage_index, share_number): + def advise_corrupt_share_immutable( + self, request, authorization, storage_index, share_number + ): """Indicate that given share is corrupt, with a text reason.""" try: bucket = self._storage_server.get_buckets(storage_index)[share_number] @@ -548,6 +570,30 @@ class HTTPServer(object): bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" + ##### Immutable APIs ##### + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.WRITE_ENABLER}, + "/v1/mutable//read-test-write", + methods=["POST"], + ) + def mutable_read_test_write(self, request, authorization, storage_index): + """Read/test/write combined operation for mutables.""" + rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) + secrets = ( + authorization[Secrets.WRITE_ENABLER], + authorization[Secrets.LEASE_RENEW], + authorization[Secrets.LEASE_CANCEL], + ) + success, read_data = self._storage_server.slot_testv_and_readv_and_writev( + storage_index, + secrets, + rtw_request["test-write-vectors"], + rtw_request["read-vectors"], + ) + return self._send_encoded(request, {"success": success, "data": read_data}) + @implementer(IStreamServerEndpoint) @attr.s From 58bd38120294a6709fc74190188d6ae4ae74a03b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:19:30 -0400 Subject: [PATCH 603/916] Switch to newer attrs API. --- src/allmydata/storage/http_client.py | 42 +++++++++++++--------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 65bcd1c4b..57ca4dae9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -8,7 +8,7 @@ from typing import Union, Set, Optional from base64 import b64encode -import attr +from attrs import define # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -121,12 +121,12 @@ def _decode_cbor(response, schema: Schema): ) -@attr.s +@define class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" - already_have = attr.ib(type=Set[int]) - allocated = attr.ib(type=Set[int]) + already_have: Set[int] + allocated: Set[int] class _TLSContextFactory(CertificateOptions): @@ -200,14 +200,14 @@ class _TLSContextFactory(CertificateOptions): @implementer(IPolicyForHTTPS) @implementer(IOpenSSLClientConnectionCreator) -@attr.s +@define class _StorageClientHTTPSPolicy: """ A HTTPS policy that ensures the SPKI hash of the public key matches a known hash, i.e. pinning-based validation. """ - expected_spki_hash = attr.ib(type=bytes) + expected_spki_hash: bytes # IPolicyForHTTPS def creatorForNetloc(self, hostname, port): @@ -220,24 +220,22 @@ class _StorageClientHTTPSPolicy: ) +@define class StorageClient(object): """ Low-level HTTP client that talks to the HTTP storage server. """ - def __init__( - self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq,HTTPClient]) -> None - """ - The URL is a HTTPS URL ("https://..."). To construct from a NURL, use - ``StorageClient.from_nurl()``. - """ - self._base_url = url - self._swissnum = swissnum - self._treq = treq + # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use + # ``StorageClient.from_nurl()``. + _base_url: DecodedURL + _swissnum: bytes + _treq: Union[treq, StubTreq, HTTPClient] @classmethod - def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> StorageClient: + def from_nurl( + cls, nurl: DecodedURL, reactor, persistent: bool = True + ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. @@ -342,25 +340,25 @@ class StorageClientGeneral(object): returnValue(decoded_response) -@attr.s +@define class UploadProgress(object): """ Progress of immutable upload, per the server. """ # True when upload has finished. - finished = attr.ib(type=bool) + finished: bool # Remaining ranges to upload. - required = attr.ib(type=RangeMap) + required: RangeMap +@define class StorageClientImmutables(object): """ APIs for interacting with immutables. """ - def __init__(self, client: StorageClient): - self._client = client + _client: StorageClient @inlineCallbacks def create( From 186aa9abc4715ef42f7d7d1c7ecd95884bdeab45 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:32:15 -0400 Subject: [PATCH 604/916] Make the utility reusable. --- src/allmydata/test/test_deferredutil.py | 28 +++++++++++++++++++ src/allmydata/test/test_storage_https.py | 17 +----------- src/allmydata/util/deferredutil.py | 35 +++++++++++++----------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index 2a155089f..a37dfdd6f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -129,3 +129,31 @@ class UntilTests(unittest.TestCase): self.assertEqual([1], counter) r1.callback(None) self.assertEqual([2], counter) + + +class AsyncToDeferred(unittest.TestCase): + """Tests for ``deferredutil.async_to_deferred.``""" + + def test_async_to_deferred_success(self): + """ + Normal results from a ``@async_to_deferred``-wrapped function get + turned into a ``Deferred`` with that value. + """ + @deferredutil.async_to_deferred + async def f(x, y): + return x + y + + result = f(1, y=2) + self.assertEqual(self.successResultOf(result), 3) + + def test_async_to_deferred_exception(self): + """ + Exceptions from a ``@async_to_deferred``-wrapped function get + turned into a ``Deferred`` with that value. + """ + @deferredutil.async_to_deferred + async def f(x, y): + return x/y + + result = f(1, 0) + self.assertIsInstance(self.failureResultOf(result).value, ZeroDivisionError) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 73c99725a..3b41e8308 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -6,14 +6,12 @@ server authentication logic, which may one day apply outside of HTTP Storage Protocol. """ -from functools import wraps from contextlib import asynccontextmanager from cryptography import x509 from twisted.internet.endpoints import serverFromString from twisted.internet import reactor -from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data @@ -31,6 +29,7 @@ from .certs import ( from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper +from ..util.deferredutil import async_to_deferred class HTTPSNurlTests(SyncTestCase): @@ -73,20 +72,6 @@ ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG self.assertEqual(get_spki_hash(certificate), expected_hash) -def async_to_deferred(f): - """ - Wrap an async function to return a Deferred instead. - - Maybe solution to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3886 - """ - - @wraps(f) - def not_async(*args, **kwargs): - return Deferred.fromCoroutine(f(*args, **kwargs)) - - return not_async - - class PinningHTTPSValidation(AsyncTestCase): """ Test client-side validation logic of HTTPS certificates that uses diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index ed2a11ee4..782663e8b 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -4,24 +4,13 @@ Utilities for working with Twisted Deferreds. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time +from functools import wraps -try: - from typing import ( - Callable, - Any, - ) -except ImportError: - pass +from typing import ( + Callable, + Any, +) from foolscap.api import eventually from eliot.twisted import ( @@ -231,3 +220,17 @@ def until( yield action() if condition(): break + + +def async_to_deferred(f): + """ + Wrap an async function to return a Deferred instead. + + Maybe solution to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3886 + """ + + @wraps(f) + def not_async(*args, **kwargs): + return defer.Deferred.fromCoroutine(f(*args, **kwargs)) + + return not_async From 24548dee0b37184cef975d4febe9ca4425920266 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:56:06 -0400 Subject: [PATCH 605/916] Sketch of read/write APIs interface for mutables on client side. --- src/allmydata/storage/http_client.py | 134 +++++++++++++++++++++++++-- src/allmydata/storage/http_server.py | 2 +- 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 57ca4dae9..1bff34699 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,10 +5,10 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations from typing import Union, Set, Optional - +from enum import Enum from base64 import b64encode -from attrs import define +from attrs import define, field # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -39,6 +39,7 @@ from .http_common import ( ) from .common import si_b2a from ..util.hashutil import timing_safe_compare +from ..util.deferredutil import async_to_deferred _OPENSSL = Binding().lib @@ -64,7 +65,7 @@ class ClientException(Exception): _SCHEMAS = { "get_version": Schema( """ - message = {'http://allmydata.org/tahoe/protocols/storage/v1' => { + response = {'http://allmydata.org/tahoe/protocols/storage/v1' => { 'maximum-immutable-share-size' => uint 'maximum-mutable-share-size' => uint 'available-space' => uint @@ -79,7 +80,7 @@ _SCHEMAS = { ), "allocate_buckets": Schema( """ - message = { + response = { already-have: #6.258([* uint]) allocated: #6.258([* uint]) } @@ -87,16 +88,25 @@ _SCHEMAS = { ), "immutable_write_share_chunk": Schema( """ - message = { + response = { required: [* {begin: uint, end: uint}] } """ ), "list_shares": Schema( """ - message = #6.258([* uint]) + response = #6.258([* uint]) """ ), + "mutable_read_test_write": Schema( + """ + response = { + "success": bool, + "data": [* share_number: [* bstr]] + } + share_number = uint + """ + ), } @@ -571,3 +581,115 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) + + +@define +class WriteVector: + """Data to write to a chunk.""" + + offset: int + data: bytes + + +class TestVectorOperator(Enum): + """Possible operators for test vectors.""" + + LT = b"lt" + LE = b"le" + EQ = b"eq" + NE = b"ne" + GE = b"ge" + GT = b"gt" + + +@define +class TestVector: + """Checks to make on a chunk before writing to it.""" + + offset: int + size: int + operator: TestVectorOperator = field(default=TestVectorOperator.EQ) + specimen: bytes + + +@define +class ReadVector: + """ + Reads to do on chunks, as part of a read/test/write operation. + """ + + offset: int + size: int + + +@define +class TestWriteVectors: + """Test and write vectors for a specific share.""" + + test_vectors: list[TestVector] + write_vectors: list[WriteVector] + new_length: Optional[int] = field(default=None) + + +@define +class ReadTestWriteResult: + """Result of sending read-test-write vectors.""" + + success: bool + # Map share numbers to reads corresponding to the request's list of + # ReadVectors: + reads: dict[int, list[bytes]] + + +@define +class StorageClientMutables: + """ + APIs for interacting with mutables. + """ + + _client: StorageClient + + @async_to_deferred + async def read_test_write_chunks( + storage_index: bytes, + write_enabled_secret: bytes, + lease_renew_secret: bytes, + lease_cancel_secret: bytes, + testwrite_vectors: dict[int, TestWriteVectors], + read_vector: list[ReadVector], + ) -> ReadTestWriteResult: + """ + Read, test, and possibly write chunks to a particular mutable storage + index. + + Reads are done before writes. + + Given a mapping between share numbers and test/write vectors, the tests + are done and if they are valid the writes are done. + """ + pass + + @async_to_deferred + async def read_share_chunk( + self, + storage_index: bytes, + share_number: int, + # TODO is this really optional? + # TODO if yes, test non-optional variants + offset: Optional[int], + length: Optional[int], + ) -> bytes: + """ + Download a chunk of data from a share. + + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bcb4b22c9..7f279580b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -261,7 +261,7 @@ _SCHEMAS = { * share_number: { "test": [* {"offset": uint, "size": uint, "specimen": bstr}] "write": [* {"offset": uint, "data": bstr}] - "new-length": uint + "new-length": (uint // null) } } "read-vector": [* {"offset": uint, "size": uint}] From b0d547ee53540649e18ceb437b7508d22a12dbaa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Apr 2022 14:56:20 -0400 Subject: [PATCH 606/916] Progress on implementing client side of mutable writes. --- src/allmydata/storage/http_client.py | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1bff34699..8899614b8 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -8,7 +8,7 @@ from typing import Union, Set, Optional from enum import Enum from base64 import b64encode -from attrs import define, field +from attrs import define, field, asdict # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -288,6 +288,7 @@ class StorageClient(object): lease_renew_secret=None, lease_cancel_secret=None, upload_secret=None, + write_enabler_secret=None, headers=None, message_to_serialize=None, **kwargs @@ -306,6 +307,7 @@ class StorageClient(object): (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), (Secrets.UPLOAD, upload_secret), + (Secrets.WRITE_ENABLER, write_enabler_secret), ]: if value is None: continue @@ -651,8 +653,9 @@ class StorageClientMutables: @async_to_deferred async def read_test_write_chunks( + self, storage_index: bytes, - write_enabled_secret: bytes, + write_enabler_secret: bytes, lease_renew_secret: bytes, lease_cancel_secret: bytes, testwrite_vectors: dict[int, TestWriteVectors], @@ -667,7 +670,31 @@ class StorageClientMutables: Given a mapping between share numbers and test/write vectors, the tests are done and if they are valid the writes are done. """ - pass + # TODO unit test all the things + url = self._client.relative_url( + "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) + ) + message = { + "test-write-vectors": { + share_number: asdict(twv) + for (share_number, twv) in testwrite_vectors.items() + }, + "read-vector": [asdict(r) for r in read_vector], + } + response = yield self._client.request( + "POST", + url, + write_enabler_secret=write_enabler_secret, + lease_renew_secret=lease_renew_secret, + lease_cancel_secret=lease_cancel_secret, + message_to_serialize=message, + ) + if response.code == http.OK: + return _decode_cbor(response, _SCHEMAS["mutable_test_read_write"]) + else: + raise ClientException( + response.code, + ) @async_to_deferred async def read_share_chunk( From 2ca5e22af9788aa4fccd3b25ed5c3188ef049ecb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 11:47:22 -0400 Subject: [PATCH 607/916] Make mutable reading match immutable reading. --- docs/proposed/http-storage-node-protocol.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 9354bc185..3926d9f4a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -743,11 +743,15 @@ For example:: [1, 5] -``GET /v1/mutable/:storage_index?share=:s0&share=:sN&offset=:o1&size=:z0&offset=:oN&size=:zN`` +``GET /v1/mutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Read data from the indicated mutable shares. -Just like ``GET /v1/mutable/:storage_index``. +Read data from the indicated mutable shares, just like ``GET /v1/immutable/:storage_index`` + +The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). +Interpretation and response behavior is as specified in RFC 7233 § 4.1. +Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. + ``POST /v1/mutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 898fe0bc0e48489668cf58981a18a252c9ed587f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 13:18:31 -0400 Subject: [PATCH 608/916] Closer to running end-to-end mutable tests. --- src/allmydata/storage/http_client.py | 107 ++++++++++++---------- src/allmydata/storage/http_server.py | 4 +- src/allmydata/storage_client.py | 54 ++++++++++- src/allmydata/test/test_istorageserver.py | 8 +- 4 files changed, 122 insertions(+), 51 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8899614b8..52177f401 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -364,6 +364,46 @@ class UploadProgress(object): required: RangeMap +@inlineCallbacks +def read_share_chunk( + client: StorageClient, + share_type: str, + storage_index: bytes, + share_number: int, + offset: int, + length: int, +) -> Deferred[bytes]: + """ + Download a chunk of data from a share. + + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ + url = client.relative_url( + "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number) + ) + response = yield client.request( + "GET", + url, + headers=Headers( + {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} + ), + ) + if response.code == http.PARTIAL_CONTENT: + body = yield response.content() + returnValue(body) + else: + raise ClientException(response.code) + + @define class StorageClientImmutables(object): """ @@ -484,39 +524,15 @@ class StorageClientImmutables(object): remaining.set(True, chunk["begin"], chunk["end"]) returnValue(UploadProgress(finished=finished, required=remaining)) - @inlineCallbacks def read_share_chunk( self, storage_index, share_number, offset, length ): # type: (bytes, int, int, int) -> Deferred[bytes] """ Download a chunk of data from a share. - - TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed - downloads should be transparently retried and redownloaded by the - implementation a few times so that if a failure percolates up, the - caller can assume the failure isn't a short-term blip. - - NOTE: the underlying HTTP protocol is much more flexible than this API, - so a future refactor may expand this in order to simplify the calling - code and perhaps download data more efficiently. But then again maybe - the HTTP protocol will be simplified, see - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - url = self._client.relative_url( - "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) + return read_share_chunk( + self._client, "immutable", storage_index, share_number, offset, length ) - response = yield self._client.request( - "GET", - url, - headers=Headers( - {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} - ), - ) - if response.code == http.PARTIAL_CONTENT: - body = yield response.content() - returnValue(body) - else: - raise ClientException(response.code) @inlineCallbacks def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] @@ -610,7 +626,7 @@ class TestVector: offset: int size: int - operator: TestVectorOperator = field(default=TestVectorOperator.EQ) + operator: TestVectorOperator specimen: bytes @@ -632,6 +648,14 @@ class TestWriteVectors: write_vectors: list[WriteVector] new_length: Optional[int] = field(default=None) + def asdict(self) -> dict: + """Return dictionary suitable for sending over CBOR.""" + d = asdict(self) + d["test"] = d.pop("test_vectors") + d["write"] = d.pop("write_vectors") + d["new-length"] = d.pop("new_length") + return d + @define class ReadTestWriteResult: @@ -676,12 +700,12 @@ class StorageClientMutables: ) message = { "test-write-vectors": { - share_number: asdict(twv) + share_number: twv.asdict() for (share_number, twv) in testwrite_vectors.items() }, "read-vector": [asdict(r) for r in read_vector], } - response = yield self._client.request( + response = await self._client.request( "POST", url, write_enabler_secret=write_enabler_secret, @@ -692,31 +716,20 @@ class StorageClientMutables: if response.code == http.OK: return _decode_cbor(response, _SCHEMAS["mutable_test_read_write"]) else: - raise ClientException( - response.code, - ) + raise ClientException(response.code, (await response.content())) @async_to_deferred async def read_share_chunk( self, storage_index: bytes, share_number: int, - # TODO is this really optional? - # TODO if yes, test non-optional variants - offset: Optional[int], - length: Optional[int], + offset: int, + length: int, ) -> bytes: """ Download a chunk of data from a share. - - TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed - downloads should be transparently retried and redownloaded by the - implementation a few times so that if a failure percolates up, the - caller can assume the failure isn't a short-term blip. - - NOTE: the underlying HTTP protocol is much more flexible than this API, - so a future refactor may expand this in order to simplify the calling - code and perhaps download data more efficiently. But then again maybe - the HTTP protocol will be simplified, see - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ + # TODO unit test all the things + return read_share_chunk( + self._client, "mutable", storage_index, share_number, offset, length + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7f279580b..6def5aeeb 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -261,7 +261,7 @@ _SCHEMAS = { * share_number: { "test": [* {"offset": uint, "size": uint, "specimen": bstr}] "write": [* {"offset": uint, "data": bstr}] - "new-length": (uint // null) + "new-length": uint // null } } "read-vector": [* {"offset": uint, "size": uint}] @@ -590,7 +590,7 @@ class HTTPServer(object): storage_index, secrets, rtw_request["test-write-vectors"], - rtw_request["read-vectors"], + rtw_request["read-vector"], ) return self._send_encoded(request, {"success": success, "data": read_data}) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 55b6cfb05..afed0e274 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -77,7 +77,8 @@ from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, - ClientException as HTTPClientException + ClientException as HTTPClientException, StorageClientMutables, + ReadVector, TestWriteVectors, WriteVector, TestVector, TestVectorOperator ) @@ -1189,3 +1190,54 @@ class _HTTPStorageServer(object): ) else: raise NotImplementedError() # future tickets + + @defer.inlineCallbacks + def slot_readv(self, storage_index, shares, readv): + mutable_client = StorageClientMutables(self._http_client) + reads = {} + for share_number in shares: + share_reads = reads[share_number] = [] + for (offset, length) in readv: + d = mutable_client.read_share_chunk( + storage_index, share_number, offset, length + ) + share_reads.append(d) + result = { + share_number: [(yield d) for d in share_reads] + for (share_number, reads) in reads.items() + } + defer.returnValue(result) + + @defer.inlineCallbacks + def slot_testv_and_readv_and_writev( + self, + storage_index, + secrets, + tw_vectors, + r_vector, + ): + mutable_client = StorageClientMutables(self._http_client) + we_secret, lr_secret, lc_secret = secrets + client_tw_vectors = {} + for share_num, (test_vector, data_vector, new_length) in tw_vectors.items(): + client_test_vectors = [ + TestVector(offset=offset, size=size, operator=TestVectorOperator[op], specimen=specimen) + for (offset, size, op, specimen) in test_vector + ] + client_write_vectors = [ + WriteVector(offset=offset, data=data) for (offset, data) in data_vector + ] + client_tw_vectors[share_num] = TestWriteVectors( + test_vectors=client_test_vectors, + write_vectors=client_write_vectors, + new_length=new_length + ) + client_read_vectors = [ + ReadVector(offset=offset, size=size) + for (offset, size) in r_vector + ] + client_result = yield mutable_client.read_test_write_chunks( + storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, + client_read_vectors, + ) + defer.returnValue((client_result.success, client_result.reads)) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 3d6f610be..702c66952 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1140,4 +1140,10 @@ class HTTPImmutableAPIsTests( class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): - """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + """Foolscap-specific tests for mutable ``IStorageServer`` APIs.""" + + +class HTTPMutableAPIsTests( + _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" From f5c4513cd38c79ec2af2fdba18811f023134c42b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 13:35:09 -0400 Subject: [PATCH 609/916] A little closer to serialization and deserialization working correctly, with some tests passing. --- src/allmydata/storage/http_client.py | 20 ++++---------------- src/allmydata/storage/http_server.py | 11 +++++++++-- src/allmydata/storage_client.py | 6 +++--- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 52177f401..7b80ec602 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -102,7 +102,7 @@ _SCHEMAS = { """ response = { "success": bool, - "data": [* share_number: [* bstr]] + "data": {* share_number: [* bstr]} } share_number = uint """ @@ -609,24 +609,12 @@ class WriteVector: data: bytes -class TestVectorOperator(Enum): - """Possible operators for test vectors.""" - - LT = b"lt" - LE = b"le" - EQ = b"eq" - NE = b"ne" - GE = b"ge" - GT = b"gt" - - @define class TestVector: """Checks to make on a chunk before writing to it.""" offset: int size: int - operator: TestVectorOperator specimen: bytes @@ -714,12 +702,12 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - return _decode_cbor(response, _SCHEMAS["mutable_test_read_write"]) + result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) + return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: raise ClientException(response.code, (await response.content())) - @async_to_deferred - async def read_share_chunk( + def read_share_chunk( self, storage_index: bytes, share_number: int, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6def5aeeb..3eae476b7 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -589,8 +589,15 @@ class HTTPServer(object): success, read_data = self._storage_server.slot_testv_and_readv_and_writev( storage_index, secrets, - rtw_request["test-write-vectors"], - rtw_request["read-vector"], + { + k: ( + [(d["offset"], d["size"], b"eq", d["specimen"]) for d in v["test"]], + [(d["offset"], d["data"]) for d in v["write"]], + v["new-length"], + ) + for (k, v) in rtw_request["test-write-vectors"].items() + }, + [(d["offset"], d["size"]) for d in rtw_request["read-vector"]], ) return self._send_encoded(request, {"success": success, "data": read_data}) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index afed0e274..5321efb7d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -78,7 +78,7 @@ from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, - ReadVector, TestWriteVectors, WriteVector, TestVector, TestVectorOperator + ReadVector, TestWriteVectors, WriteVector, TestVector ) @@ -1221,8 +1221,8 @@ class _HTTPStorageServer(object): client_tw_vectors = {} for share_num, (test_vector, data_vector, new_length) in tw_vectors.items(): client_test_vectors = [ - TestVector(offset=offset, size=size, operator=TestVectorOperator[op], specimen=specimen) - for (offset, size, op, specimen) in test_vector + TestVector(offset=offset, size=size, specimen=specimen) + for (offset, size, specimen) in test_vector ] client_write_vectors = [ WriteVector(offset=offset, data=data) for (offset, data) in data_vector From 21c3c50e37114a7a3e7e6d02ebe08af1101c45f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:07:57 -0400 Subject: [PATCH 610/916] Basic mutable read support. --- src/allmydata/storage/http_server.py | 43 ++++++++++++++++++++++++++++ src/allmydata/storage_client.py | 13 ++++----- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3eae476b7..dbb79cf2b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -601,6 +601,49 @@ class HTTPServer(object): ) return self._send_encoded(request, {"success": success, "data": read_data}) + @_authorized_route( + _app, + set(), + "/v1/mutable//", + methods=["GET"], + ) + def read_mutable_chunk(self, request, authorization, storage_index, share_number): + """Read a chunk from a mutable.""" + if request.getHeader("range") is None: + # TODO in follow-up ticket + raise NotImplementedError() + + # TODO reduce duplication with immutable reads? + # TODO unit tests, perhaps shared if possible + range_header = parse_range_header(request.getHeader("range")) + if ( + range_header is None + or range_header.units != "bytes" + or len(range_header.ranges) > 1 # more than one range + or range_header.ranges[0][1] is None # range without end + ): + request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) + return b"" + + offset, end = range_header.ranges[0] + + # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + data = self._storage_server.slot_readv( + storage_index, [share_number], [(offset, end - offset)] + )[share_number][0] + + # TODO reduce duplication? + request.setResponseCode(http.PARTIAL_CONTENT) + if len(data): + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) + return data + @implementer(IStreamServerEndpoint) @attr.s diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5321efb7d..857b19ed7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1195,18 +1195,17 @@ class _HTTPStorageServer(object): def slot_readv(self, storage_index, shares, readv): mutable_client = StorageClientMutables(self._http_client) reads = {} + # TODO if shares list is empty, that means list all shares, so we need + # to do a query to get that. + assert shares # TODO replace with call to list shares for share_number in shares: share_reads = reads[share_number] = [] for (offset, length) in readv: - d = mutable_client.read_share_chunk( + r = yield mutable_client.read_share_chunk( storage_index, share_number, offset, length ) - share_reads.append(d) - result = { - share_number: [(yield d) for d in share_reads] - for (share_number, reads) in reads.items() - } - defer.returnValue(result) + share_reads.append(r) + defer.returnValue(reads) @defer.inlineCallbacks def slot_testv_and_readv_and_writev( From f03feb0595c63c5cfbcd72cfa1243d9e7e375fe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:08:07 -0400 Subject: [PATCH 611/916] TODOs for later. --- src/allmydata/storage/http_server.py | 1 + src/allmydata/test/test_istorageserver.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index dbb79cf2b..b71877bbf 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -580,6 +580,7 @@ class HTTPServer(object): ) def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" + # TODO unit tests rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) secrets = ( authorization[Secrets.WRITE_ENABLER], diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 702c66952..e7b869713 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1147,3 +1147,12 @@ class HTTPMutableAPIsTests( _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" + + # TODO will be implemented in later tickets + SKIP_TESTS = { + "test_STARAW_write_enabler_must_match", + "test_add_lease_renewal", + "test_add_new_lease", + "test_advise_corrupt_share", + "test_slot_readv_no_shares", + } From 5ccafb2a032cdb5a91abd12be6fb0756465711f4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:08:30 -0400 Subject: [PATCH 612/916] News file. --- newsfragments/3890.mi | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3890.mi diff --git a/newsfragments/3890.mi b/newsfragments/3890.mi new file mode 100644 index 000000000..e69de29bb From 72c59b5f1a9a2960e53379d3753c0ee1fd5b5de3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:09:02 -0400 Subject: [PATCH 613/916] Unused import. --- src/allmydata/storage/http_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7b80ec602..09aada555 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,7 +5,6 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations from typing import Union, Set, Optional -from enum import Enum from base64 import b64encode from attrs import define, field, asdict From 3d710406ef0b222f29af023acc7dedcb156f41f5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Apr 2022 15:50:23 -0400 Subject: [PATCH 614/916] News file. --- newsfragments/3890.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3890.minor diff --git a/newsfragments/3890.minor b/newsfragments/3890.minor new file mode 100644 index 000000000..e69de29bb From 49c16f2a1acf804a6cd2c9e66ab46e10c888d463 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 27 Apr 2022 08:38:22 -0400 Subject: [PATCH 615/916] Delete 3890.mi remove spurious news fragment --- newsfragments/3890.mi | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 newsfragments/3890.mi diff --git a/newsfragments/3890.mi b/newsfragments/3890.mi deleted file mode 100644 index e69de29bb..000000000 From e16eb6dddfcb46dc530ba5ad81c4710578b6c609 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 11:46:37 -0400 Subject: [PATCH 616/916] Better type definitions. --- src/allmydata/storage/http_client.py | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 09aada555..da350e0c6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -4,10 +4,10 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations -from typing import Union, Set, Optional +from typing import Union, Optional, Sequence, Mapping from base64 import b64encode -from attrs import define, field, asdict +from attrs import define, asdict, frozen # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -134,8 +134,8 @@ def _decode_cbor(response, schema: Schema): class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" - already_have: Set[int] - allocated: Set[int] + already_have: set[int] + allocated: set[int] class _TLSContextFactory(CertificateOptions): @@ -420,7 +420,7 @@ class StorageClientImmutables(object): upload_secret, lease_renew_secret, lease_cancel_secret, - ): # type: (bytes, Set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + ): # type: (bytes, set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] """ Create a new storage index for an immutable. @@ -534,7 +534,7 @@ class StorageClientImmutables(object): ) @inlineCallbacks - def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] + def list_shares(self, storage_index): # type: (bytes,) -> Deferred[set[int]] """ Return the set of shares for a given storage index. """ @@ -600,7 +600,7 @@ class StorageClientImmutables(object): ) -@define +@frozen class WriteVector: """Data to write to a chunk.""" @@ -608,7 +608,7 @@ class WriteVector: data: bytes -@define +@frozen class TestVector: """Checks to make on a chunk before writing to it.""" @@ -617,7 +617,7 @@ class TestVector: specimen: bytes -@define +@frozen class ReadVector: """ Reads to do on chunks, as part of a read/test/write operation. @@ -627,13 +627,13 @@ class ReadVector: size: int -@define +@frozen class TestWriteVectors: """Test and write vectors for a specific share.""" - test_vectors: list[TestVector] - write_vectors: list[WriteVector] - new_length: Optional[int] = field(default=None) + test_vectors: Sequence[TestVector] + write_vectors: Sequence[WriteVector] + new_length: Optional[int] = None def asdict(self) -> dict: """Return dictionary suitable for sending over CBOR.""" @@ -644,17 +644,17 @@ class TestWriteVectors: return d -@define +@frozen class ReadTestWriteResult: """Result of sending read-test-write vectors.""" success: bool # Map share numbers to reads corresponding to the request's list of # ReadVectors: - reads: dict[int, list[bytes]] + reads: Mapping[int, Sequence[bytes]] -@define +@frozen class StorageClientMutables: """ APIs for interacting with mutables. From 76d0cfb770f5f886889a0d5731a6fdde9ab3665e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 11:49:21 -0400 Subject: [PATCH 617/916] Correct comment. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b71877bbf..0169d1463 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -570,7 +570,7 @@ class HTTPServer(object): bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" - ##### Immutable APIs ##### + ##### Mutable APIs ##### @_authorized_route( _app, From b8b1d7515a392984ee34c0d9af8d693fffa9089b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 11:59:50 -0400 Subject: [PATCH 618/916] We can at least be efficient when possible. --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 857b19ed7..9c6d5faa2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1197,7 +1197,7 @@ class _HTTPStorageServer(object): reads = {} # TODO if shares list is empty, that means list all shares, so we need # to do a query to get that. - assert shares # TODO replace with call to list shares + assert shares # TODO replace with call to list shares if and only if it's empty for share_number in shares: share_reads = reads[share_number] = [] for (offset, length) in readv: From 5ce204ed8d514eeaf5344f5ec3a774311137702a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 12:18:58 -0400 Subject: [PATCH 619/916] Make queries run in parallel. --- src/allmydata/storage_client.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 9c6d5faa2..68164e697 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1194,18 +1194,29 @@ class _HTTPStorageServer(object): @defer.inlineCallbacks def slot_readv(self, storage_index, shares, readv): mutable_client = StorageClientMutables(self._http_client) + pending_reads = {} reads = {} # TODO if shares list is empty, that means list all shares, so we need # to do a query to get that. assert shares # TODO replace with call to list shares if and only if it's empty + + # Start all the queries in parallel: for share_number in shares: - share_reads = reads[share_number] = [] - for (offset, length) in readv: - r = yield mutable_client.read_share_chunk( - storage_index, share_number, offset, length - ) - share_reads.append(r) - defer.returnValue(reads) + share_reads = defer.gatherResults( + [ + mutable_client.read_share_chunk( + storage_index, share_number, offset, length + ) + for (offset, length) in readv + ] + ) + pending_reads[share_number] = share_reads + + # Wait for all the queries to finish: + for share_number, pending_result in pending_reads.items(): + reads[share_number] = yield pending_result + + return reads @defer.inlineCallbacks def slot_testv_and_readv_and_writev( @@ -1239,4 +1250,4 @@ class _HTTPStorageServer(object): storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, client_read_vectors, ) - defer.returnValue((client_result.success, client_result.reads)) + return (client_result.success, client_result.reads) From 36e3beaa482df538eed024162c0302a71af24751 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Apr 2022 10:03:43 -0400 Subject: [PATCH 620/916] Get rid of deprecations builder. --- .circleci/config.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf0c66aff..79ce57ed0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,8 +48,6 @@ workflows: {} - "pyinstaller": {} - - "deprecations": - {} - "c-locale": {} # Any locale other than C or UTF-8. @@ -297,20 +295,6 @@ jobs: # aka "Latin 1" LANG: "en_US.ISO-8859-1" - - deprecations: - <<: *DEBIAN - - environment: - <<: *UTF_8_ENVIRONMENT - # Select the deprecations tox environments. - TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations" - # Put the logs somewhere we can report them. - TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log" - # The deprecations tox environments don't do coverage measurement. - UPLOAD_COVERAGE: "" - - integration: <<: *DEBIAN From 113eeb0e5908887aac8b03bd59a23fdc6999a3c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 10:21:55 -0400 Subject: [PATCH 621/916] News file. --- newsfragments/3891.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3891.minor diff --git a/newsfragments/3891.minor b/newsfragments/3891.minor new file mode 100644 index 000000000..e69de29bb From c1ce74f88d346d92299af11c11ab01789d368c4e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 11:03:14 -0400 Subject: [PATCH 622/916] Ability to list shares, enabling more of IStorageClient to run over HTTP. --- docs/proposed/http-storage-node-protocol.rst | 2 +- src/allmydata/storage/http_client.py | 20 ++++++++++++ src/allmydata/storage/http_server.py | 14 ++++++++ src/allmydata/storage/server.py | 34 +++++++++++++------- src/allmydata/storage_client.py | 5 +-- src/allmydata/test/test_istorageserver.py | 1 - 6 files changed, 60 insertions(+), 16 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3926d9f4a..693ce9290 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -738,7 +738,7 @@ Reading ``GET /v1/mutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Retrieve a list indicating all shares available for the indicated storage index. +Retrieve a set indicating all shares available for the indicated storage index. For example:: [1, 5] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index da350e0c6..5920d5a5b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -106,6 +106,11 @@ _SCHEMAS = { share_number = uint """ ), + "mutable_list_shares": Schema( + """ + response = #6.258([* uint]) + """ + ), } @@ -720,3 +725,18 @@ class StorageClientMutables: return read_share_chunk( self._client, "mutable", storage_index, share_number, offset, length ) + + @async_to_deferred + async def list_shares(self, storage_index: bytes) -> set[int]: + """ + List the share numbers for a given storage index. + """ + # TODO unit test all the things + url = self._client.relative_url( + "/v1/mutable/{}/shares".format(_encode_si(storage_index)) + ) + response = await self._client.request("GET", url) + if response.code == http.OK: + return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"]) + else: + raise ClientException(response.code) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0169d1463..0b407a1c4 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -645,6 +645,20 @@ class HTTPServer(object): ) return data + @_authorized_route( + _app, + set(), + "/v1/mutable//shares", + methods=["GET"], + ) + def list_mutable_shares(self, request, authorization, storage_index): + """List mutable shares for a storage index.""" + try: + shares = self._storage_server.list_mutable_shares(storage_index) + except KeyError: + raise _HTTPError(http.NOT_FOUND) + return self._send_encoded(request, shares) + @implementer(IStreamServerEndpoint) @attr.s diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9d1a3d6a4..1a0255601 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -1,18 +1,9 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import bytes_to_native_str, PY2 -if PY2: - # Omit open() to get native behavior where open("w") always accepts native - # strings. Omit bytes so we don't leak future's custom bytes. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 -else: - from typing import Dict, Tuple +from __future__ import annotations +from future.utils import bytes_to_native_str +from typing import Dict, Tuple import os, re @@ -699,6 +690,25 @@ class StorageServer(service.MultiService): self) return share + def list_mutable_shares(self, storage_index) -> set[int]: + """List all share numbers for the given mutable. + + Raises ``KeyError`` if the storage index is not known. + """ + # TODO unit test + si_dir = storage_index_to_dir(storage_index) + # shares exist if there is a file for them + bucketdir = os.path.join(self.sharedir, si_dir) + if not os.path.isdir(bucketdir): + raise KeyError("Not found") + result = set() + for sharenum_s in os.listdir(bucketdir): + try: + result.add(int(sharenum_s)) + except ValueError: + continue + return result + def slot_readv(self, storage_index, shares, readv): start = self._clock.seconds() self.count("readv") diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 68164e697..8b2f68a9e 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1196,9 +1196,10 @@ class _HTTPStorageServer(object): mutable_client = StorageClientMutables(self._http_client) pending_reads = {} reads = {} - # TODO if shares list is empty, that means list all shares, so we need + # If shares list is empty, that means list all shares, so we need # to do a query to get that. - assert shares # TODO replace with call to list shares if and only if it's empty + if not shares: + shares = yield mutable_client.list_shares(storage_index) # Start all the queries in parallel: for share_number in shares: diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index e7b869713..d9fd13acb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1154,5 +1154,4 @@ class HTTPMutableAPIsTests( "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", - "test_slot_readv_no_shares", } From 852162ba0694f0405666d432d39b464f646eeca0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 11:03:35 -0400 Subject: [PATCH 623/916] More accurate docs. --- src/allmydata/storage/http_client.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5920d5a5b..2db28dc72 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -380,16 +380,14 @@ def read_share_chunk( """ Download a chunk of data from a share. - TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed - downloads should be transparently retried and redownloaded by the - implementation a few times so that if a failure percolates up, the - caller can assume the failure isn't a short-term blip. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed downloads + should be transparently retried and redownloaded by the implementation a + few times so that if a failure percolates up, the caller can assume the + failure isn't a short-term blip. - NOTE: the underlying HTTP protocol is much more flexible than this API, - so a future refactor may expand this in order to simplify the calling - code and perhaps download data more efficiently. But then again maybe - the HTTP protocol will be simplified, see - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + NOTE: the underlying HTTP protocol is somewhat more flexible than this API, + insofar as it doesn't always require a range. In practice a range is + always provided by the current callers. """ url = client.relative_url( "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number) @@ -717,7 +715,7 @@ class StorageClientMutables: share_number: int, offset: int, length: int, - ) -> bytes: + ) -> Deferred[bytes]: """ Download a chunk of data from a share. """ From 06029d2878b6ad6edc67ae79dc1f59124c6934f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 11:25:13 -0400 Subject: [PATCH 624/916] Another end-to-end test passing (albeit with ugly implementation). --- src/allmydata/storage/http_client.py | 6 +++++ src/allmydata/storage/http_server.py | 33 ++++++++++++++--------- src/allmydata/test/test_istorageserver.py | 1 - 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2db28dc72..0229bef03 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -706,6 +706,12 @@ class StorageClientMutables: if response.code == http.OK: result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) return ReadTestWriteResult(success=result["success"], reads=result["data"]) + elif response.code == http.UNAUTHORIZED: + # TODO mabye we can fix this to be nicer at some point? Custom + # exception? + from foolscap.api import RemoteException + + raise RemoteException("Authorization failed") else: raise ClientException(response.code, (await response.content())) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0b407a1c4..748790a72 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -46,6 +46,7 @@ from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet +from allmydata.interfaces import BadWriteEnablerError class ClientSecretsException(Exception): @@ -587,19 +588,25 @@ class HTTPServer(object): authorization[Secrets.LEASE_RENEW], authorization[Secrets.LEASE_CANCEL], ) - success, read_data = self._storage_server.slot_testv_and_readv_and_writev( - storage_index, - secrets, - { - k: ( - [(d["offset"], d["size"], b"eq", d["specimen"]) for d in v["test"]], - [(d["offset"], d["data"]) for d in v["write"]], - v["new-length"], - ) - for (k, v) in rtw_request["test-write-vectors"].items() - }, - [(d["offset"], d["size"]) for d in rtw_request["read-vector"]], - ) + try: + success, read_data = self._storage_server.slot_testv_and_readv_and_writev( + storage_index, + secrets, + { + k: ( + [ + (d["offset"], d["size"], b"eq", d["specimen"]) + for d in v["test"] + ], + [(d["offset"], d["data"]) for d in v["write"]], + v["new-length"], + ) + for (k, v) in rtw_request["test-write-vectors"].items() + }, + [(d["offset"], d["size"]) for d in rtw_request["read-vector"]], + ) + except BadWriteEnablerError: + raise _HTTPError(http.UNAUTHORIZED) return self._send_encoded(request, {"success": success, "data": read_data}) @_authorized_route( diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index d9fd13acb..a3e75bbac 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1150,7 +1150,6 @@ class HTTPMutableAPIsTests( # TODO will be implemented in later tickets SKIP_TESTS = { - "test_STARAW_write_enabler_must_match", "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", From 2833bec80e7e6ff069b4b6eee890f99942c3dfc4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 5 May 2022 12:04:45 -0400 Subject: [PATCH 625/916] Unit test the new storage server backend API. --- src/allmydata/storage/server.py | 1 - src/allmydata/test/test_storage.py | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 1a0255601..b46303cd8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -695,7 +695,6 @@ class StorageServer(service.MultiService): Raises ``KeyError`` if the storage index is not known. """ - # TODO unit test si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index b37f74c24..8f1ece401 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1315,6 +1315,31 @@ class MutableServer(unittest.TestCase): self.failUnless(isinstance(readv_data, dict)) self.failUnlessEqual(len(readv_data), 0) + def test_list_mutable_shares(self): + """ + ``StorageServer.list_mutable_shares()`` returns a set of share numbers + for the given storage index, or raises ``KeyError`` if it does not exist at all. + """ + ss = self.create("test_list_mutable_shares") + + # Initially, nothing exists: + with self.assertRaises(KeyError): + ss.list_mutable_shares(b"si1") + + self.allocate(ss, b"si1", b"we1", b"le1", [0, 1, 4, 2], 12) + shares0_1_2_4 = ss.list_mutable_shares(b"si1") + + # Remove share 2, by setting size to 0: + secrets = (self.write_enabler(b"we1"), + self.renew_secret(b"le1"), + self.cancel_secret(b"le1")) + ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) + shares0_1_4 = ss.list_mutable_shares(b"si1") + self.assertEqual( + (shares0_1_2_4, shares0_1_4), + ({0, 1, 2, 4}, {0, 1, 4}) + ) + def test_bad_magic(self): ss = self.create("test_bad_magic") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10) From b3fed56c00d03599b4a8479e5de0a36c696c7a97 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 5 May 2022 12:11:09 -0400 Subject: [PATCH 626/916] Move Foolscap compatibility to a better place. --- src/allmydata/storage/http_client.py | 6 ------ src/allmydata/storage_client.py | 16 +++++++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 0229bef03..2db28dc72 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -706,12 +706,6 @@ class StorageClientMutables: if response.code == http.OK: result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) return ReadTestWriteResult(success=result["success"], reads=result["data"]) - elif response.code == http.UNAUTHORIZED: - # TODO mabye we can fix this to be nicer at some point? Custom - # exception? - from foolscap.api import RemoteException - - raise RemoteException("Authorization failed") else: raise ClientException(response.code, (await response.content())) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8b2f68a9e..cd489a307 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -50,6 +50,7 @@ from zope.interface import ( Interface, implementer, ) +from twisted.web import http from twisted.internet import defer from twisted.application import service from twisted.plugin import ( @@ -78,7 +79,7 @@ from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, - ReadVector, TestWriteVectors, WriteVector, TestVector + ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ) @@ -1247,8 +1248,13 @@ class _HTTPStorageServer(object): ReadVector(offset=offset, size=size) for (offset, size) in r_vector ] - client_result = yield mutable_client.read_test_write_chunks( - storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, - client_read_vectors, - ) + try: + client_result = yield mutable_client.read_test_write_chunks( + storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, + client_read_vectors, + ) + except ClientException as e: + if e.code == http.UNAUTHORIZED: + raise RemoteException("Unauthorized write, possibly you passed the wrong write enabler?") + raise return (client_result.success, client_result.reads) From 5b0762d3a3a8fa1eb98aa6cd3b5b4d14e53047a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 10 May 2022 13:59:58 -0400 Subject: [PATCH 627/916] Workaround for autobahn issues. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c84d0ecde..2b4fd6988 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn >= 19.5.2", + "autobahn < 22.4.1", # remove this when https://github.com/crossbario/autobahn-python/issues/1566 is fixed # Support for Python 3 transition "future >= 0.18.2", From 6f5a0e43ebceb730c97598b4fbe49f65f052e44b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 10:41:36 -0400 Subject: [PATCH 628/916] Implement advise_corrupt_share for mutables. --- src/allmydata/storage/http_client.py | 51 ++++++++++++++++------- src/allmydata/storage/http_server.py | 20 +++++++++ src/allmydata/storage_client.py | 12 +++--- src/allmydata/test/test_istorageserver.py | 1 - 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2db28dc72..f39ed7d1a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -406,6 +406,30 @@ def read_share_chunk( raise ClientException(response.code) +@async_to_deferred +async def advise_corrupt_share( + client: StorageClient, + share_type: str, + storage_index: bytes, + share_number: int, + reason: str, +): + assert isinstance(reason, str) + url = client.relative_url( + "/v1/{}/{}/{}/corrupt".format( + share_type, _encode_si(storage_index), share_number + ) + ) + message = {"reason": reason} + response = await client.request("POST", url, message_to_serialize=message) + if response.code == http.OK: + return + else: + raise ClientException( + response.code, + ) + + @define class StorageClientImmutables(object): """ @@ -579,7 +603,6 @@ class StorageClientImmutables(object): else: raise ClientException(response.code) - @inlineCallbacks def advise_corrupt_share( self, storage_index: bytes, @@ -587,20 +610,9 @@ class StorageClientImmutables(object): reason: str, ): """Indicate a share has been corrupted, with a human-readable message.""" - assert isinstance(reason, str) - url = self._client.relative_url( - "/v1/immutable/{}/{}/corrupt".format( - _encode_si(storage_index), share_number - ) + return advise_corrupt_share( + self._client, "immutable", storage_index, share_number, reason ) - message = {"reason": reason} - response = yield self._client.request("POST", url, message_to_serialize=message) - if response.code == http.OK: - return - else: - raise ClientException( - response.code, - ) @frozen @@ -738,3 +750,14 @@ class StorageClientMutables: return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"]) else: raise ClientException(response.code) + + def advise_corrupt_share( + self, + storage_index: bytes, + share_number: int, + reason: str, + ): + """Indicate a share has been corrupted, with a human-readable message.""" + return advise_corrupt_share( + self._client, "mutable", storage_index, share_number, reason + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 748790a72..102a33e90 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -666,6 +666,26 @@ class HTTPServer(object): raise _HTTPError(http.NOT_FOUND) return self._send_encoded(request, shares) + @_authorized_route( + _app, + set(), + "/v1/mutable///corrupt", + methods=["POST"], + ) + def advise_corrupt_share_mutable( + self, request, authorization, storage_index, share_number + ): + """Indicate that given share is corrupt, with a text reason.""" + # TODO unit test all the paths + if not self._storage_server._share_exists(storage_index, share_number): + raise _HTTPError(http.NOT_FOUND) + + info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) + self._storage_server.advise_corrupt_share( + b"mutable", storage_index, share_number, info["reason"].encode("utf-8") + ) + return b"" + @implementer(IStreamServerEndpoint) @attr.s diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index cd489a307..c83527600 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1185,12 +1185,14 @@ class _HTTPStorageServer(object): reason: bytes ): if share_type == b"immutable": - imm_client = StorageClientImmutables(self._http_client) - return imm_client.advise_corrupt_share( - storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") - ) + client = StorageClientImmutables(self._http_client) + elif share_type == b"mutable": + client = StorageClientMutables(self._http_client) else: - raise NotImplementedError() # future tickets + raise ValueError("Unknown share type") + return client.advise_corrupt_share( + storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") + ) @defer.inlineCallbacks def slot_readv(self, storage_index, shares, readv): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a3e75bbac..70543cbf0 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1152,5 +1152,4 @@ class HTTPMutableAPIsTests( SKIP_TESTS = { "test_add_lease_renewal", "test_add_new_lease", - "test_advise_corrupt_share", } From 7ae682af27d878f7b07e4a0e533efe105348da95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 10:41:56 -0400 Subject: [PATCH 629/916] News file. --- newsfragments/3893.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3893.minor diff --git a/newsfragments/3893.minor b/newsfragments/3893.minor new file mode 100644 index 000000000..e69de29bb From 4afe3eb224d6f26ac768aaccbca68c69035d57d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 10:58:13 -0400 Subject: [PATCH 630/916] Clarify sets vs lists some more. --- docs/proposed/http-storage-node-protocol.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 693ce9290..7e0b4a542 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -350,8 +350,10 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. +The one exception is sets. For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry `_ for more details. +Sets will be represented as JSON lists in examples because JSON doesn't support sets. HTTP Design ~~~~~~~~~~~ @@ -739,7 +741,7 @@ Reading !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. -For example:: +For example (this is shown as list, since it will be list for JSON, but will be set for CBOR):: [1, 5] From 07e16b80b5df22a5620d390cb34032a55e62e795 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:00:05 -0400 Subject: [PATCH 631/916] Better name. --- src/allmydata/storage/http_server.py | 4 ++-- src/allmydata/storage/server.py | 2 +- src/allmydata/test/test_storage.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 748790a72..96e906e43 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -658,10 +658,10 @@ class HTTPServer(object): "/v1/mutable//shares", methods=["GET"], ) - def list_mutable_shares(self, request, authorization, storage_index): + def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" try: - shares = self._storage_server.list_mutable_shares(storage_index) + shares = self._storage_server.enumerate_mutable_shares(storage_index) except KeyError: raise _HTTPError(http.NOT_FOUND) return self._send_encoded(request, shares) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index b46303cd8..ab7947bf9 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -690,7 +690,7 @@ class StorageServer(service.MultiService): self) return share - def list_mutable_shares(self, storage_index) -> set[int]: + def enumerate_mutable_shares(self, storage_index) -> set[int]: """List all share numbers for the given mutable. Raises ``KeyError`` if the storage index is not known. diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 8f1ece401..9bc218fa6 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1315,26 +1315,26 @@ class MutableServer(unittest.TestCase): self.failUnless(isinstance(readv_data, dict)) self.failUnlessEqual(len(readv_data), 0) - def test_list_mutable_shares(self): + def test_enumerate_mutable_shares(self): """ - ``StorageServer.list_mutable_shares()`` returns a set of share numbers + ``StorageServer.enumerate_mutable_shares()`` returns a set of share numbers for the given storage index, or raises ``KeyError`` if it does not exist at all. """ - ss = self.create("test_list_mutable_shares") + ss = self.create("test_enumerate_mutable_shares") # Initially, nothing exists: with self.assertRaises(KeyError): - ss.list_mutable_shares(b"si1") + ss.enumerate_mutable_shares(b"si1") self.allocate(ss, b"si1", b"we1", b"le1", [0, 1, 4, 2], 12) - shares0_1_2_4 = ss.list_mutable_shares(b"si1") + shares0_1_2_4 = ss.enumerate_mutable_shares(b"si1") # Remove share 2, by setting size to 0: secrets = (self.write_enabler(b"we1"), self.renew_secret(b"le1"), self.cancel_secret(b"le1")) ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) - shares0_1_4 = ss.list_mutable_shares(b"si1") + shares0_1_4 = ss.enumerate_mutable_shares(b"si1") self.assertEqual( (shares0_1_2_4, shares0_1_4), ({0, 1, 2, 4}, {0, 1, 4}) From 6d412a017c34fc6f6528c89b1443d9bdf7caee64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:00:46 -0400 Subject: [PATCH 632/916] Type annotation. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ab7947bf9..f1b835780 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -690,7 +690,7 @@ class StorageServer(service.MultiService): self) return share - def enumerate_mutable_shares(self, storage_index) -> set[int]: + def enumerate_mutable_shares(self, storage_index: bytes) -> set[int]: """List all share numbers for the given mutable. Raises ``KeyError`` if the storage index is not known. From 4b62ec082bed7ecc33612741dfbffc16868ca3ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:11:24 -0400 Subject: [PATCH 633/916] Match Foolscap behavior for slot_readv of unknown storage index. --- src/allmydata/storage_client.py | 8 +++++++- src/allmydata/test/test_istorageserver.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index cd489a307..83a1233f5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1200,7 +1200,13 @@ class _HTTPStorageServer(object): # If shares list is empty, that means list all shares, so we need # to do a query to get that. if not shares: - shares = yield mutable_client.list_shares(storage_index) + try: + shares = yield mutable_client.list_shares(storage_index) + except ClientException as e: + if e.code == http.NOT_FOUND: + shares = set() + else: + raise # Start all the queries in parallel: for share_number in shares: diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a3e75bbac..66535ddda 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -854,6 +854,22 @@ class IStorageServerMutableAPIsTestsMixin(object): {0: [b"abcdefg"], 1: [b"0123456"], 2: [b"9876543"]}, ) + @inlineCallbacks + def test_slot_readv_unknown_storage_index(self): + """ + With unknown storage index, ``IStorageServer.slot_readv()`` TODO. + """ + storage_index = new_storage_index() + reads = yield self.storage_client.slot_readv( + storage_index, + shares=[], + readv=[(0, 7)], + ) + self.assertEqual( + reads, + {}, + ) + @inlineCallbacks def create_slot(self): """Create a slot with sharenum 0.""" From 457db8f992c62e8f5c5bc4cdcb6e99f097062f08 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:17:57 -0400 Subject: [PATCH 634/916] Get rid of the "no such storage index" edge case, since it's not really necessary. --- src/allmydata/storage/http_server.py | 5 +---- src/allmydata/storage/server.py | 7 ++----- src/allmydata/storage_client.py | 8 +------- src/allmydata/test/test_storage.py | 12 ++++++------ 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 96e906e43..a1641f563 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -660,10 +660,7 @@ class HTTPServer(object): ) def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" - try: - shares = self._storage_server.enumerate_mutable_shares(storage_index) - except KeyError: - raise _HTTPError(http.NOT_FOUND) + shares = self._storage_server.enumerate_mutable_shares(storage_index) return self._send_encoded(request, shares) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f1b835780..bcf44dc30 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -691,15 +691,12 @@ class StorageServer(service.MultiService): return share def enumerate_mutable_shares(self, storage_index: bytes) -> set[int]: - """List all share numbers for the given mutable. - - Raises ``KeyError`` if the storage index is not known. - """ + """Return all share numbers for the given mutable.""" si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): - raise KeyError("Not found") + return set() result = set() for sharenum_s in os.listdir(bucketdir): try: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 83a1233f5..cd489a307 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1200,13 +1200,7 @@ class _HTTPStorageServer(object): # If shares list is empty, that means list all shares, so we need # to do a query to get that. if not shares: - try: - shares = yield mutable_client.list_shares(storage_index) - except ClientException as e: - if e.code == http.NOT_FOUND: - shares = set() - else: - raise + shares = yield mutable_client.list_shares(storage_index) # Start all the queries in parallel: for share_number in shares: diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9bc218fa6..65d09de25 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1317,14 +1317,14 @@ class MutableServer(unittest.TestCase): def test_enumerate_mutable_shares(self): """ - ``StorageServer.enumerate_mutable_shares()`` returns a set of share numbers - for the given storage index, or raises ``KeyError`` if it does not exist at all. + ``StorageServer.enumerate_mutable_shares()`` returns a set of share + numbers for the given storage index, or an empty set if it does not + exist at all. """ ss = self.create("test_enumerate_mutable_shares") # Initially, nothing exists: - with self.assertRaises(KeyError): - ss.enumerate_mutable_shares(b"si1") + empty = ss.enumerate_mutable_shares(b"si1") self.allocate(ss, b"si1", b"we1", b"le1", [0, 1, 4, 2], 12) shares0_1_2_4 = ss.enumerate_mutable_shares(b"si1") @@ -1336,8 +1336,8 @@ class MutableServer(unittest.TestCase): ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) shares0_1_4 = ss.enumerate_mutable_shares(b"si1") self.assertEqual( - (shares0_1_2_4, shares0_1_4), - ({0, 1, 2, 4}, {0, 1, 4}) + (empty, shares0_1_2_4, shares0_1_4), + (set(), {0, 1, 2, 4}, {0, 1, 4}) ) def test_bad_magic(self): From 821bac3ddf4bc829d4a0724404242974fa2f1d0a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:50:01 -0400 Subject: [PATCH 635/916] Test another lease edge case. --- src/allmydata/storage_client.py | 16 ++++++++++++---- src/allmydata/test/test_istorageserver.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c83527600..e8d0e003a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -76,6 +76,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, @@ -1166,16 +1167,23 @@ class _HTTPStorageServer(object): for share_num in share_numbers }) - def add_lease( + @async_to_deferred + async def add_lease( self, storage_index, renew_secret, cancel_secret ): immutable_client = StorageClientImmutables(self._http_client) - return immutable_client.add_or_renew_lease( - storage_index, renew_secret, cancel_secret - ) + try: + await immutable_client.add_or_renew_lease( + storage_index, renew_secret, cancel_secret + ) + except ClientException as e: + if e.code == http.NOT_FOUND: + # Silently do nothing, as is the case for the Foolscap client + return + raise def advise_corrupt_share( self, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index abb2e0fc4..cee80f8fb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -459,6 +459,21 @@ class IStorageServerImmutableAPIsTestsMixin(object): lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) ) + @inlineCallbacks + def test_add_lease_non_existent(self): + """ + If the storage index doesn't exist, adding the lease silently does nothing. + """ + storage_index = new_storage_index() + self.assertEqual(list(self.server.get_leases(storage_index)), []) + + renew_secret = new_secret() + cancel_secret = new_secret() + + # Add a lease: + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + self.assertEqual(list(self.server.get_leases(storage_index)), []) + @inlineCallbacks def test_add_lease_renewal(self): """ From b8735c79daefb751b4d81ba7631a81b37904b732 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:50:29 -0400 Subject: [PATCH 636/916] Fix docstring. --- src/allmydata/test/test_istorageserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cee80f8fb..c0dd50590 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -872,7 +872,8 @@ class IStorageServerMutableAPIsTestsMixin(object): @inlineCallbacks def test_slot_readv_unknown_storage_index(self): """ - With unknown storage index, ``IStorageServer.slot_readv()`` TODO. + With unknown storage index, ``IStorageServer.slot_readv()`` returns + empty dict. """ storage_index = new_storage_index() reads = yield self.storage_client.slot_readv( From f3cf13154da2c03e39fa3249384ec6e01ef90735 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 12:00:27 -0400 Subject: [PATCH 637/916] Setup HTTP lease APIs for immutables too. --- src/allmydata/storage/http_client.py | 50 +++++++++++------------ src/allmydata/storage/http_server.py | 2 +- src/allmydata/storage_client.py | 2 +- src/allmydata/test/test_istorageserver.py | 6 --- src/allmydata/test/test_storage_http.py | 7 ++-- 5 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f39ed7d1a..167d2394a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -355,6 +355,31 @@ class StorageClientGeneral(object): decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) + @inlineCallbacks + def add_or_renew_lease( + self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes + ): + """ + Add or renew a lease. + + If the renewal secret matches an existing lease, it is renewed. + Otherwise a new lease is added. + """ + url = self._client.relative_url( + "/v1/lease/{}".format(_encode_si(storage_index)) + ) + response = yield self._client.request( + "PUT", + url, + lease_renew_secret=renew_secret, + lease_cancel_secret=cancel_secret, + ) + + if response.code == http.NO_CONTENT: + return + else: + raise ClientException(response.code) + @define class UploadProgress(object): @@ -578,31 +603,6 @@ class StorageClientImmutables(object): else: raise ClientException(response.code) - @inlineCallbacks - def add_or_renew_lease( - self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes - ): - """ - Add or renew a lease. - - If the renewal secret matches an existing lease, it is renewed. - Otherwise a new lease is added. - """ - url = self._client.relative_url( - "/v1/lease/{}".format(_encode_si(storage_index)) - ) - response = yield self._client.request( - "PUT", - url, - lease_renew_secret=renew_secret, - lease_cancel_secret=cancel_secret, - ) - - if response.code == http.NO_CONTENT: - return - else: - raise ClientException(response.code) - def advise_corrupt_share( self, storage_index: bytes, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index db73b6a86..709c1fda5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -539,7 +539,7 @@ class HTTPServer(object): ) def add_or_renew_lease(self, request, authorization, storage_index): """Update the lease for an immutable share.""" - if not self._storage_server.get_buckets(storage_index): + if not list(self._storage_server._get_bucket_shares(storage_index)): raise _HTTPError(http.NOT_FOUND) # Checking of the renewal secret is done by the backend. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e8d0e003a..c529c4513 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1174,7 +1174,7 @@ class _HTTPStorageServer(object): renew_secret, cancel_secret ): - immutable_client = StorageClientImmutables(self._http_client) + immutable_client = StorageClientGeneral(self._http_client) try: await immutable_client.add_or_renew_lease( storage_index, renew_secret, cancel_secret diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index c0dd50590..39675336f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1179,9 +1179,3 @@ class HTTPMutableAPIsTests( _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" - - # TODO will be implemented in later tickets - SKIP_TESTS = { - "test_add_lease_renewal", - "test_add_new_lease", - } diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index df781012e..fcc2401f2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -448,6 +448,7 @@ class ImmutableHTTPAPITests(SyncTestCase): super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) self.imm_client = StorageClientImmutables(self.http.client) + self.general_client = StorageClientGeneral(self.http.client) def create_upload(self, share_numbers, length): """ @@ -1081,7 +1082,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We renew the lease: result_of( - self.imm_client.add_or_renew_lease( + self.general_client.add_or_renew_lease( storage_index, lease_secret, lease_secret ) ) @@ -1092,7 +1093,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We create a new lease: lease_secret2 = urandom(32) result_of( - self.imm_client.add_or_renew_lease( + self.general_client.add_or_renew_lease( storage_index, lease_secret2, lease_secret2 ) ) @@ -1108,7 +1109,7 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = urandom(16) secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.imm_client.add_or_renew_lease(storage_index, secret, secret)) + result_of(self.general_client.add_or_renew_lease(storage_index, secret, secret)) def test_advise_corrupt_share(self): """ From a54b443f9d2658cb6e196570a7a74681a8bec44d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 May 2022 09:44:30 -0400 Subject: [PATCH 638/916] It's not an immutable client anymore. --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c529c4513..0f66e8e4a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1174,9 +1174,9 @@ class _HTTPStorageServer(object): renew_secret, cancel_secret ): - immutable_client = StorageClientGeneral(self._http_client) + client = StorageClientGeneral(self._http_client) try: - await immutable_client.add_or_renew_lease( + await client.add_or_renew_lease( storage_index, renew_secret, cancel_secret ) except ClientException as e: From b0b67826e8c9a026879c68c04c13d2cce9a6466e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 12:58:55 -0400 Subject: [PATCH 639/916] More verbose output is helpful when debugging. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 859cf18e0..fc95a0469 100644 --- a/tox.ini +++ b/tox.ini @@ -97,7 +97,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration} + py.test --timeout=1800 --coverage -s -v {posargs:integration} coverage combine coverage report From 20b021809c0a2cf3bd2abf5991de5638b48134b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 12:59:04 -0400 Subject: [PATCH 640/916] Fix(?) the intermittently failing test. --- integration/test_tor.py | 6 +++++- newsfragments/3895.minor | 0 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3895.minor diff --git a/integration/test_tor.py b/integration/test_tor.py index b0419f0d2..5b701287c 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -21,7 +21,8 @@ from . import util from twisted.python.filepath import ( FilePath, ) - +from twisted.internet.task import deferLater +from twisted.internet import reactor from allmydata.test.common import ( write_introducer, ) @@ -68,6 +69,9 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) + # For some reason a wait is needed, or sometimes the get fails... + yield deferLater(reactor, 2, lambda: None) + proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, diff --git a/newsfragments/3895.minor b/newsfragments/3895.minor new file mode 100644 index 000000000..e69de29bb From 757b4492d75c899d21809ff51f3383189c7eb3c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 13:29:08 -0400 Subject: [PATCH 641/916] A more semantically correct fix. --- integration/test_tor.py | 16 ++++++++-------- integration/util.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 5b701287c..d17e0f5cf 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -21,8 +21,7 @@ from . import util from twisted.python.filepath import ( FilePath, ) -from twisted.internet.task import deferLater -from twisted.internet import reactor + from allmydata.test.common import ( write_introducer, ) @@ -41,8 +40,11 @@ if PY2: @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): - yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + util.await_client_ready(carol, expected_number_of_servers=2) + util.await_client_ready(dave, expected_number_of_servers=2) + # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -69,9 +71,6 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) - # For some reason a wait is needed, or sometimes the get fails... - yield deferLater(reactor, 2, lambda: None) - proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, @@ -147,5 +146,6 @@ shares.total = 2 f.write(node_config) print("running") - yield util._run_node(reactor, node_dir.path, request, None) + result = yield util._run_node(reactor, node_dir.path, request, None) print("okay, launched") + return result diff --git a/integration/util.py b/integration/util.py index 7c7a1efd2..0ec824f82 100644 --- a/integration/util.py +++ b/integration/util.py @@ -482,14 +482,15 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2): +def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be 'ready'. A client is deemed ready if: - it answers `http:///statistics/?t=json/` - - there is at least one storage-server connected + - there is at least one storage-server connected (configurable via + ``expected_number_of_servers``) - every storage-server has a "last_received_data" and it is within the last `liveness` seconds @@ -506,8 +507,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2): time.sleep(1) continue - if len(js['servers']) == 0: - print("waiting because no servers at all") + if len(js['servers']) != expected_number_of_servers: + print("waiting because insufficient servers") time.sleep(1) continue server_times = [ From f752f547ba50e283a13abac8b9764cef305ad2a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 13:30:47 -0400 Subject: [PATCH 642/916] More servers is fine. --- integration/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/util.py b/integration/util.py index 0ec824f82..ad9249e45 100644 --- a/integration/util.py +++ b/integration/util.py @@ -482,7 +482,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_servers=1): +def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -490,7 +490,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_serv - it answers `http:///statistics/?t=json/` - there is at least one storage-server connected (configurable via - ``expected_number_of_servers``) + ``minimum_number_of_servers``) - every storage-server has a "last_received_data" and it is within the last `liveness` seconds @@ -507,7 +507,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_serv time.sleep(1) continue - if len(js['servers']) != expected_number_of_servers: + if len(js['servers']) < minimum_number_of_servers: print("waiting because insufficient servers") time.sleep(1) continue From 69f1244c5a85914535008911b231b1483fdee953 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 13:42:10 -0400 Subject: [PATCH 643/916] Fix keyword argument name. --- integration/test_tor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index d17e0f5cf..c78fa8098 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -42,8 +42,8 @@ if PY2: def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - util.await_client_ready(carol, expected_number_of_servers=2) - util.await_client_ready(dave, expected_number_of_servers=2) + util.await_client_ready(carol, minimum_number_of_servers=2) + util.await_client_ready(dave, minimum_number_of_servers=2) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. From 3abf992321c62728cf194090380dc46c32dc0156 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 14:05:53 -0400 Subject: [PATCH 644/916] Autobahn regression workaround. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c84d0ecde..2b4fd6988 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn >= 19.5.2", + "autobahn < 22.4.1", # remove this when https://github.com/crossbario/autobahn-python/issues/1566 is fixed # Support for Python 3 transition "future >= 0.18.2", From da4deab167187ec4baf02f84da8e8ae7e03a6a8a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 May 2022 11:19:46 -0400 Subject: [PATCH 645/916] Note version with fix. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b4fd6988..d07031cd9 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn < 22.4.1", # remove this when https://github.com/crossbario/autobahn-python/issues/1566 is fixed + "autobahn < 22.4.1", # remove this when 22.4.3 is released # Support for Python 3 transition "future >= 0.18.2", From d209065a6e680eb3e42e4e5bad89655d4e3d7ec0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 May 2022 11:22:44 -0400 Subject: [PATCH 646/916] Fix type issue, and modernize slightly. --- src/allmydata/storage_client.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0f66e8e4a..c63bfccff 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -5,10 +5,6 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals # roadmap: # @@ -34,14 +30,10 @@ from __future__ import unicode_literals # # 6: implement other sorts of IStorageClient classes: S3, etc -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_text - +from typing import Union import re, time, hashlib from os import urandom -# On Python 2 this will be the backport. from configparser import NoSectionError import attr @@ -1193,7 +1185,7 @@ class _HTTPStorageServer(object): reason: bytes ): if share_type == b"immutable": - client = StorageClientImmutables(self._http_client) + client : Union[StorageClientImmutables, StorageClientMutables] = StorageClientImmutables(self._http_client) elif share_type == b"mutable": client = StorageClientMutables(self._http_client) else: From 32a11662a277b976af08194a42358e727fdf8ee8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:55:55 -0400 Subject: [PATCH 647/916] Install a specific version. --- integration/install-tor.sh | 2 +- integration/test_tor.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 66fa64cb1..97a7f9465 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -791,4 +791,4 @@ keSPmmDrjl8cySCNsMo= EOF ${SUDO} apt-get --quiet update -${SUDO} apt-get --quiet --yes install tor deb.torproject.org-keyring +${SUDO} apt-get --quiet --yes install tor=0.4.4.5-1 deb.torproject.org-keyring diff --git a/integration/test_tor.py b/integration/test_tor.py index c78fa8098..e1b25e161 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -40,6 +40,7 @@ if PY2: @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): + import time; time.sleep(3) carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) util.await_client_ready(carol, minimum_number_of_servers=2) From 04198cdb73f8db0a3e9d3aeab5554e9ce15e2750 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:56:22 -0400 Subject: [PATCH 648/916] News file. --- newsfragments/3898.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3898.minor diff --git a/newsfragments/3898.minor b/newsfragments/3898.minor new file mode 100644 index 000000000..e69de29bb From d6abefb041b58df8b019a11abda47c8dc1d6efd2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:57:29 -0400 Subject: [PATCH 649/916] Temporary always build images. --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79ce57ed0..c285263f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,14 +68,14 @@ workflows: images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 5ef8fa5b8958d31e4b79091e30b5df8c6b3d7487 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:57:50 -0400 Subject: [PATCH 650/916] TEmporary only build the image we care about. --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c285263f3..7bccb01ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,16 +88,16 @@ workflows: # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "build-image-debian-10": &DOCKERHUB_CONTEXT context: "dockerhub-auth" - - "build-image-debian-11": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-35": - <<: *DOCKERHUB_CONTEXT - - "build-image-oraclelinux-8": - <<: *DOCKERHUB_CONTEXT + # - "build-image-debian-11": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-ubuntu-18-04": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-ubuntu-20-04": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-fedora-35": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-oraclelinux-8": + # <<: *DOCKERHUB_CONTEXT # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT From 33c43cb2b3da13916b2926e2c6f6692a413058f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:01:57 -0400 Subject: [PATCH 651/916] Try a different variant. --- integration/install-tor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 97a7f9465..e4ec45e78 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -791,4 +791,4 @@ keSPmmDrjl8cySCNsMo= EOF ${SUDO} apt-get --quiet update -${SUDO} apt-get --quiet --yes install tor=0.4.4.5-1 deb.torproject.org-keyring +${SUDO} apt-get --quiet --yes install tor=0.4.4.5 deb.torproject.org-keyring From 9bef8f4abdb4a2f61b6e57f5b8476ce53835b708 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:07:40 -0400 Subject: [PATCH 652/916] This appears to be the alternative to latest version :( --- integration/install-tor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index e4ec45e78..9a8fc500e 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -791,4 +791,4 @@ keSPmmDrjl8cySCNsMo= EOF ${SUDO} apt-get --quiet update -${SUDO} apt-get --quiet --yes install tor=0.4.4.5 deb.torproject.org-keyring +${SUDO} apt-get --quiet --yes install tor=0.3.5.16-1 deb.torproject.org-keyring From 012693f6b2ae7e28daf543971d9c2341b891620f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:19:13 -0400 Subject: [PATCH 653/916] Build a different image for now. --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bccb01ec..0120c6b15 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,10 +86,10 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-10": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - # - "build-image-debian-11": - # <<: *DOCKERHUB_CONTEXT + # - "build-image-debian-10": &DOCKERHUB_CONTEXT + # context: "dockerhub-auth" + - "build-image-debian-11": + <<: *DOCKERHUB_CONTEXT # - "build-image-ubuntu-18-04": # <<: *DOCKERHUB_CONTEXT # - "build-image-ubuntu-20-04": From 28e10d127aaac62be063258b6d46c7e5451a761a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:20:37 -0400 Subject: [PATCH 654/916] Do integration tests with more modern image. --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0120c6b15..7a21a7941 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,8 +18,7 @@ workflows: - "debian-10": {} - "debian-11": - requires: - - "debian-10" + {} - "ubuntu-20-04": {} @@ -58,7 +57,7 @@ workflows: requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - - "debian-10" + - "debian-11" - "typechecks": {} @@ -297,6 +296,10 @@ jobs: integration: <<: *DEBIAN + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/debian:11-py3.9" + user: "nobody" environment: <<: *UTF_8_ENVIRONMENT From 90a6cf18ac2960a73b0fd455353e828f1b4ebd54 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:20:44 -0400 Subject: [PATCH 655/916] Just use system Tor, for more stability. --- .circleci/Dockerfile.debian | 8 +- integration/install-tor.sh | 794 ------------------------------------ 2 files changed, 2 insertions(+), 800 deletions(-) delete mode 100755 integration/install-tor.sh diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index f12f19551..abab1f4fa 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -18,15 +18,11 @@ RUN apt-get --quiet update && \ libffi-dev \ libssl-dev \ libyaml-dev \ - virtualenv + virtualenv \ + tor # Get the project source. This is better than it seems. CircleCI will # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" - -# Only the integration tests currently need this but it doesn't hurt to always -# have it present and it's simpler than building a whole extra image just for -# the integration tests. -RUN ${BUILD_SRC_ROOT}/integration/install-tor.sh diff --git a/integration/install-tor.sh b/integration/install-tor.sh deleted file mode 100755 index 9a8fc500e..000000000 --- a/integration/install-tor.sh +++ /dev/null @@ -1,794 +0,0 @@ -#!/bin/bash - -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -set -euxo pipefail - -CODENAME=$(lsb_release --short --codename) - -if [ "$(id -u)" != "0" ]; then - SUDO="sudo" -else - SUDO="" -fi - -# Script to install Tor -echo "deb http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list -echo "deb-src http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list - -# # Install Tor repo signing key -${SUDO} apt-key add - < Date: Wed, 18 May 2022 13:26:07 -0400 Subject: [PATCH 656/916] Make it work temporarily. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a21a7941..8a231ea9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,8 +85,8 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - # - "build-image-debian-10": &DOCKERHUB_CONTEXT - # context: "dockerhub-auth" + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT # - "build-image-ubuntu-18-04": From 63e16166d7e0bb6c0bd802791a841880856c6609 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:43:26 -0400 Subject: [PATCH 657/916] Restore default image building setup. --- .circleci/config.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a231ea9d..051e690b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,14 +67,14 @@ workflows: images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide @@ -89,14 +89,14 @@ workflows: context: "dockerhub-auth" - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - # - "build-image-ubuntu-18-04": - # <<: *DOCKERHUB_CONTEXT - # - "build-image-ubuntu-20-04": - # <<: *DOCKERHUB_CONTEXT - # - "build-image-fedora-35": - # <<: *DOCKERHUB_CONTEXT - # - "build-image-oraclelinux-8": - # <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-18-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-20-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-fedora-35": + <<: *DOCKERHUB_CONTEXT + - "build-image-oraclelinux-8": + <<: *DOCKERHUB_CONTEXT # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT From 02bbce81115eb0f4778f1a61f5a39f19d8f266c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:44:18 -0400 Subject: [PATCH 658/916] Get rid of spurious sleep. --- integration/test_tor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index e1b25e161..c78fa8098 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -40,7 +40,6 @@ if PY2: @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): - import time; time.sleep(3) carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) util.await_client_ready(carol, minimum_number_of_servers=2) From 8c8ea4927f4c015f391913b29a47300f69cdefcd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:07:55 -0400 Subject: [PATCH 659/916] Switch to public API. --- src/allmydata/storage/http_server.py | 8 +++++--- src/allmydata/storage/server.py | 25 +++++++++++++++---------- src/allmydata/test/test_repairer.py | 2 +- src/allmydata/test/test_storage.py | 4 ++-- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 709c1fda5..033f9ec4c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -538,8 +538,8 @@ class HTTPServer(object): methods=["PUT"], ) def add_or_renew_lease(self, request, authorization, storage_index): - """Update the lease for an immutable share.""" - if not list(self._storage_server._get_bucket_shares(storage_index)): + """Update the lease for an immutable or mutable share.""" + if not list(self._storage_server.get_shares(storage_index)): raise _HTTPError(http.NOT_FOUND) # Checking of the renewal secret is done by the backend. @@ -674,7 +674,9 @@ class HTTPServer(object): ): """Indicate that given share is corrupt, with a text reason.""" # TODO unit test all the paths - if not self._storage_server._share_exists(storage_index, share_number): + if share_number not in { + shnum for (shnum, _) in self._storage_server.get_shares(storage_index) + }: raise _HTTPError(http.NOT_FOUND) info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index bcf44dc30..07b82b4d8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -3,7 +3,7 @@ Ported to Python 3. """ from __future__ import annotations from future.utils import bytes_to_native_str -from typing import Dict, Tuple +from typing import Dict, Tuple, Iterable import os, re @@ -321,7 +321,7 @@ class StorageServer(service.MultiService): # they asked about: this will save them a lot of work. Add or update # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. - for (shnum, fn) in self._get_bucket_shares(storage_index): + for (shnum, fn) in self.get_shares(storage_index): alreadygot[shnum] = ShareFile(fn) if renew_leases: self._add_or_renew_leases(alreadygot.values(), lease_info) @@ -363,7 +363,7 @@ class StorageServer(service.MultiService): return set(alreadygot), bucketwriters def _iter_share_files(self, storage_index): - for shnum, filename in self._get_bucket_shares(storage_index): + for shnum, filename in self.get_shares(storage_index): with open(filename, 'rb') as f: header = f.read(32) if MutableShareFile.is_valid_header(header): @@ -416,10 +416,12 @@ class StorageServer(service.MultiService): """ self._call_on_bucket_writer_close.append(handler) - def _get_bucket_shares(self, storage_index): - """Return a list of (shnum, pathname) tuples for files that hold + def get_shares(self, storage_index) -> Iterable[(int, str)]: + """ + Return an iterable of (shnum, pathname) tuples for files that hold shares for this storage_index. In each tuple, 'shnum' will always be - the integer form of the last component of 'pathname'.""" + the integer form of the last component of 'pathname'. + """ storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) try: for f in os.listdir(storagedir): @@ -431,12 +433,15 @@ class StorageServer(service.MultiService): pass def get_buckets(self, storage_index): + """ + Get ``BucketReaders`` for an immutable. + """ start = self._clock.seconds() self.count("get") si_s = si_b2a(storage_index) log.msg("storage: get_buckets %r" % si_s) bucketreaders = {} # k: sharenum, v: BucketReader - for shnum, filename in self._get_bucket_shares(storage_index): + for shnum, filename in self.get_shares(storage_index): bucketreaders[shnum] = BucketReader(self, filename, storage_index, shnum) self.add_latency("get", self._clock.seconds() - start) @@ -453,7 +458,7 @@ class StorageServer(service.MultiService): # since all shares get the same lease data, we just grab the leases # from the first share try: - shnum, filename = next(self._get_bucket_shares(storage_index)) + shnum, filename = next(self.get_shares(storage_index)) sf = ShareFile(filename) return sf.get_leases() except StopIteration: @@ -467,7 +472,7 @@ class StorageServer(service.MultiService): :return: An iterable of the leases attached to this slot. """ - for _, share_filename in self._get_bucket_shares(storage_index): + for _, share_filename in self.get_shares(storage_index): share = MutableShareFile(share_filename) return share.get_leases() return [] @@ -742,7 +747,7 @@ class StorageServer(service.MultiService): :return bool: ``True`` if a share with the given number exists at the given storage index, ``False`` otherwise. """ - for existing_sharenum, ignored in self._get_bucket_shares(storage_index): + for existing_sharenum, ignored in self.get_shares(storage_index): if existing_sharenum == shnum: return True return False diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index 88696000c..f9b93af72 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -717,7 +717,7 @@ class Repairer(GridTestMixin, unittest.TestCase, RepairTestMixin, ss = self.g.servers_by_number[0] # we want to delete the share corresponding to the server # we're making not-respond - share = next(ss._get_bucket_shares(self.c0_filenode.get_storage_index()))[0] + share = next(ss.get_shares(self.c0_filenode.get_storage_index()))[0] self.delete_shares_numbered(self.uri, [share]) return self.c0_filenode.check_and_repair(Monitor()) d.addCallback(_then) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 65d09de25..91d55790e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -766,7 +766,7 @@ class Server(unittest.TestCase): writer.close() # It should have a lease granted at the current time. - shares = dict(ss._get_bucket_shares(storage_index)) + shares = dict(ss.get_shares(storage_index)) self.assertEqual( [first_lease], list( @@ -789,7 +789,7 @@ class Server(unittest.TestCase): writer.close() # The first share's lease expiration time is unchanged. - shares = dict(ss._get_bucket_shares(storage_index)) + shares = dict(ss.get_shares(storage_index)) self.assertEqual( [first_lease], list( From 12927d50bafdb37d7dd0a53443b5d61ee6364be7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:09:04 -0400 Subject: [PATCH 660/916] Type annotation improvements. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 167d2394a..de9dfc518 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -358,7 +358,7 @@ class StorageClientGeneral(object): @inlineCallbacks def add_or_renew_lease( self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes - ): + ) -> Deferred[None]: """ Add or renew a lease. diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 07b82b4d8..0a1999dfb 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -416,7 +416,7 @@ class StorageServer(service.MultiService): """ self._call_on_bucket_writer_close.append(handler) - def get_shares(self, storage_index) -> Iterable[(int, str)]: + def get_shares(self, storage_index) -> Iterable[tuple[int, str]]: """ Return an iterable of (shnum, pathname) tuples for files that hold shares for this storage_index. In each tuple, 'shnum' will always be From 63624eedec9d50dd93c2812482fb6fa977e1e096 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:33:02 -0400 Subject: [PATCH 661/916] Reduce code duplication. --- src/allmydata/storage/http_server.py | 53 +++++++++++++--------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 033f9ec4c..a4f67bb5e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,6 +2,7 @@ HTTP server for storage. """ +from __future__ import annotations from typing import Dict, List, Set, Tuple, Any from functools import wraps @@ -273,6 +274,28 @@ _SCHEMAS = { } +# TODO unit tests? or rely on higher-level tests +def parse_range(request) -> tuple[int, int]: + """ + Parse the subset of ``Range`` headers we support: bytes only, only a single + range, the end must be explicitly specified. Raises a + ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not + possible or the header isn't set. + + Returns tuple of (start_offset, end_offset). + """ + range_header = parse_range_header(request.getHeader("range")) + if ( + range_header is None + or range_header.units != "bytes" + or len(range_header.ranges) > 1 # more than one range + or range_header.ranges[0][1] is None # range without end + ): + raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) + + return range_header.ranges[0] + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -505,17 +528,7 @@ class HTTPServer(object): request.write(data) start += len(data) - range_header = parse_range_header(request.getHeader("range")) - if ( - range_header is None - or range_header.units != "bytes" - or len(range_header.ranges) > 1 # more than one range - or range_header.ranges[0][1] is None # range without end - ): - request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) - return b"" - - offset, end = range_header.ranges[0] + offset, end = parse_range(request) # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 @@ -617,23 +630,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - if request.getHeader("range") is None: - # TODO in follow-up ticket - raise NotImplementedError() - - # TODO reduce duplication with immutable reads? - # TODO unit tests, perhaps shared if possible - range_header = parse_range_header(request.getHeader("range")) - if ( - range_header is None - or range_header.units != "bytes" - or len(range_header.ranges) > 1 # more than one range - or range_header.ranges[0][1] is None # range without end - ): - request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) - return b"" - - offset, end = range_header.ranges[0] + offset, end = parse_range(request) # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 From 2313195c2b94db799c93083580b643c85fabef47 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:43:42 -0400 Subject: [PATCH 662/916] Reduce duplication. --- src/allmydata/storage/http_server.py | 75 +++++++++++++--------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a4f67bb5e..2ff0c6908 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,7 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any +from typing import Dict, List, Set, Tuple, Any, Callable from functools import wraps from base64 import b64decode @@ -275,14 +275,20 @@ _SCHEMAS = { # TODO unit tests? or rely on higher-level tests -def parse_range(request) -> tuple[int, int]: +def read_range(request, read_data: Callable[int, int, bytes]) -> bytes: """ - Parse the subset of ``Range`` headers we support: bytes only, only a single - range, the end must be explicitly specified. Raises a - ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not - possible or the header isn't set. + Parse the ``Range`` header, read appropriately, return as result. - Returns tuple of (start_offset, end_offset). + Only parses a subset of ``Range`` headers that we support: must be set, + bytes only, only a single range, the end must be explicitly specified. + Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is + not possible or the header isn't set. + + Returns the bytes to return from the request handler, and sets appropriate + response headers. + + Takes a function that will do the actual reading given the start offset and + a length to read. """ range_header = parse_range_header(request.getHeader("range")) if ( @@ -293,7 +299,21 @@ def parse_range(request) -> tuple[int, int]: ): raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) - return range_header.ranges[0] + offset, end = range_header.ranges[0] + + # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + data = read_data(offset, end - offset) + + request.setResponseCode(http.PARTIAL_CONTENT) + if len(data): + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) + return data class HTTPServer(object): @@ -528,21 +548,7 @@ class HTTPServer(object): request.write(data) start += len(data) - offset, end = parse_range(request) - - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = bucket.read(offset, end - offset) - - request.setResponseCode(http.PARTIAL_CONTENT) - if len(data): - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) - return data + return read_range(request, bucket.read) @_authorized_route( _app, @@ -630,24 +636,15 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - offset, end = parse_range(request) + if request.getHeader("range") is None: + raise NotImplementedError() # should be able to move shared implementation into read_range()... - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = self._storage_server.slot_readv( - storage_index, [share_number], [(offset, end - offset)] - )[share_number][0] + def read_data(offset, length): + return self._storage_server.slot_readv( + storage_index, [share_number], [(offset, length)] + )[share_number][0] - # TODO reduce duplication? - request.setResponseCode(http.PARTIAL_CONTENT) - if len(data): - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) - return data + return read_range(request, read_data) @_authorized_route( _app, From fd306b9a61b2b4806c948ca0f2b2e7fe1f447110 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Jun 2022 13:54:54 -0400 Subject: [PATCH 663/916] Share more code across mutable and immutable reads. --- src/allmydata/storage/http_server.py | 38 +++++++++++++--------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2ff0c6908..b031cbb15 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -275,7 +275,7 @@ _SCHEMAS = { # TODO unit tests? or rely on higher-level tests -def read_range(request, read_data: Callable[int, int, bytes]) -> bytes: +def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes]: """ Parse the ``Range`` header, read appropriately, return as result. @@ -284,15 +284,28 @@ def read_range(request, read_data: Callable[int, int, bytes]) -> bytes: Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not possible or the header isn't set. - Returns the bytes to return from the request handler, and sets appropriate - response headers. + Returns a result that should be returned from the request handler, and sets + appropriate response headers. Takes a function that will do the actual reading given the start offset and a length to read. """ + if request.getHeader("range") is None: + # Return the whole thing. + start = 0 + while True: + # TODO should probably yield to event loop occasionally... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + data = read_data(start, start + 65536) + if not data: + request.finish() + return + request.write(data) + start += len(data) + range_header = parse_range_header(request.getHeader("range")) if ( - range_header is None + range_header is None # failed to parse or range_header.units != "bytes" or len(range_header.ranges) > 1 # more than one range or range_header.ranges[0][1] is None # range without end @@ -535,19 +548,6 @@ class HTTPServer(object): request.setResponseCode(http.NOT_FOUND) return b"" - if request.getHeader("range") is None: - # Return the whole thing. - start = 0 - while True: - # TODO should probably yield to event loop occasionally... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = bucket.read(start, start + 65536) - if not data: - request.finish() - return - request.write(data) - start += len(data) - return read_range(request, bucket.read) @_authorized_route( @@ -636,9 +636,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - if request.getHeader("range") is None: - raise NotImplementedError() # should be able to move shared implementation into read_range()... - + # TODO unit tests def read_data(offset, length): return self._storage_server.slot_readv( storage_index, [share_number], [(offset, length)] From f1384096fa26c97e5c3f88a9d8fb0212c7f34c16 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jun 2022 13:46:23 -0400 Subject: [PATCH 664/916] First unit test for mutables. --- src/allmydata/storage/http_client.py | 6 +- src/allmydata/storage/http_server.py | 3 +- src/allmydata/test/test_storage_http.py | 83 ++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index de9dfc518..bf6104dea 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -7,7 +7,7 @@ from __future__ import annotations from typing import Union, Optional, Sequence, Mapping from base64 import b64encode -from attrs import define, asdict, frozen +from attrs import define, asdict, frozen, field # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -646,8 +646,8 @@ class ReadVector: class TestWriteVectors: """Test and write vectors for a specific share.""" - test_vectors: Sequence[TestVector] - write_vectors: Sequence[WriteVector] + test_vectors: Sequence[TestVector] = field(factory=list) + write_vectors: Sequence[WriteVector] = field(factory=list) new_length: Optional[int] = None def asdict(self) -> dict: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b031cbb15..9735a0626 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -263,7 +263,7 @@ _SCHEMAS = { * share_number: { "test": [* {"offset": uint, "size": uint, "specimen": bstr}] "write": [* {"offset": uint, "data": bstr}] - "new-length": uint // null + "new-length": uint / null } } "read-vector": [* {"offset": uint, "size": uint}] @@ -274,7 +274,6 @@ _SCHEMAS = { } -# TODO unit tests? or rely on higher-level tests def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes]: """ Parse the ``Range`` header, read appropriately, return as result. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index fcc2401f2..37e3be8a7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -40,6 +40,10 @@ from ..storage.http_client import ( UploadProgress, StorageClientGeneral, _encode_si, + StorageClientMutables, + TestWriteVectors, + WriteVector, + ReadVector, ) @@ -1109,7 +1113,9 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = urandom(16) secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.general_client.add_or_renew_lease(storage_index, secret, secret)) + result_of( + self.general_client.add_or_renew_lease(storage_index, secret, secret) + ) def test_advise_corrupt_share(self): """ @@ -1142,3 +1148,78 @@ class ImmutableHTTPAPITests(SyncTestCase): result_of( self.imm_client.advise_corrupt_share(si, share_number, reason) ) + + +class MutableHTTPAPIsTests(SyncTestCase): + """Tests for mutable APIs.""" + + def setUp(self): + super(MutableHTTPAPIsTests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + self.mut_client = StorageClientMutables(self.http.client) + + def create_upload(self, data=b"abcdef"): + """ + Utility that creates shares 0 and 1 with bodies + ``{data}-{share_number}``. + """ + write_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=0, data=data + b"-0")] + ), + 1: TestWriteVectors( + write_vectors=[ + WriteVector(offset=0, data=data), + WriteVector(offset=len(data), data=b"-1"), + ] + ), + }, + [ReadVector(0, len(data) + 2)], + ) + ) + return storage_index, write_secret, lease_secret + + def test_upload_can_be_downloaded(self): + """ + Written data can be read, both by the combo operation and a direct + read. + """ + storage_index, _, _ = self.create_upload() + data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7)) + data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1")) + + def test_read_before_write(self): + """In combo read/test/write operation, reads happen before writes.""" + + def test_conditional_upload(self): + pass + + def test_list_shares(self): + pass + + def test_wrong_write_enabler(self): + pass + + # TODO refactor reads tests so they're shared + + def test_lease_renew_and_add(self): + pass + + def test_lease_on_unknown_storage_index(self): + pass + + def test_advise_corrupt_share(self): + pass + + def test_advise_corrupt_share_unknown(self): + pass From 3e67d2d7890ceb7f537fdfeda89072e1d99c1835 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 09:50:36 -0400 Subject: [PATCH 665/916] More tests. --- src/allmydata/test/test_storage_http.py | 78 +++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 37e3be8a7..6cf2f883b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -44,6 +44,8 @@ from ..storage.http_client import ( TestWriteVectors, WriteVector, ReadVector, + ReadTestWriteResult, + TestVector, ) @@ -1183,15 +1185,14 @@ class MutableHTTPAPIsTests(SyncTestCase): ] ), }, - [ReadVector(0, len(data) + 2)], + [], ) ) return storage_index, write_secret, lease_secret - def test_upload_can_be_downloaded(self): + def test_write_can_be_read(self): """ - Written data can be read, both by the combo operation and a direct - read. + Written data can be read using ``read_share_chunk``. """ storage_index, _, _ = self.create_upload() data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7)) @@ -1200,9 +1201,74 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_read_before_write(self): """In combo read/test/write operation, reads happen before writes.""" + storage_index, write_secret, lease_secret = self.create_upload() + result = result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=1, data=b"XYZ")] + ), + }, + [ReadVector(0, 8)], + ) + ) + # Reads are from before the write: + self.assertEqual( + result, + ReadTestWriteResult( + success=True, reads={0: [b"abcdef-0"], 1: [b"abcdef-1"]} + ), + ) + # But the write did happen: + data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)) + data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + self.assertEqual((data0, data1), (b"aXYZef-0", b"abcdef-1")) - def test_conditional_upload(self): - pass + def test_conditional_write(self): + """Uploads only happen if the test passes.""" + storage_index, write_secret, lease_secret = self.create_upload() + result_failed = result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + test_vectors=[TestVector(1, 4, b"FAIL")], + write_vectors=[WriteVector(offset=1, data=b"XYZ")], + ), + }, + [], + ) + ) + self.assertFalse(result_failed.success) + + # This time the test matches: + result = result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + test_vectors=[TestVector(1, 4, b"bcde")], + write_vectors=[WriteVector(offset=1, data=b"XYZ")], + ), + }, + [ReadVector(0, 8)], + ) + ) + self.assertTrue(result.success) + self.assertEqual( + result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + b"aXYZef-0", + ) def test_list_shares(self): pass From 797f34aec32eefbe495d9936d955e7ab4bdc3f1a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 09:59:12 -0400 Subject: [PATCH 666/916] More tests. --- src/allmydata/storage/http_client.py | 3 --- src/allmydata/test/test_storage_http.py | 35 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index bf6104dea..9711e748d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -696,7 +696,6 @@ class StorageClientMutables: Given a mapping between share numbers and test/write vectors, the tests are done and if they are valid the writes are done. """ - # TODO unit test all the things url = self._client.relative_url( "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) ) @@ -731,7 +730,6 @@ class StorageClientMutables: """ Download a chunk of data from a share. """ - # TODO unit test all the things return read_share_chunk( self._client, "mutable", storage_index, share_number, offset, length ) @@ -741,7 +739,6 @@ class StorageClientMutables: """ List the share numbers for a given storage index. """ - # TODO unit test all the things url = self._client.relative_url( "/v1/mutable/{}/shares".format(_encode_si(storage_index)) ) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6cf2f883b..65aa12e40 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1271,10 +1271,41 @@ class MutableHTTPAPIsTests(SyncTestCase): ) def test_list_shares(self): - pass + """``list_shares()`` returns the shares for a given storage index.""" + storage_index, _, _ = self.create_upload() + self.assertEqual(result_of(self.mut_client.list_shares(storage_index)), {0, 1}) + + def test_non_existent_list_shares(self): + """A non-existent storage index errors when shares are listed.""" + with self.assertRaises(ClientException) as exc: + result_of(self.mut_client.list_shares(urandom(32))) + self.assertEqual(exc.exception.code, http.NOT_FOUND) def test_wrong_write_enabler(self): - pass + """Writes with the wrong write enabler fail, and are not processed.""" + storage_index, write_secret, lease_secret = self.create_upload() + with self.assertRaises(ClientException) as exc: + result_of( + self.mut_client.read_test_write_chunks( + storage_index, + urandom(32), + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=1, data=b"XYZ")] + ), + }, + [ReadVector(0, 8)], + ) + ) + self.assertEqual(exc.exception.code, http.UNAUTHORIZED) + + # The write did not happen: + self.assertEqual( + result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + b"abcdef-0", + ) # TODO refactor reads tests so they're shared From e6efb62fd19eef08f14916438240f70bc197a4c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 10:25:06 -0400 Subject: [PATCH 667/916] Refactor immutable tests so they can shared with mutables. --- src/allmydata/test/test_storage_http.py | 460 +++++++++++++----------- 1 file changed, 246 insertions(+), 214 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 65aa12e40..fc79fbe34 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -5,7 +5,7 @@ Tests for HTTP storage client + server. from base64 import b64encode from contextlib import contextmanager from os import urandom - +from typing import Union, Callable, Tuple from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -787,141 +787,6 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) - def upload(self, share_number, data_length=26): - """ - Create a share, return (storage_index, uploaded_data). - """ - uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ - :data_length - ] - (upload_secret, _, storage_index, _) = self.create_upload( - {share_number}, data_length - ) - result_of( - self.imm_client.write_share_chunk( - storage_index, - share_number, - upload_secret, - 0, - uploaded_data, - ) - ) - return storage_index, uploaded_data - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( - self.imm_client.read_share_chunk( - b"1" * 16, - 1, - 0, - 10, - ) - ) - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - storage_index, _ = self.upload(1) - with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( - self.imm_client.read_share_chunk( - storage_index, - 7, # different share number - 0, - 10, - ) - ) - - def test_read_with_negative_offset_fails(self): - """ - Malformed or unsupported Range headers result in 416 (requested range - not satisfiable) error. - """ - storage_index, _ = self.upload(1) - - def check_bad_range(bad_range_value): - client = StorageClientImmutables( - StorageClientWithHeadersOverride( - self.http.client, {"range": bad_range_value} - ) - ) - - with assert_fails_with_http_code( - self, http.REQUESTED_RANGE_NOT_SATISFIABLE - ): - result_of( - client.read_share_chunk( - storage_index, - 1, - 0, - 10, - ) - ) - - # Bad unit - check_bad_range("molluscs=0-9") - # Negative offsets - check_bad_range("bytes=-2-9") - check_bad_range("bytes=0--10") - # Negative offset no endpoint - check_bad_range("bytes=-300-") - check_bad_range("bytes=") - # Multiple ranges are currently unsupported, even if they're - # semantically valid under HTTP: - check_bad_range("bytes=0-5, 6-7") - # Ranges without an end are currently unsupported, even if they're - # semantically valid under HTTP. - check_bad_range("bytes=0-") - - @given(data_length=st.integers(min_value=1, max_value=300000)) - def test_read_with_no_range(self, data_length): - """ - A read with no range returns the whole immutable. - """ - storage_index, uploaded_data = self.upload(1, data_length) - response = result_of( - self.http.client.request( - "GET", - self.http.client.relative_url( - "/v1/immutable/{}/1".format(_encode_si(storage_index)) - ), - ) - ) - self.assertEqual(response.code, http.OK) - self.assertEqual(result_of(response.content()), uploaded_data) - - def test_validate_content_range_response_to_read(self): - """ - The server responds to ranged reads with an appropriate Content-Range - header. - """ - storage_index, _ = self.upload(1, 26) - - def check_range(requested_range, expected_response): - headers = Headers() - headers.setRawHeaders("range", [requested_range]) - response = result_of( - self.http.client.request( - "GET", - self.http.client.relative_url( - "/v1/immutable/{}/1".format(_encode_si(storage_index)) - ), - headers=headers, - ) - ) - self.assertEqual( - response.headers.getRawHeaders("content-range"), [expected_response] - ) - - check_range("bytes=0-10", "bytes 0-10/*") - # Can't go beyond the end of the immutable! - check_range("bytes=10-100", "bytes 10-25/*") - def test_timed_out_upload_allows_reupload(self): """ If an in-progress upload times out, it is cancelled altogether, @@ -1062,52 +927,6 @@ class ImmutableHTTPAPITests(SyncTestCase): ), ) - def test_lease_renew_and_add(self): - """ - It's possible the renew the lease on an uploaded immutable, by using - the same renewal secret, or add a new lease by choosing a different - renewal secret. - """ - # Create immutable: - (upload_secret, lease_secret, storage_index, _) = self.create_upload({0}, 100) - result_of( - self.imm_client.write_share_chunk( - storage_index, - 0, - upload_secret, - 0, - b"A" * 100, - ) - ) - - [lease] = self.http.storage_server.get_leases(storage_index) - initial_expiration_time = lease.get_expiration_time() - - # Time passes: - self.http.clock.advance(167) - - # We renew the lease: - result_of( - self.general_client.add_or_renew_lease( - storage_index, lease_secret, lease_secret - ) - ) - - # More time passes: - self.http.clock.advance(10) - - # We create a new lease: - lease_secret2 = urandom(32) - result_of( - self.general_client.add_or_renew_lease( - storage_index, lease_secret2, lease_secret2 - ) - ) - - [lease1, lease2] = self.http.storage_server.get_leases(storage_index) - self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) - self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) - def test_lease_on_unknown_storage_index(self): """ An attempt to renew an unknown storage index will result in a HTTP 404. @@ -1119,38 +938,6 @@ class ImmutableHTTPAPITests(SyncTestCase): self.general_client.add_or_renew_lease(storage_index, secret, secret) ) - def test_advise_corrupt_share(self): - """ - Advising share was corrupted succeeds from HTTP client's perspective, - and calls appropriate method on server. - """ - corrupted = [] - self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append( - args - ) - - storage_index, _ = self.upload(13) - reason = "OHNO \u1235" - result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) - - self.assertEqual( - corrupted, [(b"immutable", storage_index, 13, reason.encode("utf-8"))] - ) - - def test_advise_corrupt_share_unknown(self): - """ - Advising an unknown share was corrupted results in 404. - """ - storage_index, _ = self.upload(13) - reason = "OHNO \u1235" - result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) - - for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: - with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( - self.imm_client.advise_corrupt_share(si, share_number, reason) - ) - class MutableHTTPAPIsTests(SyncTestCase): """Tests for mutable APIs.""" @@ -1320,3 +1107,248 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_advise_corrupt_share_unknown(self): pass + + +class SharedImmutableMutableTestsMixin: + """ + Shared tests for mutables and immutables where the API is the same. + """ + + KIND: str # either "mutable" or "immutable" + general_client: StorageClientGeneral + client: Union[StorageClientImmutables, StorageClientMutables] + clientFactory: Callable[ + StorageClient, Union[StorageClientImmutables, StorageClientMutables] + ] + + def upload(self, share_number: int, data_length=26) -> Tuple[bytes, bytes, bytes]: + """ + Create a share, return (storage_index, uploaded_data, lease secret). + """ + raise NotImplementedError + + def test_advise_corrupt_share(self): + """ + Advising share was corrupted succeeds from HTTP client's perspective, + and calls appropriate method on server. + """ + corrupted = [] + self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append( + args + ) + + storage_index, _, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + + self.assertEqual( + corrupted, + [(self.KIND.encode("ascii"), storage_index, 13, reason.encode("utf-8"))], + ) + + def test_advise_corrupt_share_unknown(self): + """ + Advising an unknown share was corrupted results in 404. + """ + storage_index, _, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + + for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of(self.client.advise_corrupt_share(si, share_number, reason)) + + def test_lease_renew_and_add(self): + """ + It's possible the renew the lease on an uploaded immutable, by using + the same renewal secret, or add a new lease by choosing a different + renewal secret. + """ + # Create a storage index: + storage_index, _, lease_secret = self.upload(0) + + [lease] = self.http.storage_server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.http.clock.advance(167) + + # We renew the lease: + result_of( + self.general_client.add_or_renew_lease( + storage_index, lease_secret, lease_secret + ) + ) + + # More time passes: + self.http.clock.advance(10) + + # We create a new lease: + lease_secret2 = urandom(32) + result_of( + self.general_client.add_or_renew_lease( + storage_index, lease_secret2, lease_secret2 + ) + ) + + [lease1, lease2] = self.http.storage_server.get_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) + self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of( + self.client.read_share_chunk( + b"1" * 16, + 1, + 0, + 10, + ) + ) + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + storage_index, _, _ = self.upload(1) + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of( + self.client.read_share_chunk( + storage_index, + 7, # different share number + 0, + 10, + ) + ) + + def test_read_with_negative_offset_fails(self): + """ + Malformed or unsupported Range headers result in 416 (requested range + not satisfiable) error. + """ + storage_index, _, _ = self.upload(1) + + def check_bad_range(bad_range_value): + client = StorageClientImmutables( + StorageClientWithHeadersOverride( + self.http.client, {"range": bad_range_value} + ) + ) + + with assert_fails_with_http_code( + self, http.REQUESTED_RANGE_NOT_SATISFIABLE + ): + result_of( + client.read_share_chunk( + storage_index, + 1, + 0, + 10, + ) + ) + + # Bad unit + check_bad_range("molluscs=0-9") + # Negative offsets + check_bad_range("bytes=-2-9") + check_bad_range("bytes=0--10") + # Negative offset no endpoint + check_bad_range("bytes=-300-") + check_bad_range("bytes=") + # Multiple ranges are currently unsupported, even if they're + # semantically valid under HTTP: + check_bad_range("bytes=0-5, 6-7") + # Ranges without an end are currently unsupported, even if they're + # semantically valid under HTTP. + check_bad_range("bytes=0-") + + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole immutable. + """ + storage_index, uploaded_data, _ = self.upload(1, data_length) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + ), + ) + ) + self.assertEqual(response.code, http.OK) + self.assertEqual(result_of(response.content()), uploaded_data) + + def test_validate_content_range_response_to_read(self): + """ + The server responds to ranged reads with an appropriate Content-Range + header. + """ + storage_index, _, _ = self.upload(1, 26) + + def check_range(requested_range, expected_response): + headers = Headers() + headers.setRawHeaders("range", [requested_range]) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + ), + headers=headers, + ) + ) + self.assertEqual( + response.headers.getRawHeaders("content-range"), [expected_response] + ) + + check_range("bytes=0-10", "bytes 0-10/*") + # Can't go beyond the end of the immutable! + check_range("bytes=10-100", "bytes 10-25/*") + + +class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): + """Shared tests, running on immutables.""" + + KIND = "immutable" + clientFactory = StorageClientImmutables + + def setUp(self): + super(ImmutableSharedTests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + self.client = self.clientFactory(self.http.client) + self.general_client = StorageClientGeneral(self.http.client) + + def upload(self, share_number, data_length=26): + """ + Create a share, return (storage_index, uploaded_data). + """ + uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ + :data_length + ] + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + result_of( + self.client.create( + storage_index, + {share_number}, + data_length, + upload_secret, + lease_secret, + lease_secret, + ) + ) + result_of( + self.client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + uploaded_data, + ) + ) + return storage_index, uploaded_data, lease_secret From 85774ced9526df771ed4b0ec14bc4fe83eaeb1dd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 10:56:37 -0400 Subject: [PATCH 668/916] Run shared tests on mutables too, with appropriate fixes to the tests and the server. --- src/allmydata/storage/http_server.py | 12 ++-- src/allmydata/test/test_storage_http.py | 82 +++++++++++++++++-------- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 9735a0626..46023be72 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -599,7 +599,6 @@ class HTTPServer(object): ) def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - # TODO unit tests rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) secrets = ( authorization[Secrets.WRITE_ENABLER], @@ -635,11 +634,13 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - # TODO unit tests def read_data(offset, length): - return self._storage_server.slot_readv( - storage_index, [share_number], [(offset, length)] - )[share_number][0] + try: + return self._storage_server.slot_readv( + storage_index, [share_number], [(offset, length)] + )[share_number][0] + except KeyError: + raise _HTTPError(http.NOT_FOUND) return read_range(request, read_data) @@ -664,7 +665,6 @@ class HTTPServer(object): self, request, authorization, storage_index, share_number ): """Indicate that given share is corrupt, with a text reason.""" - # TODO unit test all the paths if share_number not in { shnum for (shnum, _) in self._storage_server.get_shares(storage_index) }: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index fc79fbe34..7ed4cd235 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -5,7 +5,7 @@ Tests for HTTP storage client + server. from base64 import b64encode from contextlib import contextmanager from os import urandom -from typing import Union, Callable, Tuple +from typing import Union, Callable, Tuple, Iterable from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -23,6 +23,7 @@ from werkzeug.exceptions import NotFound as WNotFound from .common import SyncTestCase from ..storage.http_common import get_content_type, CBOR_MIME_TYPE from ..storage.common import si_b2a +from ..storage.lease import LeaseInfo from ..storage.server import StorageServer from ..storage.http_server import ( HTTPServer, @@ -1094,20 +1095,6 @@ class MutableHTTPAPIsTests(SyncTestCase): b"abcdef-0", ) - # TODO refactor reads tests so they're shared - - def test_lease_renew_and_add(self): - pass - - def test_lease_on_unknown_storage_index(self): - pass - - def test_advise_corrupt_share(self): - pass - - def test_advise_corrupt_share_unknown(self): - pass - class SharedImmutableMutableTestsMixin: """ @@ -1127,6 +1114,10 @@ class SharedImmutableMutableTestsMixin: """ raise NotImplementedError + def get_leases(self, storage_index: bytes) -> Iterable[LeaseInfo]: + """Get leases for the storage index.""" + raise NotImplementedError() + def test_advise_corrupt_share(self): """ Advising share was corrupted succeeds from HTTP client's perspective, @@ -1160,14 +1151,14 @@ class SharedImmutableMutableTestsMixin: def test_lease_renew_and_add(self): """ - It's possible the renew the lease on an uploaded immutable, by using - the same renewal secret, or add a new lease by choosing a different - renewal secret. + It's possible the renew the lease on an uploaded mutable/immutable, by + using the same renewal secret, or add a new lease by choosing a + different renewal secret. """ # Create a storage index: storage_index, _, lease_secret = self.upload(0) - [lease] = self.http.storage_server.get_leases(storage_index) + [lease] = self.get_leases(storage_index) initial_expiration_time = lease.get_expiration_time() # Time passes: @@ -1191,7 +1182,7 @@ class SharedImmutableMutableTestsMixin: ) ) - [lease1, lease2] = self.http.storage_server.get_leases(storage_index) + [lease1, lease2] = self.get_leases(storage_index) self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) @@ -1232,7 +1223,7 @@ class SharedImmutableMutableTestsMixin: storage_index, _, _ = self.upload(1) def check_bad_range(bad_range_value): - client = StorageClientImmutables( + client = self.clientFactory( StorageClientWithHeadersOverride( self.http.client, {"range": bad_range_value} ) @@ -1268,7 +1259,7 @@ class SharedImmutableMutableTestsMixin: @given(data_length=st.integers(min_value=1, max_value=300000)) def test_read_with_no_range(self, data_length): """ - A read with no range returns the whole immutable. + A read with no range returns the whole mutable/immutable. """ storage_index, uploaded_data, _ = self.upload(1, data_length) response = result_of( @@ -1306,7 +1297,7 @@ class SharedImmutableMutableTestsMixin: ) check_range("bytes=0-10", "bytes 0-10/*") - # Can't go beyond the end of the immutable! + # Can't go beyond the end of the mutable/immutable! check_range("bytes=10-100", "bytes 10-25/*") @@ -1324,7 +1315,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def upload(self, share_number, data_length=26): """ - Create a share, return (storage_index, uploaded_data). + Create a share, return (storage_index, uploaded_data, lease_secret). """ uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ :data_length @@ -1352,3 +1343,46 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): ) ) return storage_index, uploaded_data, lease_secret + + def get_leases(self, storage_index): + return self.http.storage_server.get_leases(storage_index) + + +class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): + """Shared tests, running on mutables.""" + + KIND = "mutable" + clientFactory = StorageClientMutables + + def setUp(self): + super(MutableSharedTests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + self.client = self.clientFactory(self.http.client) + self.general_client = StorageClientGeneral(self.http.client) + + def upload(self, share_number, data_length=26): + """ + Create a share, return (storage_index, uploaded_data, lease_secret). + """ + data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[:data_length] + write_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + result_of( + self.client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + share_number: TestWriteVectors( + write_vectors=[WriteVector(offset=0, data=data)] + ), + }, + [], + ) + ) + return storage_index, data, lease_secret + + def get_leases(self, storage_index): + return self.http.storage_server.get_slot_leases(storage_index) From ca0f311861aaefb2ae532abc6299b668d166862e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 10:59:29 -0400 Subject: [PATCH 669/916] News file. --- newsfragments/3896.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3896.minor diff --git a/newsfragments/3896.minor b/newsfragments/3896.minor new file mode 100644 index 000000000..e69de29bb From c3a304e1cc0d2eadd62017cbdea367063ec5bed2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 11:00:07 -0400 Subject: [PATCH 670/916] Lint and mypy fixes. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 6 +++--- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9711e748d..9203d02ab 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -586,7 +586,7 @@ class StorageClientImmutables(object): ) @inlineCallbacks - def list_shares(self, storage_index): # type: (bytes,) -> Deferred[set[int]] + def list_shares(self, storage_index: bytes) -> Deferred[set[int]]: """ Return the set of shares for a given storage index. """ diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 46023be72..bcad0e972 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,7 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable +from typing import Dict, List, Set, Tuple, Any, Callable, Optional from functools import wraps from base64 import b64decode @@ -274,7 +274,7 @@ _SCHEMAS = { } -def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes]: +def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[bytes]: """ Parse the ``Range`` header, read appropriately, return as result. @@ -298,7 +298,7 @@ def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes] data = read_data(start, start + 65536) if not data: request.finish() - return + return None request.write(data) start += len(data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 7ed4cd235..5e0b35d88 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1105,7 +1105,7 @@ class SharedImmutableMutableTestsMixin: general_client: StorageClientGeneral client: Union[StorageClientImmutables, StorageClientMutables] clientFactory: Callable[ - StorageClient, Union[StorageClientImmutables, StorageClientMutables] + [StorageClient], Union[StorageClientImmutables, StorageClientMutables] ] def upload(self, share_number: int, data_length=26) -> Tuple[bytes, bytes, bytes]: From 528d902460ae5bddaf3f140743072b3324fca5b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 11:15:25 -0400 Subject: [PATCH 671/916] News file. --- newsfragments/3900.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3900.minor diff --git a/newsfragments/3900.minor b/newsfragments/3900.minor new file mode 100644 index 000000000..e69de29bb From 8694543659a36007c0d7a0787808fa119df15931 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 11:15:51 -0400 Subject: [PATCH 672/916] Work with Sphinx 5. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index af05e5900..cc9a11166 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,7 +63,7 @@ release = u'1.x' # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: From 00381bc24fefc3a831d4f253819d154609266423 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jun 2022 13:52:45 -0400 Subject: [PATCH 673/916] Correction now that it does more than what it did before. --- src/allmydata/storage/http_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bcad0e972..63be2f270 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -276,7 +276,8 @@ _SCHEMAS = { def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[bytes]: """ - Parse the ``Range`` header, read appropriately, return as result. + Read an optional ``Range`` header, reads data appropriately via the given + callable, return as result. Only parses a subset of ``Range`` headers that we support: must be set, bytes only, only a single range, the end must be explicitly specified. From db426513558bd328fc803c74c382f2ae0a214b92 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jun 2022 13:55:47 -0400 Subject: [PATCH 674/916] Be more consistent and just always write to the request in `read_range`. --- src/allmydata/storage/http_server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 63be2f270..543fceb98 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -274,21 +274,20 @@ _SCHEMAS = { } -def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[bytes]: +def read_range(request, read_data: Callable[[int, int], bytes]) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given - callable, return as result. + callable, writes the data to the request. Only parses a subset of ``Range`` headers that we support: must be set, bytes only, only a single range, the end must be explicitly specified. Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not possible or the header isn't set. - Returns a result that should be returned from the request handler, and sets - appropriate response headers. - Takes a function that will do the actual reading given the start offset and a length to read. + + The resulting data is written to the request. """ if request.getHeader("range") is None: # Return the whole thing. @@ -299,7 +298,7 @@ def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[byte data = read_data(start, start + 65536) if not data: request.finish() - return None + return request.write(data) start += len(data) @@ -326,7 +325,8 @@ def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[byte "content-range", ContentRange("bytes", offset, offset + len(data)).to_header(), ) - return data + request.write(data) + request.finish() class HTTPServer(object): From d37f187c078868833a98a362e528f559b97cdd94 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jun 2022 13:56:23 -0400 Subject: [PATCH 675/916] Lint fix. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 543fceb98..06a6863fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,7 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable, Optional +from typing import Dict, List, Set, Tuple, Any, Callable from functools import wraps from base64 import b64decode From e1daa192fbe8ff8168392ec9989664ddd4b3905a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jun 2022 17:20:08 -0400 Subject: [PATCH 676/916] Sketch of protocol switcher experiment. --- src/allmydata/node.py | 2 + src/allmydata/protocol_switch.py | 84 ++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/allmydata/protocol_switch.py diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 3ac4c507b..0547d3fe6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -51,6 +51,7 @@ from allmydata.util import configutil from allmydata.util.yamlutil import ( safe_load, ) +from .protocol_switch import FoolscapOrHttp from . import ( __full_version__, @@ -707,6 +708,7 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) + tub.negotiationClass = FoolscapOrHttp for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py new file mode 100644 index 000000000..59e1b609f --- /dev/null +++ b/src/allmydata/protocol_switch.py @@ -0,0 +1,84 @@ +""" +Support for listening with both HTTP and Foolscap on the same port. +""" + +from enum import Enum +from typing import Optional + +from twisted.internet.protocol import Protocol +from twisted.python.failure import Failure + +from foolscap.negotiate import Negotiation + +class ProtocolMode(Enum): + """Listening mode.""" + UNDECIDED = 0 + FOOLSCAP = 1 + HTTP = 2 + + +class PretendToBeNegotiation(type): + """😱""" + + def __instancecheck__(self, instance): + return (instance.__class__ == self) or isinstance(instance, Negotiation) + + +class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): + """ + Based on initial query, decide whether we're talking Foolscap or HTTP. + + Pretends to be a ``foolscap.negotiate.Negotiation`` instance. + """ + _foolscap : Optional[Negotiation] = None + _protocol_mode : ProtocolMode = ProtocolMode.UNDECIDED + _buffer: bytes = b"" + + def __init__(self, *args, **kwargs): + self._foolscap = Negotiation(*args, **kwargs) + + def __setattr__(self, name, value): + if name in {"_foolscap", "_protocol_mode", "_buffer", "transport"}: + object.__setattr__(self, name, value) + else: + setattr(self._foolscap, name, value) + + def __getattr__(self, name): + return getattr(self._foolscap, name) + + def makeConnection(self, transport): + Protocol.makeConnection(self, transport) + self._foolscap.makeConnection(transport) + + def initServer(self, *args, **kwargs): + return self._foolscap.initServer(*args, **kwargs) + + def initClient(self, *args, **kwargs): + assert not self._buffer + self._protocol_mode = ProtocolMode.FOOLSCAP + return self._foolscap.initClient(*args, **kwargs) + + def dataReceived(self, data: bytes) -> None: + if self._protocol_mode == ProtocolMode.FOOLSCAP: + return self._foolscap.dataReceived(data) + if self._protocol_mode == ProtocolMode.HTTP: + raise NotImplementedError() + + # UNDECIDED mode. + self._buffer += data + if len(self._buffer) < 8: + return + + # Check if it looks like Foolscap request. If so, it can handle this + # and later data: + if self._buffer.startswith(b"GET /id/"): + self._protocol_mode = ProtocolMode.FOOLSCAP + buf, self._buffer = self._buffer, b"" + return self._foolscap.dataReceived(buf) + else: + self._protocol_mode = ProtocolMode.HTTP + raise NotImplementedError("") + + def connectionLost(self, reason: Failure) -> None: + if self._protocol_mode == ProtocolMode.FOOLSCAP: + return self._foolscap.connectionLost(reason) From 7910867be6b8154f2b10031f3790b1a8c5eba821 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jun 2022 10:23:23 -0400 Subject: [PATCH 677/916] It actually works(?!) now. --- src/allmydata/protocol_switch.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 59e1b609f..fa23738d2 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -10,8 +10,10 @@ from twisted.python.failure import Failure from foolscap.negotiate import Negotiation + class ProtocolMode(Enum): """Listening mode.""" + UNDECIDED = 0 FOOLSCAP = 1 HTTP = 2 @@ -30,15 +32,22 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Pretends to be a ``foolscap.negotiate.Negotiation`` instance. """ - _foolscap : Optional[Negotiation] = None - _protocol_mode : ProtocolMode = ProtocolMode.UNDECIDED + + _foolscap: Optional[Negotiation] = None + _protocol_mode: ProtocolMode = ProtocolMode.UNDECIDED _buffer: bytes = b"" def __init__(self, *args, **kwargs): self._foolscap = Negotiation(*args, **kwargs) def __setattr__(self, name, value): - if name in {"_foolscap", "_protocol_mode", "_buffer", "transport"}: + if name in { + "_foolscap", + "_protocol_mode", + "_buffer", + "transport", + "__class__", + }: object.__setattr__(self, name, value) else: setattr(self._foolscap, name, value) @@ -50,13 +59,15 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Protocol.makeConnection(self, transport) self._foolscap.makeConnection(transport) - def initServer(self, *args, **kwargs): - return self._foolscap.initServer(*args, **kwargs) - def initClient(self, *args, **kwargs): + # After creation, a Negotiation instance either has initClient() or + # initServer() called. SInce this is a client, we're never going to do + # HTTP. Relying on __getattr__/__setattr__ doesn't work, for some + # reason, so just mutate ourselves appropriately. assert not self._buffer - self._protocol_mode = ProtocolMode.FOOLSCAP - return self._foolscap.initClient(*args, **kwargs) + self.__class__ = Negotiation + self.__dict__ = self._foolscap.__dict__ + return self.initClient(*args, **kwargs) def dataReceived(self, data: bytes) -> None: if self._protocol_mode == ProtocolMode.FOOLSCAP: @@ -69,7 +80,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): if len(self._buffer) < 8: return - # Check if it looks like Foolscap request. If so, it can handle this + # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): self._protocol_mode = ProtocolMode.FOOLSCAP From 7577d1e24ca8f8a93a11b3bd87deb251f40cbce8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jun 2022 14:19:29 -0400 Subject: [PATCH 678/916] Sketch of HTTP support, still untested WIP. --- src/allmydata/client.py | 8 ++++++++ src/allmydata/node.py | 2 -- src/allmydata/protocol_switch.py | 24 ++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 56ecdc6ed..ad5feb2ed 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -64,6 +64,7 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata import node +from .protocol_switch import FoolscapOrHttp KiB=1024 @@ -818,6 +819,13 @@ class _Client(node.Node, pollmixin.PollMixin): if anonymous_storage_enabled(self.config): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) + (_, _, swissnum) = furl.rpartition("/") + class FoolscapOrHttpWithCert(FoolscapOrHttp): + certificate = self.tub.myCertificate + storage_server = ss + swissnum = swissnum + self.tub.negotiationClass = FoolscapOrHttpWithCert + announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0547d3fe6..3ac4c507b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -51,7 +51,6 @@ from allmydata.util import configutil from allmydata.util.yamlutil import ( safe_load, ) -from .protocol_switch import FoolscapOrHttp from . import ( __full_version__, @@ -708,7 +707,6 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) - tub.negotiationClass = FoolscapOrHttp for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index fa23738d2..5a9589c17 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -7,9 +7,14 @@ from typing import Optional from twisted.internet.protocol import Protocol from twisted.python.failure import Failure +from twisted.internet.ssl import CertificateOptions +from twisted.web.server import Site +from twisted.protocols.tls import TLSMemoryBIOFactory from foolscap.negotiate import Negotiation +from .storage.http_server import HTTPServer + class ProtocolMode(Enum): """Listening mode.""" @@ -47,6 +52,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): "_buffer", "transport", "__class__", + "_http", }: object.__setattr__(self, name, value) else: @@ -73,7 +79,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): if self._protocol_mode == ProtocolMode.FOOLSCAP: return self._foolscap.dataReceived(data) if self._protocol_mode == ProtocolMode.HTTP: - raise NotImplementedError() + return self._http.dataReceived(data) # UNDECIDED mode. self._buffer += data @@ -83,12 +89,26 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): + # TODO or maybe just self.__class__ here too? self._protocol_mode = ProtocolMode.FOOLSCAP buf, self._buffer = self._buffer, b"" return self._foolscap.dataReceived(buf) else: self._protocol_mode = ProtocolMode.HTTP - raise NotImplementedError("") + + certificate_options = CertificateOptions( + privateKey=self.certificate.privateKey.original, + certificate=self.certificate.original, + ) + http_server = HTTPServer(self.storage_server, self.swissnum) + factory = TLSMemoryBIOFactory( + certificate_options, False, Site(http_server.get_resource()) + ) + protocol = factory.buildProtocol(self.transport.getPeer()) + protocol.makeConnection(self.transport) + protocol.dataReceived(self._buffer) + # TODO __getattr__ or maybe change the __class__ + self._http = protocol def connectionLost(self, reason: Failure) -> None: if self._protocol_mode == ProtocolMode.FOOLSCAP: From c5724c1d0a70ad1aa539aa0063a300bc359ddf21 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jun 2022 14:20:42 -0400 Subject: [PATCH 679/916] Clarify. --- src/allmydata/protocol_switch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 5a9589c17..50a7b1476 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -107,7 +107,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): protocol = factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) - # TODO __getattr__ or maybe change the __class__ + # TODO maybe change the __class__ self._http = protocol def connectionLost(self, reason: Failure) -> None: From 1579530895c5e66997f95ff8424d26be73dec011 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 07:59:43 -0400 Subject: [PATCH 680/916] Add working HTTP support. --- src/allmydata/client.py | 8 ++------ src/allmydata/node.py | 3 +++ src/allmydata/protocol_switch.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ad5feb2ed..294684b58 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -64,7 +64,7 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata import node -from .protocol_switch import FoolscapOrHttp +from .protocol_switch import update_foolscap_or_http_class KiB=1024 @@ -820,11 +820,7 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - class FoolscapOrHttpWithCert(FoolscapOrHttp): - certificate = self.tub.myCertificate - storage_server = ss - swissnum = swissnum - self.tub.negotiationClass = FoolscapOrHttpWithCert + update_foolscap_or_http_class(self.tub.negotiationClass, self.tub.myCertificate, ss, swissnum.encode("ascii")) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 3ac4c507b..93fa6a8e1 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -55,6 +55,8 @@ from allmydata.util.yamlutil import ( from . import ( __full_version__, ) +from .protocol_switch import create_foolscap_or_http_class + def _common_valid_config(): return configutil.ValidConfiguration({ @@ -707,6 +709,7 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) + tub.negotiationClass = create_foolscap_or_http_class() for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 50a7b1476..bb1a59bef 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -14,6 +14,7 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from foolscap.negotiate import Negotiation from .storage.http_server import HTTPServer +from .storage.server import StorageServer class ProtocolMode(Enum): @@ -38,6 +39,11 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Pretends to be a ``foolscap.negotiate.Negotiation`` instance. """ + # These three will be set by a subclass + swissnum: bytes + certificate = None # TODO figure out type + storage_server: StorageServer + _foolscap: Optional[Negotiation] = None _protocol_mode: ProtocolMode = ProtocolMode.UNDECIDED _buffer: bytes = b"" @@ -113,3 +119,16 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def connectionLost(self, reason: Failure) -> None: if self._protocol_mode == ProtocolMode.FOOLSCAP: return self._foolscap.connectionLost(reason) + + +def create_foolscap_or_http_class(): + class FoolscapOrHttpWithCert(FoolscapOrHttp): + pass + + return FoolscapOrHttpWithCert + + +def update_foolscap_or_http_class(cls, certificate, storage_server, swissnum): + cls.certificate = certificate + cls.storage_server = storage_server + cls.swissnum = swissnum From 04156db74ef28c63fb2273277f3f52b5f6d4883c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:32:43 -0400 Subject: [PATCH 681/916] Delay Negotiation.connectionMade so we don't create unnecessary timeouts. --- src/allmydata/protocol_switch.py | 34 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index bb1a59bef..2f834081b 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -3,10 +3,10 @@ Support for listening with both HTTP and Foolscap on the same port. """ from enum import Enum -from typing import Optional +from typing import Optional, Tuple from twisted.internet.protocol import Protocol -from twisted.python.failure import Failure +from twisted.internet.interfaces import ITransport from twisted.internet.ssl import CertificateOptions from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory @@ -21,8 +21,7 @@ class ProtocolMode(Enum): """Listening mode.""" UNDECIDED = 0 - FOOLSCAP = 1 - HTTP = 2 + HTTP = 1 class PretendToBeNegotiation(type): @@ -67,9 +66,13 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def __getattr__(self, name): return getattr(self._foolscap, name) - def makeConnection(self, transport): - Protocol.makeConnection(self, transport) - self._foolscap.makeConnection(transport) + def _convert_to_negotiation(self) -> Tuple[bytes, ITransport]: + """Convert self to a ``Negotiation`` instance, return any buffered bytes""" + transport = self.transport + buf = self._buffer + self.__class__ = Negotiation # type: ignore + self.__dict__ = self._foolscap.__dict__ + return buf, transport def initClient(self, *args, **kwargs): # After creation, a Negotiation instance either has initClient() or @@ -77,13 +80,10 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # HTTP. Relying on __getattr__/__setattr__ doesn't work, for some # reason, so just mutate ourselves appropriately. assert not self._buffer - self.__class__ = Negotiation - self.__dict__ = self._foolscap.__dict__ + self._convert_to_negotiation() return self.initClient(*args, **kwargs) def dataReceived(self, data: bytes) -> None: - if self._protocol_mode == ProtocolMode.FOOLSCAP: - return self._foolscap.dataReceived(data) if self._protocol_mode == ProtocolMode.HTTP: return self._http.dataReceived(data) @@ -95,10 +95,10 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): - # TODO or maybe just self.__class__ here too? - self._protocol_mode = ProtocolMode.FOOLSCAP - buf, self._buffer = self._buffer, b"" - return self._foolscap.dataReceived(buf) + buf, transport = self._convert_to_negotiation() + self.makeConnection(transport) + self.dataReceived(buf) + return else: self._protocol_mode = ProtocolMode.HTTP @@ -116,10 +116,6 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # TODO maybe change the __class__ self._http = protocol - def connectionLost(self, reason: Failure) -> None: - if self._protocol_mode == ProtocolMode.FOOLSCAP: - return self._foolscap.connectionLost(reason) - def create_foolscap_or_http_class(): class FoolscapOrHttpWithCert(FoolscapOrHttp): From d86f8519dcdd14622eb5d695b880a77db2038e89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:41:01 -0400 Subject: [PATCH 682/916] Simplify implementation. --- src/allmydata/protocol_switch.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2f834081b..11a35c324 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -2,7 +2,6 @@ Support for listening with both HTTP and Foolscap on the same port. """ -from enum import Enum from typing import Optional, Tuple from twisted.internet.protocol import Protocol @@ -17,13 +16,6 @@ from .storage.http_server import HTTPServer from .storage.server import StorageServer -class ProtocolMode(Enum): - """Listening mode.""" - - UNDECIDED = 0 - HTTP = 1 - - class PretendToBeNegotiation(type): """😱""" @@ -43,21 +35,16 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): certificate = None # TODO figure out type storage_server: StorageServer - _foolscap: Optional[Negotiation] = None - _protocol_mode: ProtocolMode = ProtocolMode.UNDECIDED - _buffer: bytes = b"" - def __init__(self, *args, **kwargs): - self._foolscap = Negotiation(*args, **kwargs) + self._foolscap: Negotiation = Negotiation(*args, **kwargs) + self._buffer: bytes = b"" def __setattr__(self, name, value): if name in { "_foolscap", - "_protocol_mode", "_buffer", "transport", "__class__", - "_http", }: object.__setattr__(self, name, value) else: @@ -66,7 +53,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def __getattr__(self, name): return getattr(self._foolscap, name) - def _convert_to_negotiation(self) -> Tuple[bytes, ITransport]: + def _convert_to_negotiation(self) -> Tuple[bytes, Optional[ITransport]]: """Convert self to a ``Negotiation`` instance, return any buffered bytes""" transport = self.transport buf = self._buffer @@ -84,10 +71,11 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): return self.initClient(*args, **kwargs) def dataReceived(self, data: bytes) -> None: - if self._protocol_mode == ProtocolMode.HTTP: - return self._http.dataReceived(data) + """Handle incoming data. - # UNDECIDED mode. + Once we've decided which protocol we are, update self.__class__, at + which point all methods will be called on the new class. + """ self._buffer += data if len(self._buffer) < 8: return @@ -100,8 +88,6 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self.dataReceived(buf) return else: - self._protocol_mode = ProtocolMode.HTTP - certificate_options = CertificateOptions( privateKey=self.certificate.privateKey.original, certificate=self.certificate.original, @@ -113,8 +99,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): protocol = factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) - # TODO maybe change the __class__ - self._http = protocol + self.__class__ = protocol.__class__ + self.__dict__ = protocol.__dict__ def create_foolscap_or_http_class(): From 026d63cd6a83f274fb1336945afe5145f0afc226 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:41:47 -0400 Subject: [PATCH 683/916] Fix some mypy warnings. --- src/allmydata/protocol_switch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 11a35c324..f3a624318 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -6,7 +6,7 @@ from typing import Optional, Tuple from twisted.internet.protocol import Protocol from twisted.internet.interfaces import ITransport -from twisted.internet.ssl import CertificateOptions +from twisted.internet.ssl import CertificateOptions, PrivateCertificate from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory @@ -32,7 +32,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # These three will be set by a subclass swissnum: bytes - certificate = None # TODO figure out type + certificate: PrivateCertificate storage_server: StorageServer def __init__(self, *args, **kwargs): @@ -96,6 +96,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): factory = TLSMemoryBIOFactory( certificate_options, False, Site(http_server.get_resource()) ) + assert self.transport is not None protocol = factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) From d70f583172e409268cd83c58d935815ec70296b4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:43:46 -0400 Subject: [PATCH 684/916] More cleanups. --- src/allmydata/protocol_switch.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index f3a624318..23d7dda84 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -30,7 +30,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Pretends to be a ``foolscap.negotiate.Negotiation`` instance. """ - # These three will be set by a subclass + # These three will be set by a subclass in update_foolscap_or_http_class() + # below. swissnum: bytes certificate: PrivateCertificate storage_server: StorageServer @@ -53,19 +54,18 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def __getattr__(self, name): return getattr(self._foolscap, name) - def _convert_to_negotiation(self) -> Tuple[bytes, Optional[ITransport]]: - """Convert self to a ``Negotiation`` instance, return any buffered bytes""" - transport = self.transport - buf = self._buffer + def _convert_to_negotiation(self): + """ + Convert self to a ``Negotiation`` instance, return any buffered + bytes and the transport if any. + """ self.__class__ = Negotiation # type: ignore self.__dict__ = self._foolscap.__dict__ - return buf, transport def initClient(self, *args, **kwargs): # After creation, a Negotiation instance either has initClient() or - # initServer() called. SInce this is a client, we're never going to do - # HTTP. Relying on __getattr__/__setattr__ doesn't work, for some - # reason, so just mutate ourselves appropriately. + # initServer() called. Since this is a client, we're never going to do + # HTTP, so we can immediately become a Negotiation instance. assert not self._buffer self._convert_to_negotiation() return self.initClient(*args, **kwargs) @@ -83,7 +83,9 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): - buf, transport = self._convert_to_negotiation() + transport = self.transport + buf = self._buffer + self._convert_to_negotiation() self.makeConnection(transport) self.dataReceived(buf) return From 0c99a9f7b0b14d340586d42cb589d3c888c3db1d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:44:17 -0400 Subject: [PATCH 685/916] Make it more accurate. --- src/allmydata/protocol_switch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 23d7dda84..899c1258f 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -56,8 +56,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def _convert_to_negotiation(self): """ - Convert self to a ``Negotiation`` instance, return any buffered - bytes and the transport if any. + Convert self to a ``Negotiation`` instance. """ self.__class__ = Negotiation # type: ignore self.__dict__ = self._foolscap.__dict__ From eb1e48bcc367e78945024cc642a59693aa3ecf09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:47:33 -0400 Subject: [PATCH 686/916] Add a timeout. --- src/allmydata/protocol_switch.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 899c1258f..2d2590977 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -2,13 +2,14 @@ Support for listening with both HTTP and Foolscap on the same port. """ -from typing import Optional, Tuple +from typing import Optional from twisted.internet.protocol import Protocol -from twisted.internet.interfaces import ITransport +from twisted.internet.interfaces import IDelayedCall from twisted.internet.ssl import CertificateOptions, PrivateCertificate from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory +from twisted.internet import reactor from foolscap.negotiate import Negotiation @@ -36,6 +37,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): certificate: PrivateCertificate storage_server: StorageServer + _timeout: IDelayedCall + def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) self._buffer: bytes = b"" @@ -69,6 +72,9 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self._convert_to_negotiation() return self.initClient(*args, **kwargs) + def connectionMade(self): + self._timeout = reactor.callLater(30, self.transport.abortConnection) + def dataReceived(self, data: bytes) -> None: """Handle incoming data. @@ -80,7 +86,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): return # Check if it looks like a Foolscap request. If so, it can handle this - # and later data: + # and later data, otherwise assume HTTPS. + self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): transport = self.transport buf = self._buffer From 01d8cc7ab66745d4371820334619f3ecd4ca2881 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:49:07 -0400 Subject: [PATCH 687/916] Put the attribute on the correct object. --- src/allmydata/protocol_switch.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2d2590977..d26bad745 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -44,12 +44,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self._buffer: bytes = b"" def __setattr__(self, name, value): - if name in { - "_foolscap", - "_buffer", - "transport", - "__class__", - }: + if name in {"_foolscap", "_buffer", "transport", "__class__", "_timeout"}: object.__setattr__(self, name, value) else: setattr(self._foolscap, name, value) From 1154371d22abd859a48efdee8eee9146b3164b1c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:51:07 -0400 Subject: [PATCH 688/916] Clarifying comments. --- src/allmydata/protocol_switch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index d26bad745..9b4e30671 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -84,6 +84,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # and later data, otherwise assume HTTPS. self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): + # We're a Foolscap Negotiation server protocol instance: transport = self.transport buf = self._buffer self._convert_to_negotiation() @@ -91,6 +92,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self.dataReceived(buf) return else: + # We're a HTTPS protocol instance, serving the storage protocol: certificate_options = CertificateOptions( privateKey=self.certificate.privateKey.original, certificate=self.certificate.original, From bfd54dc6eadf4e012c3dbf32a2356243c0aa505c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 11:30:49 -0400 Subject: [PATCH 689/916] Switch to newer attrs API, for consistency across the module. --- src/allmydata/storage/http_server.py | 31 ++++++++++++---------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 06a6863fa..ebd2323ef 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -19,7 +19,7 @@ from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath -import attr +from attrs import define, field from werkzeug.http import ( parse_range_header, parse_content_range_header, @@ -137,31 +137,31 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): return decorator -@attr.s +@define class StorageIndexUploads(object): """ In-progress upload to storage index. """ # Map share number to BucketWriter - shares = attr.ib(factory=dict) # type: Dict[int,BucketWriter] + shares: dict[int, BucketWriter] = field(factory=dict) # Map share number to the upload secret (different shares might have # different upload secrets). - upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] + upload_secrets: dict[int, bytes] = field(factory=dict) -@attr.s +@define class UploadsInProgress(object): """ Keep track of uploads for storage indexes. """ # Map storage index to corresponding uploads-in-progress - _uploads = attr.ib(type=Dict[bytes, StorageIndexUploads], factory=dict) + _uploads: dict[bytes, StorageIndexUploads] = field(factory=dict) # Map BucketWriter to (storage index, share number) - _bucketwriters = attr.ib(type=Dict[BucketWriter, Tuple[bytes, int]], factory=dict) + _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = field(factory=dict) def add_write_bucket( self, @@ -445,10 +445,7 @@ class HTTPServer(object): return self._send_encoded( request, - { - "already-have": set(already_got), - "allocated": set(sharenum_to_bucket), - }, + {"already-have": set(already_got), "allocated": set(sharenum_to_bucket)}, ) @_authorized_route( @@ -635,6 +632,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" + def read_data(offset, length): try: return self._storage_server.slot_readv( @@ -646,10 +644,7 @@ class HTTPServer(object): return read_range(request, read_data) @_authorized_route( - _app, - set(), - "/v1/mutable//shares", - methods=["GET"], + _app, set(), "/v1/mutable//shares", methods=["GET"] ) def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" @@ -679,7 +674,7 @@ class HTTPServer(object): @implementer(IStreamServerEndpoint) -@attr.s +@define class _TLSEndpointWrapper(object): """ Wrap an existing endpoint with the server-side storage TLS policy. This is @@ -687,8 +682,8 @@ class _TLSEndpointWrapper(object): example there's Tor and i2p. """ - endpoint = attr.ib(type=IStreamServerEndpoint) - context_factory = attr.ib(type=CertificateOptions) + endpoint: IStreamServerEndpoint + context_factory: CertificateOptions @classmethod def from_paths( From 06eca79263382fab3b742d1d3243463735bc79f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 14:03:05 -0400 Subject: [PATCH 690/916] Minimal streaming implementation. --- src/allmydata/storage/http_server.py | 55 ++++++++++++++++++------- src/allmydata/test/test_storage_http.py | 21 ++++++++-- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ebd2323ef..b8887bb4e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -12,10 +12,15 @@ import binascii from zope.interface import implementer from klein import Klein from twisted.web import http -from twisted.internet.interfaces import IListeningPort, IStreamServerEndpoint +from twisted.web.server import NOT_DONE_YET +from twisted.internet.interfaces import ( + IListeningPort, + IStreamServerEndpoint, + IPullProducer, +) from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate -from twisted.web.server import Site +from twisted.web.server import Site, Request from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath @@ -274,7 +279,37 @@ _SCHEMAS = { } -def read_range(request, read_data: Callable[[int, int], bytes]) -> None: +@implementer(IPullProducer) +@define +class _ReadProducer: + """ + Producer that calls a read function, and writes to a request. + """ + + request: Request + read_data: Callable[[int, int], bytes] + result: Deferred + start: int = field(default=0) + + def resumeProducing(self): + data = self.read_data(self.start, self.start + 65536) + if not data: + self.request.unregisterProducer() + d = self.result + del self.result + d.callback(b"") + return + self.request.write(data) + self.start += len(data) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + +def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -290,17 +325,9 @@ def read_range(request, read_data: Callable[[int, int], bytes]) -> None: The resulting data is written to the request. """ if request.getHeader("range") is None: - # Return the whole thing. - start = 0 - while True: - # TODO should probably yield to event loop occasionally... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = read_data(start, start + 65536) - if not data: - request.finish() - return - request.write(data) - start += len(data) + d = Deferred() + request.registerProducer(_ReadProducer(request, read_data, d), False) + return d range_header = parse_range_header(request.getHeader("range")) if ( diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5e0b35d88..23d9bc276 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -6,6 +6,7 @@ from base64 import b64encode from contextlib import contextmanager from os import urandom from typing import Union, Callable, Tuple, Iterable +from time import sleep, time from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -14,7 +15,8 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap -from twisted.internet.task import Clock +from twisted.internet.task import Clock, Cooperator +from twisted.internet import task from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -316,10 +318,11 @@ class HttpTestFixture(Fixture): self.tempdir.path, b"\x00" * 20, clock=self.clock ) self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.treq = StubTreq(self.http_server.get_resource()) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), + treq=self.treq, ) @@ -1261,8 +1264,20 @@ class SharedImmutableMutableTestsMixin: """ A read with no range returns the whole mutable/immutable. """ + self.patch( + task, + "_theCooperator", + Cooperator(scheduler=lambda c: self.http.clock.callLater(0.000001, c)), + ) + + def result_of_with_flush(d): + for i in range(100): + self.http.clock.advance(0.001) + self.http.treq.flush() + return result_of(d) + storage_index, uploaded_data, _ = self.upload(1, data_length) - response = result_of( + response = result_of_with_flush( self.http.client.request( "GET", self.http.client.relative_url( From 6dd2b2d58357f30e7b663008e1f68ad798846f91 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 14:44:51 -0400 Subject: [PATCH 691/916] More streaming, with tests passing again. --- src/allmydata/storage/http_server.py | 88 ++++++++++++++++++++----- src/allmydata/test/test_storage_http.py | 74 +++++++++++++-------- 2 files changed, 115 insertions(+), 47 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b8887bb4e..a91b7963e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -281,9 +281,10 @@ _SCHEMAS = { @implementer(IPullProducer) @define -class _ReadProducer: +class _ReadAllProducer: """ - Producer that calls a read function, and writes to a request. + Producer that calls a read function repeatedly to read all the data, and + writes to a request. """ request: Request @@ -292,7 +293,7 @@ class _ReadProducer: start: int = field(default=0) def resumeProducing(self): - data = self.read_data(self.start, self.start + 65536) + data = self.read_data(self.start, 65536) if not data: self.request.unregisterProducer() d = self.result @@ -309,6 +310,52 @@ class _ReadProducer: pass +@implementer(IPullProducer) +@define +class _ReadRangeProducer: + """ + Producer that calls a read function to read a range of data, and writes to + a request. + """ + + request: Request + read_data: Callable[[int, int], bytes] + result: Deferred + start: int + remaining: int + first_read: bool = field(default=True) + + def resumeProducing(self): + to_read = min(self.remaining, 65536) + data = self.read_data(self.start, to_read) + assert len(data) <= to_read + if self.first_read and data: + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + self.request.setHeader( + "content-range", + ContentRange("bytes", self.start, self.start + len(data)).to_header(), + ) + self.request.write(data) + + if not data or len(data) < to_read: + self.request.unregisterProducer() + d = self.result + del self.result + d.callback(b"") + return + + self.start += len(data) + self.remaining -= len(data) + assert self.remaining >= 0 + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given @@ -324,9 +371,20 @@ def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None The resulting data is written to the request. """ + + def read_data_with_error_handling(offset: int, length: int) -> bytes: + try: + return read_data(offset, length) + except _HTTPError as e: + request.setResponseCode(e.code) + # Empty read means we're done. + return b"" + if request.getHeader("range") is None: d = Deferred() - request.registerProducer(_ReadProducer(request, read_data, d), False) + request.registerProducer( + _ReadAllProducer(request, read_data_with_error_handling, d), False + ) return d range_header = parse_range_header(request.getHeader("range")) @@ -339,21 +397,15 @@ def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) offset, end = range_header.ranges[0] - - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = read_data(offset, end - offset) - request.setResponseCode(http.PARTIAL_CONTENT) - if len(data): - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) - request.write(data) - request.finish() + d = Deferred() + request.registerProducer( + _ReadRangeProducer( + request, read_data_with_error_handling, d, offset, end - offset + ), + False, + ) + return d class HTTPServer(object): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 23d9bc276..2382211df 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -10,7 +10,7 @@ from time import sleep, time from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir +from fixtures import Fixture, TempDir, MockPatch from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL @@ -314,6 +314,12 @@ class HttpTestFixture(Fixture): def _setUp(self): self.clock = Clock() self.tempdir = self.useFixture(TempDir()) + self.mock = self.useFixture( + MockPatch( + "twisted.internet.task._theCooperator", + Cooperator(scheduler=lambda c: self.clock.callLater(0.000001, c)), + ) + ) self.storage_server = StorageServer( self.tempdir.path, b"\x00" * 20, clock=self.clock ) @@ -325,6 +331,12 @@ class HttpTestFixture(Fixture): treq=self.treq, ) + def result_of_with_flush(self, d): + for i in range(100): + self.clock.advance(0.001) + self.treq.flush() + return result_of(d) + class StorageClientWithHeadersOverride(object): """Wrap ``StorageClient`` and override sent headers.""" @@ -548,7 +560,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: - downloaded = result_of( + downloaded = self.http.result_of_with_flush( self.imm_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) @@ -623,7 +635,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # The upload of share 1 succeeded, demonstrating that second create() # call didn't overwrite work-in-progress. - downloaded = result_of( + downloaded = self.http.result_of_with_flush( self.imm_client.read_share_chunk(storage_index, 1, 0, 100) ) self.assertEqual(downloaded, b"a" * 50 + b"b" * 50) @@ -753,11 +765,15 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) self.assertEqual( - result_of(self.imm_client.read_share_chunk(storage_index, 1, 0, 10)), + self.http.result_of_with_flush( + self.imm_client.read_share_chunk(storage_index, 1, 0, 10) + ), b"1" * 10, ) self.assertEqual( - result_of(self.imm_client.read_share_chunk(storage_index, 2, 0, 10)), + self.http.result_of_with_flush( + self.imm_client.read_share_chunk(storage_index, 2, 0, 10) + ), b"2" * 10, ) @@ -921,7 +937,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Abort didn't prevent reading: self.assertEqual( uploaded_data, - result_of( + self.http.result_of_with_flush( self.imm_client.read_share_chunk( storage_index, 0, @@ -986,8 +1002,12 @@ class MutableHTTPAPIsTests(SyncTestCase): Written data can be read using ``read_share_chunk``. """ storage_index, _, _ = self.create_upload() - data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7)) - data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + data0 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 1, 7) + ) + data1 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 1, 0, 8) + ) self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1")) def test_read_before_write(self): @@ -1015,8 +1035,12 @@ class MutableHTTPAPIsTests(SyncTestCase): ), ) # But the write did happen: - data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)) - data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + data0 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, 8) + ) + data1 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 1, 0, 8) + ) self.assertEqual((data0, data1), (b"aXYZef-0", b"abcdef-1")) def test_conditional_write(self): @@ -1057,7 +1081,9 @@ class MutableHTTPAPIsTests(SyncTestCase): ) self.assertTrue(result.success) self.assertEqual( - result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, 8) + ), b"aXYZef-0", ) @@ -1094,7 +1120,9 @@ class MutableHTTPAPIsTests(SyncTestCase): # The write did not happen: self.assertEqual( - result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, 8) + ), b"abcdef-0", ) @@ -1194,7 +1222,7 @@ class SharedImmutableMutableTestsMixin: Reading from unknown storage index results in 404. """ with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.client.read_share_chunk( b"1" * 16, 1, @@ -1209,7 +1237,7 @@ class SharedImmutableMutableTestsMixin: """ storage_index, _, _ = self.upload(1) with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.client.read_share_chunk( storage_index, 7, # different share number @@ -1235,7 +1263,7 @@ class SharedImmutableMutableTestsMixin: with assert_fails_with_http_code( self, http.REQUESTED_RANGE_NOT_SATISFIABLE ): - result_of( + self.http.result_of_with_flush( client.read_share_chunk( storage_index, 1, @@ -1264,20 +1292,8 @@ class SharedImmutableMutableTestsMixin: """ A read with no range returns the whole mutable/immutable. """ - self.patch( - task, - "_theCooperator", - Cooperator(scheduler=lambda c: self.http.clock.callLater(0.000001, c)), - ) - - def result_of_with_flush(d): - for i in range(100): - self.http.clock.advance(0.001) - self.http.treq.flush() - return result_of(d) - storage_index, uploaded_data, _ = self.upload(1, data_length) - response = result_of_with_flush( + response = self.http.result_of_with_flush( self.http.client.request( "GET", self.http.client.relative_url( @@ -1298,7 +1314,7 @@ class SharedImmutableMutableTestsMixin: def check_range(requested_range, expected_response): headers = Headers() headers.setRawHeaders("range", [requested_range]) - response = result_of( + response = self.http.result_of_with_flush( self.http.client.request( "GET", self.http.client.relative_url( From 75f33022cd201f2477b86af9c22641f3c69a2188 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 17:00:41 -0400 Subject: [PATCH 692/916] News file. --- newsfragments/3872.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3872.minor diff --git a/newsfragments/3872.minor b/newsfragments/3872.minor new file mode 100644 index 000000000..e69de29bb From efe9575d28dc18089525e9004159ddbe291997d0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 10:51:35 -0400 Subject: [PATCH 693/916] Nicer testing infrastructure so you don't have to switch back and forth between sync and async test APIs. --- src/allmydata/test/test_storage_http.py | 171 ++++++++++++++++-------- 1 file changed, 117 insertions(+), 54 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2382211df..1f860cca0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1,5 +1,21 @@ """ Tests for HTTP storage client + server. + +The tests here are synchronous and don't involve running a real reactor. This +works, but has some caveats when it comes to testing HTTP endpoints: + +* Some HTTP endpoints are synchronous, some are not. +* For synchronous endpoints, the result is immediately available on the + ``Deferred`` coming out of ``StubTreq``. +* For asynchronous endpoints, you need to use ``StubTreq.flush()`` and + iterate the fake in-memory clock/reactor to advance time . + +So for HTTP endpoints, you should use ``HttpTestFixture.result_of_with_flush()`` +which handles both, and patches and moves forward the global Twisted +``Cooperator`` since that is used to drive pull producers. This is, +sadly, an internal implementation detail of Twisted being leaked to tests... + +For definitely synchronous calls, you can just use ``result_of()``. """ from base64 import b64encode @@ -332,10 +348,33 @@ class HttpTestFixture(Fixture): ) def result_of_with_flush(self, d): + """ + Like ``result_of``, but supports fake reactor and ``treq`` testing + infrastructure necessary to support asynchronous HTTP server endpoints. + """ + result = [] + error = [] + d.addCallbacks(result.append, error.append) + + # Check for synchronous HTTP endpoint handler: + if result: + return result[0] + if error: + error[0].raiseException() + + # OK, no result yet, probably async HTTP endpoint handler, so advance + # time, flush treq, and try again: for i in range(100): self.clock.advance(0.001) self.treq.flush() - return result_of(d) + if result: + return result[0] + if error: + error[0].raiseException() + raise RuntimeError( + "We expected given Deferred to have result already, but it wasn't. " + + "This is probably a test design issue." + ) class StorageClientWithHeadersOverride(object): @@ -393,7 +432,7 @@ class GenericHTTPAPITests(SyncTestCase): ) ) with assert_fails_with_http_code(self, http.UNAUTHORIZED): - result_of(client.get_version()) + self.http.result_of_with_flush(client.get_version()) def test_unsupported_mime_type(self): """ @@ -404,7 +443,7 @@ class GenericHTTPAPITests(SyncTestCase): StorageClientWithHeadersOverride(self.http.client, {"accept": "image/gif"}) ) with assert_fails_with_http_code(self, http.NOT_ACCEPTABLE): - result_of(client.get_version()) + self.http.result_of_with_flush(client.get_version()) def test_version(self): """ @@ -414,7 +453,7 @@ class GenericHTTPAPITests(SyncTestCase): might change across calls. """ client = StorageClientGeneral(self.http.client) - version = result_of(client.get_version()) + version = self.http.result_of_with_flush(client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) @@ -448,7 +487,7 @@ class GenericHTTPAPITests(SyncTestCase): ) message = {"bad-message": "missing expected keys"} - response = result_of( + response = self.http.result_of_with_flush( self.http.client.request( "POST", url, @@ -481,7 +520,7 @@ class ImmutableHTTPAPITests(SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - created = result_of( + created = self.http.result_of_with_flush( self.imm_client.create( storage_index, share_numbers, @@ -525,35 +564,35 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - upload_progress = result_of(write(10, 10)) + upload_progress = self.http.result_of_with_flush(write(10, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(30, 10)) + upload_progress = self.http.result_of_with_flush(write(30, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(50, 10)) + upload_progress = self.http.result_of_with_flush(write(50, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) # Then, an overlapping write with matching data (15-35): - upload_progress = result_of(write(15, 20)) + upload_progress = self.http.result_of_with_flush(write(15, 20)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) # Now fill in the holes: - upload_progress = result_of(write(0, 10)) + upload_progress = self.http.result_of_with_flush(write(0, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(40, 10)) + upload_progress = self.http.result_of_with_flush(write(40, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(60, 40)) + upload_progress = self.http.result_of_with_flush(write(60, 40)) self.assertEqual( upload_progress, UploadProgress(finished=True, required=RangeMap()) ) @@ -572,7 +611,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) with assert_fails_with_http_code(self, http.UNAUTHORIZED): - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -594,7 +633,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) # Write half of share 1 - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -608,7 +647,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # existing shares, this call shouldn't overwrite the existing # work-in-progress. upload_secret2 = b"x" * 2 - created2 = result_of( + created2 = self.http.result_of_with_flush( self.imm_client.create( storage_index, {1, 4, 6}, @@ -622,7 +661,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Write second half of share 1 self.assertTrue( - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -642,7 +681,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can successfully upload the shares created with the second upload secret. self.assertTrue( - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 4, @@ -660,11 +699,14 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10) # Initially there are no shares: - self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) + self.assertEqual( + self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)), + set(), + ) # Upload shares 1 and 3: for share_number in [1, 3]: - progress = result_of( + progress = self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, share_number, @@ -676,7 +718,10 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertTrue(progress.finished) # Now shares 1 and 3 exist: - self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), {1, 3}) + self.assertEqual( + self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)), + {1, 3}, + ) def test_upload_bad_content_range(self): """ @@ -694,7 +739,7 @@ class ImmutableHTTPAPITests(SyncTestCase): with assert_fails_with_http_code( self, http.REQUESTED_RANGE_NOT_SATISFIABLE ): - result_of( + self.http.result_of_with_flush( client.write_share_chunk( storage_index, 1, @@ -714,7 +759,10 @@ class ImmutableHTTPAPITests(SyncTestCase): Listing unknown storage index's shares results in empty list of shares. """ storage_index = bytes(range(16)) - self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) + self.assertEqual( + self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)), + set(), + ) def test_upload_non_existent_storage_index(self): """ @@ -725,7 +773,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def unknown_check(storage_index, share_number): with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, share_number, @@ -746,7 +794,7 @@ class ImmutableHTTPAPITests(SyncTestCase): stored separately and can be downloaded separately. """ (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -755,7 +803,7 @@ class ImmutableHTTPAPITests(SyncTestCase): b"1" * 10, ) ) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 2, @@ -785,7 +833,7 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) # Write: - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -797,7 +845,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Conflicting write: with assert_fails_with_http_code(self, http.CONFLICT): - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -823,7 +871,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ def abort(storage_index, share_number, upload_secret): - return result_of( + return self.http.result_of_with_flush( self.imm_client.abort_upload(storage_index, share_number, upload_secret) ) @@ -836,7 +884,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ # Start an upload: (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -855,7 +903,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # complaint: upload_secret = urandom(32) lease_secret = urandom(32) - created = result_of( + created = self.http.result_of_with_flush( self.imm_client.create( storage_index, {1}, @@ -868,7 +916,7 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertEqual(created.allocated, {1}) # And write to it, too: - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -887,7 +935,9 @@ class ImmutableHTTPAPITests(SyncTestCase): for si, num in [(storage_index, 3), (b"x" * 16, 1)]: with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.imm_client.abort_upload(si, num, upload_secret)) + self.http.result_of_with_flush( + self.imm_client.abort_upload(si, num, upload_secret) + ) def test_unauthorized_abort(self): """ @@ -898,12 +948,12 @@ class ImmutableHTTPAPITests(SyncTestCase): # Failed to abort becaues wrong upload secret: with assert_fails_with_http_code(self, http.UNAUTHORIZED): - result_of( + self.http.result_of_with_flush( self.imm_client.abort_upload(storage_index, 1, upload_secret + b"X") ) # We can still write to it: - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -920,7 +970,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ uploaded_data = b"123" (upload_secret, _, storage_index, _) = self.create_upload({0}, 3) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 0, @@ -932,7 +982,9 @@ class ImmutableHTTPAPITests(SyncTestCase): # Can't abort, we finished upload: with assert_fails_with_http_code(self, http.NOT_ALLOWED): - result_of(self.imm_client.abort_upload(storage_index, 0, upload_secret)) + self.http.result_of_with_flush( + self.imm_client.abort_upload(storage_index, 0, upload_secret) + ) # Abort didn't prevent reading: self.assertEqual( @@ -954,7 +1006,7 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = urandom(16) secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.general_client.add_or_renew_lease(storage_index, secret, secret) ) @@ -975,7 +1027,7 @@ class MutableHTTPAPIsTests(SyncTestCase): write_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - result_of( + self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1013,7 +1065,7 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_read_before_write(self): """In combo read/test/write operation, reads happen before writes.""" storage_index, write_secret, lease_secret = self.create_upload() - result = result_of( + result = self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1046,7 +1098,7 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_conditional_write(self): """Uploads only happen if the test passes.""" storage_index, write_secret, lease_secret = self.create_upload() - result_failed = result_of( + result_failed = self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1064,7 +1116,7 @@ class MutableHTTPAPIsTests(SyncTestCase): self.assertFalse(result_failed.success) # This time the test matches: - result = result_of( + result = self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1090,19 +1142,22 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_list_shares(self): """``list_shares()`` returns the shares for a given storage index.""" storage_index, _, _ = self.create_upload() - self.assertEqual(result_of(self.mut_client.list_shares(storage_index)), {0, 1}) + self.assertEqual( + self.http.result_of_with_flush(self.mut_client.list_shares(storage_index)), + {0, 1}, + ) def test_non_existent_list_shares(self): """A non-existent storage index errors when shares are listed.""" with self.assertRaises(ClientException) as exc: - result_of(self.mut_client.list_shares(urandom(32))) + self.http.result_of_with_flush(self.mut_client.list_shares(urandom(32))) self.assertEqual(exc.exception.code, http.NOT_FOUND) def test_wrong_write_enabler(self): """Writes with the wrong write enabler fail, and are not processed.""" storage_index, write_secret, lease_secret = self.create_upload() with self.assertRaises(ClientException) as exc: - result_of( + self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, urandom(32), @@ -1161,7 +1216,9 @@ class SharedImmutableMutableTestsMixin: storage_index, _, _ = self.upload(13) reason = "OHNO \u1235" - result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + self.http.result_of_with_flush( + self.client.advise_corrupt_share(storage_index, 13, reason) + ) self.assertEqual( corrupted, @@ -1174,11 +1231,15 @@ class SharedImmutableMutableTestsMixin: """ storage_index, _, _ = self.upload(13) reason = "OHNO \u1235" - result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + self.http.result_of_with_flush( + self.client.advise_corrupt_share(storage_index, 13, reason) + ) for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.client.advise_corrupt_share(si, share_number, reason)) + self.http.result_of_with_flush( + self.client.advise_corrupt_share(si, share_number, reason) + ) def test_lease_renew_and_add(self): """ @@ -1196,7 +1257,7 @@ class SharedImmutableMutableTestsMixin: self.http.clock.advance(167) # We renew the lease: - result_of( + self.http.result_of_with_flush( self.general_client.add_or_renew_lease( storage_index, lease_secret, lease_secret ) @@ -1207,7 +1268,7 @@ class SharedImmutableMutableTestsMixin: # We create a new lease: lease_secret2 = urandom(32) - result_of( + self.http.result_of_with_flush( self.general_client.add_or_renew_lease( storage_index, lease_secret2, lease_secret2 ) @@ -1302,7 +1363,9 @@ class SharedImmutableMutableTestsMixin: ) ) self.assertEqual(response.code, http.OK) - self.assertEqual(result_of(response.content()), uploaded_data) + self.assertEqual( + self.http.result_of_with_flush(response.content()), uploaded_data + ) def test_validate_content_range_response_to_read(self): """ @@ -1354,7 +1417,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - result_of( + self.http.result_of_with_flush( self.client.create( storage_index, {share_number}, @@ -1364,7 +1427,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): lease_secret, ) ) - result_of( + self.http.result_of_with_flush( self.client.write_share_chunk( storage_index, share_number, @@ -1399,7 +1462,7 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): write_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - result_of( + self.http.result_of_with_flush( self.client.read_test_write_chunks( storage_index, write_secret, From 520456bdc0411845715798ac72cd8a88686b798f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 11:26:25 -0400 Subject: [PATCH 694/916] Add streaming to CBOR results. --- src/allmydata/storage/http_server.py | 45 ++++++++++++++++++---------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a91b7963e..f354fd837 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,16 +3,16 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable +from typing import Dict, List, Set, Tuple, Any, Callable from functools import wraps from base64 import b64decode import binascii +from tempfile import TemporaryFile from zope.interface import implementer from klein import Klein from twisted.web import http -from twisted.web.server import NOT_DONE_YET from twisted.internet.interfaces import ( IListeningPort, IStreamServerEndpoint, @@ -37,7 +37,7 @@ from cryptography.x509 import load_pem_x509_certificate # TODO Make sure to use pure Python versions? -from cbor2 import dumps, loads +from cbor2 import dump, loads from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import ( @@ -279,6 +279,10 @@ _SCHEMAS = { } +# Callabale that takes offset and length, returns the data at that range. +ReadData = Callable[[int, int], bytes] + + @implementer(IPullProducer) @define class _ReadAllProducer: @@ -288,10 +292,20 @@ class _ReadAllProducer: """ request: Request - read_data: Callable[[int, int], bytes] - result: Deferred + read_data: ReadData + result: Deferred = field(factory=Deferred) start: int = field(default=0) + @classmethod + def produce_to(cls, request: Request, read_data: ReadData) -> Deferred: + """ + Create and register the producer, returning ``Deferred`` that should be + returned from a HTTP server endpoint. + """ + producer = cls(request, read_data) + request.registerProducer(producer, False) + return producer.result + def resumeProducing(self): data = self.read_data(self.start, 65536) if not data: @@ -319,7 +333,7 @@ class _ReadRangeProducer: """ request: Request - read_data: Callable[[int, int], bytes] + read_data: ReadData result: Deferred start: int remaining: int @@ -356,7 +370,7 @@ class _ReadRangeProducer: pass -def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None: +def read_range(request: Request, read_data: ReadData) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -381,11 +395,7 @@ def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None return b"" if request.getHeader("range") is None: - d = Deferred() - request.registerProducer( - _ReadAllProducer(request, read_data_with_error_handling, d), False - ) - return d + return _ReadAllProducer.produce_to(request, read_data_with_error_handling) range_header = parse_range_header(request.getHeader("range")) if ( @@ -459,9 +469,14 @@ class HTTPServer(object): accept = parse_accept_header(accept_headers[0]) if accept.best == CBOR_MIME_TYPE: request.setHeader("Content-Type", CBOR_MIME_TYPE) - # TODO if data is big, maybe want to use a temporary file eventually... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return dumps(data) + f = TemporaryFile() + dump(data, f) + + def read_data(offset: int, length: int) -> bytes: + f.seek(offset) + return f.read(length) + + return _ReadAllProducer.produce_to(request, read_data) else: # TODO Might want to optionally send JSON someday: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 From 0e8f2aa7024c75ba01943fb3f1fbce7160c8a799 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 11:48:54 -0400 Subject: [PATCH 695/916] More memory usage reductions. --- src/allmydata/storage/http_server.py | 38 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 9 ++++++ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f354fd837..98bd419c1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -245,6 +245,8 @@ class _HTTPError(Exception): # Tags are of the form #6.nnn, where the number is documented at # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. +# +# TODO 3872 length limits in the schema. _SCHEMAS = { "allocate_buckets": Schema( """ @@ -485,12 +487,18 @@ class HTTPServer(object): def _read_encoded(self, request, schema: Schema) -> Any: """ Read encoded request body data, decoding it with CBOR by default. + + Somewhat arbitrarily, limit body size to 1MB; this may be too low, we + may want to customize per query type, but this is the starting point + for now. """ content_type = get_content_type(request.requestHeaders) if content_type == CBOR_MIME_TYPE: - # TODO limit memory usage, client could send arbitrarily large data... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - message = request.content.read() + # Read 1 byte more than 1MB. We expect length to be 1MB or + # less; if it's more assume it's not a legitimate message. + message = request.content.read(1024 * 1024 + 1) + if len(message) > 1024 * 1024: + raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) schema.validate_cbor(message) result = loads(message) return result @@ -586,20 +594,24 @@ class HTTPServer(object): request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) return b"" - offset = content_range.start - - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = request.content.read(content_range.stop - content_range.start + 1) bucket = self._uploads.get_write_bucket( storage_index, share_number, authorization[Secrets.UPLOAD] ) + offset = content_range.start + remaining = content_range.stop - content_range.start + finished = False - try: - finished = bucket.write(offset, data) - except ConflictingWriteError: - request.setResponseCode(http.CONFLICT) - return b"" + while remaining > 0: + data = request.content.read(min(remaining, 65536)) + assert data, "uploaded data length doesn't match range" + + try: + finished = bucket.write(offset, data) + except ConflictingWriteError: + request.setResponseCode(http.CONFLICT) + return b"" + remaining -= len(data) + offset += len(data) if finished: bucket.close() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 1f860cca0..5418660c0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1139,6 +1139,15 @@ class MutableHTTPAPIsTests(SyncTestCase): b"aXYZef-0", ) + def test_too_large_write(self): + """ + Writing too large of a chunk results in a REQUEST ENTITY TOO LARGE http + error. + """ + with self.assertRaises(ClientException) as e: + self.create_upload(b"0123456789" * 1024 * 1024) + self.assertEqual(e.exception.code, http.REQUEST_ENTITY_TOO_LARGE) + def test_list_shares(self): """``list_shares()`` returns the shares for a given storage index.""" storage_index, _, _ = self.create_upload() From ab80c0f0a17affc87489cb29c031fb072803fb90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 14:04:42 -0400 Subject: [PATCH 696/916] Set some length limits on various queries lengths. --- src/allmydata/storage/http_server.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 98bd419c1..50e4ec946 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -246,12 +246,14 @@ class _HTTPError(Exception): # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. # -# TODO 3872 length limits in the schema. +# Somewhat arbitrary limits are set to reduce e.g. number of shares, number of +# vectors, etc.. These may need to be iterated on in future revisions of the +# code. _SCHEMAS = { "allocate_buckets": Schema( """ request = { - share-numbers: #6.258([* uint]) + share-numbers: #6.258([*30 uint]) allocated-size: uint } """ @@ -267,13 +269,15 @@ _SCHEMAS = { """ request = { "test-write-vectors": { - * share_number: { - "test": [* {"offset": uint, "size": uint, "specimen": bstr}] - "write": [* {"offset": uint, "data": bstr}] + ; TODO Add length limit here, after + ; https://github.com/anweiss/cddl/issues/128 is fixed + * share_number => { + "test": [*30 {"offset": uint, "size": uint, "specimen": bstr}] + "write": [*30 {"offset": uint, "data": bstr}] "new-length": uint / null } } - "read-vector": [* {"offset": uint, "size": uint}] + "read-vector": [*30 {"offset": uint, "size": uint}] } share_number = uint """ From bee46fae93494206c843633592ac04cbd65849b5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 13:48:33 -0400 Subject: [PATCH 697/916] Resource limits on the client side. --- src/allmydata/storage/http_client.py | 34 ++++++++++++++++++--- src/allmydata/test/test_storage_http.py | 40 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9203d02ab..b8bd0bf20 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing import Union, Optional, Sequence, Mapping from base64 import b64encode +from io import BytesIO from attrs import define, asdict, frozen, field @@ -114,6 +115,33 @@ _SCHEMAS = { } +@define +class _LengthLimitedCollector: + """ + Collect data using ``treq.collect()``, with limited length. + """ + + remaining_length: int + f: BytesIO = field(factory=BytesIO) + + def __call__(self, data: bytes): + if len(data) > self.remaining_length: + raise ValueError("Response length was too long") + self.f.write(data) + + +def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred: + """ + Like ``treq.content()``, but limit data read from the response to a set + length. If the response is longer than the max allowed length, the result + fails with a ``ValueError``. + """ + collector = _LengthLimitedCollector(max_length) + d = treq.collect(response, collector) + d.addCallback(lambda _: collector.f.getvalue()) + return d + + def _decode_cbor(response, schema: Schema): """Given HTTP response, return decoded CBOR body.""" @@ -124,9 +152,7 @@ def _decode_cbor(response, schema: Schema): if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) if content_type == CBOR_MIME_TYPE: - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return treq.content(response).addCallback(got_content) + return limited_content(response).addCallback(got_content) else: raise ClientException(-1, "Server didn't send CBOR") else: @@ -295,7 +321,7 @@ class StorageClient(object): write_enabler_secret=None, headers=None, message_to_serialize=None, - **kwargs + **kwargs, ): """ Like ``treq.request()``, but with optional secrets that get translated diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5418660c0..915cd33f2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -65,6 +65,7 @@ from ..storage.http_client import ( ReadVector, ReadTestWriteResult, TestVector, + limited_content, ) @@ -255,6 +256,11 @@ class TestApp(object): request.setHeader("content-type", CBOR_MIME_TYPE) return dumps({"garbage": 123}) + @_authorized_route(_app, set(), "/millionbytes", methods=["GET"]) + def million_bytes(self, request, authorization): + """Return 1,000,000 bytes.""" + return b"0123456789" * 100_000 + def result_of(d): """ @@ -320,6 +326,40 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(CDDLValidationError): result_of(client.get_version()) + def test_limited_content_fits(self): + """ + ``http_client.limited_content()`` returns the body if it is less than + the max length. + """ + for at_least_length in (1_000_000, 1_000_001): + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/millionbytes", + ) + ) + + self.assertEqual( + result_of(limited_content(response, at_least_length)), + b"0123456789" * 100_000, + ) + + def test_limited_content_does_not_fit(self): + """ + If the body is longer than than max length, + ``http_client.limited_content()`` fails with a ``ValueError``. + """ + for too_short in (999_999, 10): + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/millionbytes", + ) + ) + + with self.assertRaises(ValueError): + result_of(limited_content(response, too_short)) + class HttpTestFixture(Fixture): """ From 451e68795cf5cfb02fabdf6baa870289b978a8f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 13:54:58 -0400 Subject: [PATCH 698/916] Lints, better explanation. --- src/allmydata/test/test_storage_http.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 915cd33f2..3108ffae8 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -22,7 +22,6 @@ from base64 import b64encode from contextlib import contextmanager from os import urandom from typing import Union, Callable, Tuple, Iterable -from time import sleep, time from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -32,7 +31,6 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock, Cooperator -from twisted.internet import task from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -370,6 +368,10 @@ class HttpTestFixture(Fixture): def _setUp(self): self.clock = Clock() self.tempdir = self.useFixture(TempDir()) + # The global Cooperator used by Twisted (a) used by pull producers in + # twisted.web, (b) is driven by a real reactor. We want to push time + # forward ourselves since we rely on pull producers in the HTTP storage + # server. self.mock = self.useFixture( MockPatch( "twisted.internet.task._theCooperator", From 03c515191edc519ad045191f18c5d558d4a19e35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:21:21 -0400 Subject: [PATCH 699/916] Better docs. --- src/allmydata/protocol_switch.py | 40 +++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 9b4e30671..20984c615 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -1,8 +1,15 @@ """ -Support for listening with both HTTP and Foolscap on the same port. -""" +Support for listening with both HTTPS and Foolscap on the same port. -from typing import Optional +The goal is to make the transition from Foolscap to HTTPS-based protocols as +simple as possible, with no extra configuration needed. Listening on the same +port means a user upgrading Tahoe-LAFS will automatically get HTTPS working +with no additional changes. + +Use ``create_foolscap_or_http_class()`` to create a new subclass per ``Tub``, +and then ``update_foolscap_or_http_class()`` to add the relevant information to +the subclass once it becomes available later in the configuration process. +""" from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall @@ -17,18 +24,26 @@ from .storage.http_server import HTTPServer from .storage.server import StorageServer -class PretendToBeNegotiation(type): - """😱""" +class _PretendToBeNegotiation(type): + """ + Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a ``Negotiation`` + instance, since Foolscap has some ``assert isinstance(protocol, + Negotiation`` checks. + """ def __instancecheck__(self, instance): return (instance.__class__ == self) or isinstance(instance, Negotiation) -class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): +class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): """ Based on initial query, decide whether we're talking Foolscap or HTTP. - Pretends to be a ``foolscap.negotiate.Negotiation`` instance. + Additionally, pretends to be a ``foolscap.negotiate.Negotiation`` instance, + since these are created by Foolscap's ``Tub``, by setting this to be the + tub's ``negotiationClass``. + + Do not use directly; this needs to be subclassed per ``Tub``. """ # These three will be set by a subclass in update_foolscap_or_http_class() @@ -110,13 +125,22 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def create_foolscap_or_http_class(): - class FoolscapOrHttpWithCert(FoolscapOrHttp): + """ + Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` + instance. + """ + + class FoolscapOrHttpWithCert(_FoolscapOrHttps): pass return FoolscapOrHttpWithCert def update_foolscap_or_http_class(cls, certificate, storage_server, swissnum): + """ + Add the various parameters needed by a ``Tub``-specific + ``_FoolscapOrHttps`` subclass. + """ cls.certificate = certificate cls.storage_server = storage_server cls.swissnum = swissnum From d1bdce9682f9c6c8eefd1488db7c8c8bfc7cdf6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:26:36 -0400 Subject: [PATCH 700/916] A nicer API. --- src/allmydata/client.py | 5 +++-- src/allmydata/node.py | 4 ++-- src/allmydata/protocol_switch.py | 32 +++++++++++++++++--------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 294684b58..2f68c1cb4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -64,7 +64,6 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata import node -from .protocol_switch import update_foolscap_or_http_class KiB=1024 @@ -820,7 +819,9 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - update_foolscap_or_http_class(self.tub.negotiationClass, self.tub.myCertificate, ss, swissnum.encode("ascii")) + self.tub.negotiationClass.add_storage_server( + self.tub.myCertificate, ss, swissnum.encode("ascii") + ) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 93fa6a8e1..597221e9b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -55,7 +55,7 @@ from allmydata.util.yamlutil import ( from . import ( __full_version__, ) -from .protocol_switch import create_foolscap_or_http_class +from .protocol_switch import support_foolscap_and_https def _common_valid_config(): @@ -709,7 +709,7 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) - tub.negotiationClass = create_foolscap_or_http_class() + support_foolscap_and_https(tub) for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 20984c615..7623d68e5 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -6,9 +6,10 @@ simple as possible, with no extra configuration needed. Listening on the same port means a user upgrading Tahoe-LAFS will automatically get HTTPS working with no additional changes. -Use ``create_foolscap_or_http_class()`` to create a new subclass per ``Tub``, -and then ``update_foolscap_or_http_class()`` to add the relevant information to -the subclass once it becomes available later in the configuration process. +Use ``support_foolscap_and_https()`` to create a new subclass for a ``Tub`` +instance, and then ``add_storage_server()`` on the resulting class to add the +relevant information for a storage server once it becomes available later in +the configuration process. """ from twisted.internet.protocol import Protocol @@ -19,6 +20,7 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.internet import reactor from foolscap.negotiate import Negotiation +from foolscap.api import Tub from .storage.http_server import HTTPServer from .storage.server import StorageServer @@ -54,6 +56,16 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): _timeout: IDelayedCall + @classmethod + def add_storage_server(cls, certificate, storage_server, swissnum): + """ + Add the various parameters needed by a ``Tub``-specific + ``_FoolscapOrHttps`` subclass. + """ + cls.certificate = certificate + cls.storage_server = storage_server + cls.swissnum = swissnum + def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) self._buffer: bytes = b"" @@ -124,7 +136,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): self.__dict__ = protocol.__dict__ -def create_foolscap_or_http_class(): +def support_foolscap_and_https(tub: Tub): """ Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` instance. @@ -133,14 +145,4 @@ def create_foolscap_or_http_class(): class FoolscapOrHttpWithCert(_FoolscapOrHttps): pass - return FoolscapOrHttpWithCert - - -def update_foolscap_or_http_class(cls, certificate, storage_server, swissnum): - """ - Add the various parameters needed by a ``Tub``-specific - ``_FoolscapOrHttps`` subclass. - """ - cls.certificate = certificate - cls.storage_server = storage_server - cls.swissnum = swissnum + tub.negotiationClass = FoolscapOrHttpWithCert # type: ignore From 03d9ff395cce0aeff1b1e08b80c8c073907cd3ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:30:19 -0400 Subject: [PATCH 701/916] News file. --- newsfragments/3902.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3902.feature diff --git a/newsfragments/3902.feature b/newsfragments/3902.feature new file mode 100644 index 000000000..2477d0ae6 --- /dev/null +++ b/newsfragments/3902.feature @@ -0,0 +1 @@ +The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well. \ No newline at end of file From 1798966f03c652396d434e14fb41e76607015cc4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:52:12 -0400 Subject: [PATCH 702/916] Store the tub on the subclass, since we'll want it (or rather its Listeners) for NURL construction. --- src/allmydata/client.py | 4 +--- src/allmydata/protocol_switch.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 2f68c1cb4..e737f93e6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -819,9 +819,7 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.tub.negotiationClass.add_storage_server( - self.tub.myCertificate, ss, swissnum.encode("ascii") - ) + self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 7623d68e5..059339575 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -48,21 +48,25 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Do not use directly; this needs to be subclassed per ``Tub``. """ - # These three will be set by a subclass in update_foolscap_or_http_class() - # below. + # These will be set by support_foolscap_and_https() and add_storage_server(). + + # The swissnum for the storage_server. swissnum: bytes - certificate: PrivateCertificate + # The storage server we're exposing. storage_server: StorageServer + # The tub that created us: + tub: Tub + # The certificate for the endpoint: + certificate: PrivateCertificate _timeout: IDelayedCall @classmethod - def add_storage_server(cls, certificate, storage_server, swissnum): + def add_storage_server(cls, storage_server, swissnum): """ - Add the various parameters needed by a ``Tub``-specific - ``_FoolscapOrHttps`` subclass. + Add the various storage server-related attributes needed by a + ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ - cls.certificate = certificate cls.storage_server = storage_server cls.swissnum = swissnum @@ -141,8 +145,10 @@ def support_foolscap_and_https(tub: Tub): Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` instance. """ + the_tub = tub class FoolscapOrHttpWithCert(_FoolscapOrHttps): - pass + tub = the_tub + certificate = tub.myCertificate tub.negotiationClass = FoolscapOrHttpWithCert # type: ignore From 3db6080f6d62907e9eeaa8de5b6cd1a480b96e23 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 15:18:22 -0400 Subject: [PATCH 703/916] Make the factories a class-level attribute. --- src/allmydata/protocol_switch.py | 45 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 059339575..21d896793 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -14,7 +14,7 @@ the configuration process. from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall -from twisted.internet.ssl import CertificateOptions, PrivateCertificate +from twisted.internet.ssl import CertificateOptions from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.internet import reactor @@ -50,25 +50,35 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # These will be set by support_foolscap_and_https() and add_storage_server(). - # The swissnum for the storage_server. - swissnum: bytes - # The storage server we're exposing. - storage_server: StorageServer + # The HTTP storage server API we're exposing. + http_storage_server: HTTPServer + # The Twisted HTTPS protocol factory wrapping the storage server API: + https_factory: TLSMemoryBIOFactory # The tub that created us: tub: Tub - # The certificate for the endpoint: - certificate: PrivateCertificate + # This will be created by the instance in connectionMade(): _timeout: IDelayedCall @classmethod - def add_storage_server(cls, storage_server, swissnum): + def add_storage_server(cls, storage_server: StorageServer, swissnum): """ Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ - cls.storage_server = storage_server - cls.swissnum = swissnum + # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate + # instance. + certificate_options = CertificateOptions( + privateKey=cls.tub.myCertificate.privateKey.original, + certificate=cls.tub.myCertificate.original, + ) + + cls.http_storage_server = HTTPServer(storage_server, swissnum) + cls.https_factory = TLSMemoryBIOFactory( + certificate_options, + False, + Site(cls.http_storage_server.get_resource()), + ) def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) @@ -124,16 +134,8 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): return else: # We're a HTTPS protocol instance, serving the storage protocol: - certificate_options = CertificateOptions( - privateKey=self.certificate.privateKey.original, - certificate=self.certificate.original, - ) - http_server = HTTPServer(self.storage_server, self.swissnum) - factory = TLSMemoryBIOFactory( - certificate_options, False, Site(http_server.get_resource()) - ) assert self.transport is not None - protocol = factory.buildProtocol(self.transport.getPeer()) + protocol = self.https_factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) self.__class__ = protocol.__class__ @@ -147,8 +149,7 @@ def support_foolscap_and_https(tub: Tub): """ the_tub = tub - class FoolscapOrHttpWithCert(_FoolscapOrHttps): + class FoolscapOrHttpForTub(_FoolscapOrHttps): tub = the_tub - certificate = tub.myCertificate - tub.negotiationClass = FoolscapOrHttpWithCert # type: ignore + tub.negotiationClass = FoolscapOrHttpForTub # type: ignore From 70dfc4484173bb9592d02834e14bb85d8356a14c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 15:45:30 -0400 Subject: [PATCH 704/916] Fix for 3905. --- src/allmydata/storage/http_server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 06a6863fa..f61030844 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -188,7 +188,12 @@ class UploadsInProgress(object): def remove_write_bucket(self, bucket: BucketWriter): """Stop tracking the given ``BucketWriter``.""" - storage_index, share_number = self._bucketwriters.pop(bucket) + try: + storage_index, share_number = self._bucketwriters.pop(bucket) + except KeyError: + # This is probably a BucketWriter created by Foolscap, so just + # ignore it. + return uploads_index = self._uploads[storage_index] uploads_index.shares.pop(share_number) uploads_index.upload_secrets.pop(share_number) From f2acf71998475bb8b5eef981f3c3b93e23432561 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 15:58:52 -0400 Subject: [PATCH 705/916] Document next steps: NURL generation. --- src/allmydata/protocol_switch.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 21d896793..9f33560e7 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -66,6 +66,14 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ + # TODO tub.locationHints will be in the format ["tcp:hostname:port"] + # (and maybe some other things we can ignore for now). We also have + # access to the certificate. Together, this should be sufficient to + # construct NURLs, one per hint. The code for NURls should be + # refactored out of http_server.py's build_nurl; that code might want + # to skip around for the future when we don't do foolscap, but for now + # this module will be main way we set up HTTPS. + # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. certificate_options = CertificateOptions( From 249f43184972d124d5144dccf52f3cb78662a523 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:14:52 -0400 Subject: [PATCH 706/916] Use MonkeyPatch instead of MockPatch, since we're not mocking. --- src/allmydata/test/test_storage_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 3108ffae8..811cc2ac1 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,7 +25,7 @@ from typing import Union, Callable, Tuple, Iterable from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir, MockPatch +from fixtures import Fixture, TempDir, MonkeyPatch from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL @@ -373,7 +373,7 @@ class HttpTestFixture(Fixture): # forward ourselves since we rely on pull producers in the HTTP storage # server. self.mock = self.useFixture( - MockPatch( + MonkeyPatch( "twisted.internet.task._theCooperator", Cooperator(scheduler=lambda c: self.clock.callLater(0.000001, c)), ) From 97d0ba23ebc48c3b5af378446b5adc3189b608a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:21:46 -0400 Subject: [PATCH 707/916] Switch to hypothesis-based test. --- src/allmydata/test/test_storage_http.py | 31 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 811cc2ac1..5c429af88 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -235,6 +235,13 @@ class RouteConverterTests(SyncTestCase): SWISSNUM_FOR_TEST = b"abcd" +def gen_bytes(length: int) -> bytes: + """Generate bytes to the given length.""" + result = (b"0123456789abcdef" * ((length // 16) + 1))[:length] + assert len(result) == length + return result + + class TestApp(object): """HTTP API for testing purposes.""" @@ -254,10 +261,10 @@ class TestApp(object): request.setHeader("content-type", CBOR_MIME_TYPE) return dumps({"garbage": 123}) - @_authorized_route(_app, set(), "/millionbytes", methods=["GET"]) - def million_bytes(self, request, authorization): - """Return 1,000,000 bytes.""" - return b"0123456789" * 100_000 + @_authorized_route(_app, set(), "/bytes/", methods=["GET"]) + def generate_bytes(self, request, authorization, length): + """Return bytes to the given length using ``gen_bytes()``.""" + return gen_bytes(length) def result_of(d): @@ -324,34 +331,36 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(CDDLValidationError): result_of(client.get_version()) - def test_limited_content_fits(self): + @given(length=st.integers(min_value=1, max_value=1_000_000)) + def test_limited_content_fits(self, length): """ ``http_client.limited_content()`` returns the body if it is less than the max length. """ - for at_least_length in (1_000_000, 1_000_001): + for at_least_length in (length, length + 1, length + 1000): response = result_of( self.client.request( "GET", - "http://127.0.0.1/millionbytes", + f"http://127.0.0.1/bytes/{length}", ) ) self.assertEqual( result_of(limited_content(response, at_least_length)), - b"0123456789" * 100_000, + gen_bytes(length), ) - def test_limited_content_does_not_fit(self): + @given(length=st.integers(min_value=10, max_value=1_000_000)) + def test_limited_content_does_not_fit(self, length): """ If the body is longer than than max length, ``http_client.limited_content()`` fails with a ``ValueError``. """ - for too_short in (999_999, 10): + for too_short in (length - 1, 5): response = result_of( self.client.request( "GET", - "http://127.0.0.1/millionbytes", + f"http://127.0.0.1/bytes/{length}", ) ) From 1e6864ac0116b834be34ae556b80a7ca52f07e28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:30:01 -0400 Subject: [PATCH 708/916] Typo. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50e4ec946..ffba354bb 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -285,7 +285,7 @@ _SCHEMAS = { } -# Callabale that takes offset and length, returns the data at that range. +# Callable that takes offset and length, returns the data at that range. ReadData = Callable[[int, int], bytes] From 3270d24c45d1613b5418f6f189517b859b4afdaa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:30:48 -0400 Subject: [PATCH 709/916] Slight simplification. --- src/allmydata/storage/http_server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ffba354bb..c727b5e95 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -24,7 +24,7 @@ from twisted.web.server import Site, Request from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath -from attrs import define, field +from attrs import define, field, Factory from werkzeug.http import ( parse_range_header, parse_content_range_header, @@ -149,11 +149,11 @@ class StorageIndexUploads(object): """ # Map share number to BucketWriter - shares: dict[int, BucketWriter] = field(factory=dict) + shares: dict[int, BucketWriter] = Factory(dict) # Map share number to the upload secret (different shares might have # different upload secrets). - upload_secrets: dict[int, bytes] = field(factory=dict) + upload_secrets: dict[int, bytes] = Factory(dict) @define @@ -163,10 +163,10 @@ class UploadsInProgress(object): """ # Map storage index to corresponding uploads-in-progress - _uploads: dict[bytes, StorageIndexUploads] = field(factory=dict) + _uploads: dict[bytes, StorageIndexUploads] = Factory(dict) # Map BucketWriter to (storage index, share number) - _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = field(factory=dict) + _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = Factory(dict) def add_write_bucket( self, @@ -299,7 +299,7 @@ class _ReadAllProducer: request: Request read_data: ReadData - result: Deferred = field(factory=Deferred) + result: Deferred = Factory(Deferred) start: int = field(default=0) @classmethod From 6e3ca256b9eaf4240a782abfcde887d360b10f10 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 15:36:21 -0400 Subject: [PATCH 710/916] Some refactoring to handle edge cases better, in progress. --- src/allmydata/storage/http_server.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c727b5e95..d55d12711 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -349,26 +349,35 @@ class _ReadRangeProducer: to_read = min(self.remaining, 65536) data = self.read_data(self.start, to_read) assert len(data) <= to_read - if self.first_read and data: + + if self.first_read and self.remaining > 0: # For empty bodies the content-range header makes no sense since # the end of the range is inclusive. self.request.setHeader( "content-range", - ContentRange("bytes", self.start, self.start + len(data)).to_header(), + ContentRange( + "bytes", self.start, self.start + self.remaining + ).to_header(), ) + self.first_read = False + + if not data and self.remaining > 0: + # Either data is missing locally (storage issue?) or a bug + pass # TODO abort. TODO test + + self.start += len(data) + self.remaining -= len(data) + assert self.remaining >= 0 + self.request.write(data) - if not data or len(data) < to_read: + if self.remaining == 0: self.request.unregisterProducer() d = self.result del self.result d.callback(b"") return - self.start += len(data) - self.remaining -= len(data) - assert self.remaining >= 0 - def pauseProducing(self): pass @@ -412,6 +421,8 @@ def read_range(request: Request, read_data: ReadData) -> None: ): raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) + # TODO if end is beyond the end of the share, either return error, or maybe + # just return what we can... offset, end = range_header.ranges[0] request.setResponseCode(http.PARTIAL_CONTENT) d = Deferred() From 69c4dbf2b5e04cb3dd9e79ea9b98686178d777c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 17:17:38 -0400 Subject: [PATCH 711/916] Fix tests and point to future work. --- src/allmydata/storage/http_server.py | 15 ++++++++++++--- src/allmydata/test/test_storage_http.py | 4 +++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d55d12711..9d90ba960 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -353,6 +353,11 @@ class _ReadRangeProducer: if self.first_read and self.remaining > 0: # For empty bodies the content-range header makes no sense since # the end of the range is inclusive. + # + # TODO this is wrong for requests that go beyond the end of the + # share. This will be fixed in + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 by making that + # edge case not happen. self.request.setHeader( "content-range", ContentRange( @@ -362,8 +367,11 @@ class _ReadRangeProducer: self.first_read = False if not data and self.remaining > 0: - # Either data is missing locally (storage issue?) or a bug - pass # TODO abort. TODO test + # TODO Either data is missing locally (storage issue?) or a bug, + # abort response with error? Until + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 is implemented + # we continue anyway. + pass self.start += len(data) self.remaining -= len(data) @@ -371,7 +379,8 @@ class _ReadRangeProducer: self.request.write(data) - if self.remaining == 0: + # TODO remove the second clause in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 + if self.remaining == 0 or not data: self.request.unregisterProducer() d = self.result del self.result diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5c429af88..4e44a9f96 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1451,8 +1451,10 @@ class SharedImmutableMutableTestsMixin: ) check_range("bytes=0-10", "bytes 0-10/*") + check_range("bytes=3-17", "bytes 3-17/*") + # TODO re-enable in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 # Can't go beyond the end of the mutable/immutable! - check_range("bytes=10-100", "bytes 10-25/*") + #check_range("bytes=10-100", "bytes 10-25/*") class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): From 5c5556d91505b659ce44e33c31e2ef82d4b079d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:38:31 -0400 Subject: [PATCH 712/916] More robust usage. --- src/allmydata/storage/http_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b8bd0bf20..0ccc3c4a1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -18,7 +18,7 @@ from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS -from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred +from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.internet.ssl import CertificateOptions from twisted.web.client import Agent, HTTPConnectionPool @@ -137,7 +137,10 @@ def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred: fails with a ``ValueError``. """ collector = _LengthLimitedCollector(max_length) - d = treq.collect(response, collector) + # Make really sure everything gets called in Deferred context, treq might + # call collector directly... + d = succeed(None) + d.addCallback(lambda _: treq.collect(response, collector)) d.addCallback(lambda _: collector.f.getvalue()) return d From dac0080ea26cbbe83dfaaf06a777e7b5a554fa63 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:40:46 -0400 Subject: [PATCH 713/916] Make sure we update remaining length, and update test to catch the edge case this fixes. --- src/allmydata/storage/http_client.py | 3 ++- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 0ccc3c4a1..daadebb28 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -125,7 +125,8 @@ class _LengthLimitedCollector: f: BytesIO = field(factory=BytesIO) def __call__(self, data: bytes): - if len(data) > self.remaining_length: + self.remaining_length -= len(data) + if self.remaining_length < 0: raise ValueError("Response length was too long") self.f.write(data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4e44a9f96..533771866 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -337,7 +337,7 @@ class CustomHTTPServerTests(SyncTestCase): ``http_client.limited_content()`` returns the body if it is less than the max length. """ - for at_least_length in (length, length + 1, length + 1000): + for at_least_length in (length, length + 1, length + 1000, length + 100_000): response = result_of( self.client.request( "GET", From fd8a385d1d70a52ecf26eade6c9f3933d73fef79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:46:59 -0400 Subject: [PATCH 714/916] Reformat with black. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 533771866..885750441 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1454,7 +1454,7 @@ class SharedImmutableMutableTestsMixin: check_range("bytes=3-17", "bytes 3-17/*") # TODO re-enable in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 # Can't go beyond the end of the mutable/immutable! - #check_range("bytes=10-100", "bytes 10-25/*") + # check_range("bytes=10-100", "bytes 10-25/*") class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): From 0b5132745ddfd8c2b14f66359535dc8e3c7a1eab Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:47:08 -0400 Subject: [PATCH 715/916] A nicer interface. --- src/allmydata/storage/http_client.py | 18 ++++++++++++++---- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index daadebb28..b8ba1641a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -4,7 +4,7 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations -from typing import Union, Optional, Sequence, Mapping +from typing import Union, Optional, Sequence, Mapping, BinaryIO from base64 import b64encode from io import BytesIO @@ -131,25 +131,35 @@ class _LengthLimitedCollector: self.f.write(data) -def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred: +def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[BinaryIO]: """ Like ``treq.content()``, but limit data read from the response to a set length. If the response is longer than the max allowed length, the result fails with a ``ValueError``. + + A potentially useful future improvement would be using a temporary file to + store the content; since filesystem buffering means that would use memory + for small responses and disk for large responses. """ collector = _LengthLimitedCollector(max_length) # Make really sure everything gets called in Deferred context, treq might # call collector directly... d = succeed(None) d.addCallback(lambda _: treq.collect(response, collector)) - d.addCallback(lambda _: collector.f.getvalue()) + + def done(_): + collector.f.seek(0) + return collector.f + + d.addCallback(done) return d def _decode_cbor(response, schema: Schema): """Given HTTP response, return decoded CBOR body.""" - def got_content(data): + def got_content(f: BinaryIO): + data = f.read() schema.validate_cbor(data) return loads(data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 885750441..419052282 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -346,7 +346,7 @@ class CustomHTTPServerTests(SyncTestCase): ) self.assertEqual( - result_of(limited_content(response, at_least_length)), + result_of(limited_content(response, at_least_length)).read(), gen_bytes(length), ) From 87932e3444267a50c4a00700d356fda4057a9b14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:50:16 -0400 Subject: [PATCH 716/916] Correct type. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 9d90ba960..c53906218 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable +from typing import Dict, List, Set, Tuple, Any, Callable, Union from functools import wraps from base64 import b64decode import binascii @@ -394,7 +394,7 @@ class _ReadRangeProducer: pass -def read_range(request: Request, read_data: ReadData) -> None: +def read_range(request: Request, read_data: ReadData) -> Union[Deferred, bytes]: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. From a24aefaebf8f0487b4a8cc981c7cb238d0aca1d2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Jul 2022 11:35:28 -0400 Subject: [PATCH 717/916] There can be up to 256 shares. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c53906218..a29742bab 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -253,7 +253,7 @@ _SCHEMAS = { "allocate_buckets": Schema( """ request = { - share-numbers: #6.258([*30 uint]) + share-numbers: #6.258([*256 uint]) allocated-size: uint } """ From 49dfc8445cec28d6d903d0a15ee69c411b1b70a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 14:12:12 -0400 Subject: [PATCH 718/916] Implementation of getting length of shares (albeit inefficiently for now). --- src/allmydata/storage/immutable.py | 8 ++++++++ src/allmydata/storage/mutable.py | 10 +++++----- src/allmydata/storage/server.py | 10 ++++++++++ src/allmydata/test/test_storage.py | 22 ++++++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 920bd3c5e..2c65304b8 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -199,8 +199,13 @@ class ShareFile(object): raise UnknownImmutableContainerVersionError(filename, version) self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) + self._length = filesize - 0xc - (num_leases * self.LEASE_SIZE) + self._data_offset = 0xc + def get_length(self): + return self._length + def unlink(self): os.unlink(self.home) @@ -544,6 +549,9 @@ class BucketReader(object): self.shnum, reason) + def get_length(self): + return self._share_file.get_length() + @implementer(RIBucketReader) class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index bd59d96b8..9a99979e9 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -412,11 +412,11 @@ class MutableShareFile(object): datav.append(self._read_share_data(f, offset, length)) return datav -# def remote_get_length(self): -# f = open(self.home, 'rb') -# data_length = self._read_data_length(f) -# f.close() -# return data_length + def get_length(self): + f = open(self.home, 'rb') + data_length = self._read_data_length(f) + f.close() + return data_length def check_write_enabler(self, write_enabler, si_s): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0a1999dfb..f452885d0 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -794,6 +794,16 @@ class StorageServer(service.MultiService): return None + def get_immutable_share_length(self, storage_index: bytes, share_number: int) -> int: + """Returns the length (in bytes) of an immutable.""" + return self.get_buckets(storage_index)[share_number].get_length() + + def get_mutable_share_length(self, storage_index: bytes, share_number: int) -> int: + """Returns the length (in bytes) of a mutable.""" + return MutableShareFile( + dict(self.get_shares(storage_index))[share_number] + ).get_length() + @implementer(RIStorageServer) class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 91d55790e..bb8d48d2f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -688,6 +688,15 @@ class Server(unittest.TestCase): writer.abort() self.failUnlessEqual(ss.allocated_size(), 0) + def test_immutable_length(self): + """``get_immutable_share_length()`` returns the length of an immutable share.""" + ss = self.create("test_immutable_length") + _, writers = self.allocate(ss, b"allocate", [22], 75) + bucket = writers[22] + bucket.write(0, b"X" * 75) + bucket.close() + self.assertEqual(ss.get_immutable_share_length(b"allocate", 22), 75) + def test_allocate(self): ss = self.create("test_allocate") @@ -1340,6 +1349,19 @@ class MutableServer(unittest.TestCase): (set(), {0, 1, 2, 4}, {0, 1, 4}) ) + def test_mutable_share_length(self): + """``get_mutable_share_length()`` returns the length of the share.""" + ss = self.create("test_mutable_share_length") + self.allocate(ss, b"si1", b"we1", b"le1", [16], 23) + ss.slot_testv_and_readv_and_writev( + b"si1", (self.write_enabler(b"we1"), + self.renew_secret(b"le1"), + self.cancel_secret(b"le1")), + {16: ([], [(0, b"x" * 23)], None)}, + [] + ) + self.assertEqual(ss.get_mutable_share_length(b"si1", 16), 23) + def test_bad_magic(self): ss = self.create("test_bad_magic") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10) From b3aff5c43b73718de0ecffa6c39d5efac2f6d336 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jul 2022 14:37:46 -0400 Subject: [PATCH 719/916] More efficient implementations. --- src/allmydata/storage/server.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f452885d0..1b9b051eb 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -796,13 +796,16 @@ class StorageServer(service.MultiService): def get_immutable_share_length(self, storage_index: bytes, share_number: int) -> int: """Returns the length (in bytes) of an immutable.""" - return self.get_buckets(storage_index)[share_number].get_length() + si_dir = storage_index_to_dir(storage_index) + path = os.path.join(self.sharedir, si_dir, str(share_number)) + bucket = BucketReader(self, path, storage_index, share_number) + return bucket.get_length() def get_mutable_share_length(self, storage_index: bytes, share_number: int) -> int: """Returns the length (in bytes) of a mutable.""" - return MutableShareFile( - dict(self.get_shares(storage_index))[share_number] - ).get_length() + si_dir = storage_index_to_dir(storage_index) + path = os.path.join(self.sharedir, si_dir, str(share_number)) + return MutableShareFile(path).get_length() @implementer(RIStorageServer) From 1b8b71b3068e73486d406f6c79fbbdbf8ecaf261 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jul 2022 16:10:22 -0400 Subject: [PATCH 720/916] Content-Range headers are now checked (somewhat) and the server now sends correct headers when reading beyond the end. --- src/allmydata/storage/http_client.py | 29 ++++++++++++++++++++++-- src/allmydata/storage/http_server.py | 34 ++++++++++++++++++---------- src/allmydata/storage/server.py | 2 ++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b8ba1641a..11c9ab2fc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -7,6 +7,7 @@ from __future__ import annotations from typing import Union, Optional, Sequence, Mapping, BinaryIO from base64 import b64encode from io import BytesIO +from os import SEEK_END from attrs import define, asdict, frozen, field @@ -29,6 +30,7 @@ from treq.client import HTTPClient from treq.testing import StubTreq from OpenSSL import SSL from cryptography.hazmat.bindings.openssl.binding import Binding +from werkzeug.http import parse_content_range_header from .http_common import ( swissnum_auth_header, @@ -461,13 +463,36 @@ def read_share_chunk( "GET", url, headers=Headers( + # Ranges in HTTP are _inclusive_, Python's convention is exclusive, + # but Range constructor does that the conversion for us. {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), ) if response.code == http.PARTIAL_CONTENT: - body = yield response.content() - returnValue(body) + content_range = parse_content_range_header( + response.headers.getRawHeaders("content-range")[0] + ) + supposed_length = content_range.stop - content_range.start + if supposed_length > length: + raise ValueError("Server sent more than we asked for?!") + # It might also send less than we asked for. That's (probably) OK, e.g. + # if we went past the end of the file. + body = yield limited_content(response, supposed_length) + body.seek(0, SEEK_END) + actual_length = body.tell() + if actual_length != supposed_length: + # Most likely a mutable that got changed out from under us, but + # concievably could be a bug... + raise ValueError( + f"Length of response sent from server ({actual_length}) " + + f"didn't match Content-Range header ({supposed_length})" + ) + body.seek(0) + returnValue(body.read()) else: + # Technically HTTP allows sending an OK with full body under these + # circumstances, but the server is not designed to do that so we ignore + # than possibility for now... raise ClientException(response.code) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a29742bab..4eecf7f2f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -352,12 +352,10 @@ class _ReadRangeProducer: if self.first_read and self.remaining > 0: # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - # - # TODO this is wrong for requests that go beyond the end of the - # share. This will be fixed in - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 by making that - # edge case not happen. + # the end of the range is inclusive. Actual conversion from + # Python's exclusive ranges to inclusive ranges is handled by + # werkzeug. The case where we're reading beyond the end of the + # share is handled by caller (read_range().) self.request.setHeader( "content-range", ContentRange( @@ -368,7 +366,7 @@ class _ReadRangeProducer: if not data and self.remaining > 0: # TODO Either data is missing locally (storage issue?) or a bug, - # abort response with error? Until + # abort response with error. Until # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 is implemented # we continue anyway. pass @@ -394,7 +392,9 @@ class _ReadRangeProducer: pass -def read_range(request: Request, read_data: ReadData) -> Union[Deferred, bytes]: +def read_range( + request: Request, read_data: ReadData, share_length: int +) -> Union[Deferred, bytes]: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -430,9 +430,12 @@ def read_range(request: Request, read_data: ReadData) -> Union[Deferred, bytes]: ): raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) - # TODO if end is beyond the end of the share, either return error, or maybe - # just return what we can... offset, end = range_header.ranges[0] + # If we're being ask to read beyond the length of the share, just read + # less: + end = min(end, share_length) + # TODO when if end is now <= offset? + request.setResponseCode(http.PARTIAL_CONTENT) d = Deferred() request.registerProducer( @@ -675,7 +678,7 @@ class HTTPServer(object): request.setResponseCode(http.NOT_FOUND) return b"" - return read_range(request, bucket.read) + return read_range(request, bucket.read, bucket.get_length()) @_authorized_route( _app, @@ -763,6 +766,13 @@ class HTTPServer(object): def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" + try: + share_length = self._storage_server.get_mutable_share_length( + storage_index, share_number + ) + except KeyError: + raise _HTTPError(http.NOT_FOUND) + def read_data(offset, length): try: return self._storage_server.slot_readv( @@ -771,7 +781,7 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - return read_range(request, read_data) + return read_range(request, read_data, share_length) @_authorized_route( _app, set(), "/v1/mutable//shares", methods=["GET"] diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 1b9b051eb..88b650bb9 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -805,6 +805,8 @@ class StorageServer(service.MultiService): """Returns the length (in bytes) of a mutable.""" si_dir = storage_index_to_dir(storage_index) path = os.path.join(self.sharedir, si_dir, str(share_number)) + if not os.path.exists(path): + raise KeyError("No such storage index or share number") return MutableShareFile(path).get_length() From 43c6af6fde66ca49aed735aba9d927ae44e86a2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 11:28:14 -0400 Subject: [PATCH 721/916] More error handling for edge cases. --- src/allmydata/storage/http_server.py | 42 ++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4eecf7f2f..82d3d4794 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -355,7 +355,7 @@ class _ReadRangeProducer: # the end of the range is inclusive. Actual conversion from # Python's exclusive ranges to inclusive ranges is handled by # werkzeug. The case where we're reading beyond the end of the - # share is handled by caller (read_range().) + # share is handled by the caller, read_range(). self.request.setHeader( "content-range", ContentRange( @@ -365,11 +365,24 @@ class _ReadRangeProducer: self.first_read = False if not data and self.remaining > 0: - # TODO Either data is missing locally (storage issue?) or a bug, - # abort response with error. Until - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 is implemented - # we continue anyway. - pass + d, self.result = self.result, None + d.errback( + ValueError( + f"Should be {remaining} bytes left, but we got an empty read" + ) + ) + self.stopProducing() + return + + if len(data) > self.remaining: + d, self.result = self.result, None + d.errback( + ValueError( + f"Should be {remaining} bytes left, but we got more than that ({len(data)})!" + ) + ) + self.stopProducing() + return self.start += len(data) self.remaining -= len(data) @@ -377,19 +390,20 @@ class _ReadRangeProducer: self.request.write(data) - # TODO remove the second clause in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 - if self.remaining == 0 or not data: - self.request.unregisterProducer() - d = self.result - del self.result - d.callback(b"") - return + if self.remaining == 0: + self.stopProducing() def pauseProducing(self): pass def stopProducing(self): - pass + if self.request is not None: + self.request.unregisterProducer() + self.request = None + if self.result is not None: + d = self.result + self.result = None + d.callback(b"") def read_range( From 69739f5f9bf28d6e35c8ceacafad4554056fabd5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 11:42:01 -0400 Subject: [PATCH 722/916] Handle case where requested range results in empty response. --- docs/proposed/http-storage-node-protocol.rst | 2 ++ src/allmydata/storage/http_client.py | 6 +++- src/allmydata/storage/http_server.py | 29 +++++++++----------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 7e0b4a542..6a4e4136a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -654,6 +654,8 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. + Discussion `````````` diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 11c9ab2fc..236ec970f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -468,6 +468,10 @@ def read_share_chunk( {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), ) + + if response.code == http.NO_CONTENT: + return b"" + if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( response.headers.getRawHeaders("content-range")[0] @@ -488,7 +492,7 @@ def read_share_chunk( + f"didn't match Content-Range header ({supposed_length})" ) body.seek(0) - returnValue(body.read()) + return body.read() else: # Technically HTTP allows sending an OK with full body under these # circumstances, but the server is not designed to do that so we ignore diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 82d3d4794..cb55afffe 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -343,27 +343,12 @@ class _ReadRangeProducer: result: Deferred start: int remaining: int - first_read: bool = field(default=True) def resumeProducing(self): to_read = min(self.remaining, 65536) data = self.read_data(self.start, to_read) assert len(data) <= to_read - if self.first_read and self.remaining > 0: - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. Actual conversion from - # Python's exclusive ranges to inclusive ranges is handled by - # werkzeug. The case where we're reading beyond the end of the - # share is handled by the caller, read_range(). - self.request.setHeader( - "content-range", - ContentRange( - "bytes", self.start, self.start + self.remaining - ).to_header(), - ) - self.first_read = False - if not data and self.remaining > 0: d, self.result = self.result, None d.errback( @@ -448,9 +433,21 @@ def read_range( # If we're being ask to read beyond the length of the share, just read # less: end = min(end, share_length) - # TODO when if end is now <= offset? + if offset >= end: + # Basically we'd need to return an empty body. However, the + # Content-Range header can't actually represent empty lengths... so + # (mis)use 204 response code to indicate that. + raise _HTTPError(http.NO_CONTENT) request.setResponseCode(http.PARTIAL_CONTENT) + + # Actual conversion from Python's exclusive ranges to inclusive ranges is + # handled by werkzeug. + request.setHeader( + "content-range", + ContentRange("bytes", offset, end).to_header(), + ) + d = Deferred() request.registerProducer( _ReadRangeProducer( From 92392501a7439c582599cc7cc36a6270926770c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 11:47:15 -0400 Subject: [PATCH 723/916] Expand spec. --- docs/proposed/http-storage-node-protocol.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 6a4e4136a..09b523c87 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -654,6 +654,9 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +If the response reads beyond the end fo the data, the response may be shorter than then requested range. +The resulting ``Content-Range`` header will be consistent with the returned data. + If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. Discussion @@ -756,6 +759,11 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +If the response reads beyond the end fo the data, the response may be shorter than then requested range. +The resulting ``Content-Range`` header will be consistent with the returned data. + +If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. + ``POST /v1/mutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From da8a36fac9af1f37eb34bf7ab0a46253e906980b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 12:07:46 -0400 Subject: [PATCH 724/916] Improve test coverage. --- src/allmydata/test/test_storage.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bb8d48d2f..c3f2a35e1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -689,13 +689,17 @@ class Server(unittest.TestCase): self.failUnlessEqual(ss.allocated_size(), 0) def test_immutable_length(self): - """``get_immutable_share_length()`` returns the length of an immutable share.""" + """ + ``get_immutable_share_length()`` returns the length of an immutable + share, as does ``BucketWriter.get_length()``.. + """ ss = self.create("test_immutable_length") _, writers = self.allocate(ss, b"allocate", [22], 75) bucket = writers[22] bucket.write(0, b"X" * 75) bucket.close() self.assertEqual(ss.get_immutable_share_length(b"allocate", 22), 75) + self.assertEqual(ss.get_buckets(b"allocate")[22].get_length(), 75) def test_allocate(self): ss = self.create("test_allocate") @@ -1362,6 +1366,26 @@ class MutableServer(unittest.TestCase): ) self.assertEqual(ss.get_mutable_share_length(b"si1", 16), 23) + def test_mutable_share_length_unknown(self): + """ + ``get_mutable_share_length()`` raises a ``KeyError`` on unknown shares. + """ + ss = self.create("test_mutable_share_length_unknown") + self.allocate(ss, b"si1", b"we1", b"le1", [16], 23) + ss.slot_testv_and_readv_and_writev( + b"si1", (self.write_enabler(b"we1"), + self.renew_secret(b"le1"), + self.cancel_secret(b"le1")), + {16: ([], [(0, b"x" * 23)], None)}, + [] + ) + with self.assertRaises(KeyError): + # Wrong share number. + ss.get_mutable_share_length(b"si1", 17) + with self.assertRaises(KeyError): + # Wrong storage index + ss.get_mutable_share_length(b"unknown", 16) + def test_bad_magic(self): ss = self.create("test_bad_magic") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10) From d85b20b62d92d9cde835e1fb2438660f5e725962 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 12:47:18 -0400 Subject: [PATCH 725/916] Fix lint. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index cb55afffe..68d0740b1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -353,7 +353,7 @@ class _ReadRangeProducer: d, self.result = self.result, None d.errback( ValueError( - f"Should be {remaining} bytes left, but we got an empty read" + f"Should be {self.remaining} bytes left, but we got an empty read" ) ) self.stopProducing() @@ -363,7 +363,7 @@ class _ReadRangeProducer: d, self.result = self.result, None d.errback( ValueError( - f"Should be {remaining} bytes left, but we got more than that ({len(data)})!" + f"Should be {self.remaining} bytes left, but we got more than that ({len(data)})!" ) ) self.stopProducing() From 3b7345205bbb68a79f079b7c619ca2a912c7c918 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 14:24:10 -0400 Subject: [PATCH 726/916] News file. --- newsfragments/3709.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3709.minor diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor new file mode 100644 index 000000000..e69de29bb From 11f4ebc0d90ed80f612d29945cd2436f43f658ea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 15:12:00 -0400 Subject: [PATCH 727/916] Hook up NURL generation to the new Foolscap/HTTPS protocol switch. --- src/allmydata/client.py | 16 ++++++++ src/allmydata/protocol_switch.py | 8 ---- src/allmydata/storage/http_server.py | 46 +++++++++++++++-------- src/allmydata/test/test_istorageserver.py | 33 +++------------- 4 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e737f93e6..3318bbfa4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -37,6 +37,7 @@ import allmydata from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix from allmydata.storage.server import StorageServer, FoolscapStorageServer +from allmydata.storage.http_server import build_nurl from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper @@ -658,6 +659,12 @@ class _Client(node.Node, pollmixin.PollMixin): if webport: self.init_web(webport) # strports string + # TODO this may be the wrong location for now? but as temporary measure + # it allows us to get NURLs for testing in test_istorageserver.py Will + # eventually get fixed one way or another in + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901 + self.storage_nurls = [] + def init_stats_provider(self): self.stats_provider = StatsProvider(self) self.stats_provider.setServiceParent(self) @@ -820,6 +827,15 @@ class _Client(node.Node, pollmixin.PollMixin): furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + for location_hint in self.tub.locationHints: + if location_hint.startswith("tcp:"): + _, hostname, port = location_hint.split(":") + port = int(port) + self.storage_nurls.append( + build_nurl( + hostname, port, swissnum, self.tub.myCertificate.original.to_cryptography() + ) + ) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 9f33560e7..21d896793 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -66,14 +66,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ - # TODO tub.locationHints will be in the format ["tcp:hostname:port"] - # (and maybe some other things we can ignore for now). We also have - # access to the certificate. Together, this should be sufficient to - # construct NURLs, one per hint. The code for NURls should be - # refactored out of http_server.py's build_nurl; that code might want - # to skip around for the future when we don't do foolscap, but for now - # this module will be main way we set up HTTPS. - # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. certificate_options = CertificateOptions( diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index e2b754b0d..7f7c1c0ae 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -10,6 +10,7 @@ from base64 import b64decode import binascii from tempfile import TemporaryFile +from cryptography.x509 import Certificate from zope.interface import implementer from klein import Klein from twisted.web import http @@ -843,6 +844,29 @@ class _TLSEndpointWrapper(object): ) +def build_nurl( + hostname: str, port: int, swissnum: str, certificate: Certificate +) -> DecodedURL: + """ + Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509 + certificate for the server. Clients can then connect to the server using + this NURL. + """ + return DecodedURL().replace( + fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) + host=hostname, + port=port, + path=(swissnum,), + userinfo=( + str( + get_spki_hash(certificate), + "ascii", + ), + ), + scheme="pb", + ) + + def listen_tls( server: HTTPServer, hostname: str, @@ -862,22 +886,14 @@ def listen_tls( """ endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path) - def build_nurl(listening_port: IListeningPort) -> DecodedURL: - nurl = DecodedURL().replace( - fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) - host=hostname, - port=listening_port.getHost().port, - path=(str(server._swissnum, "ascii"),), - userinfo=( - str( - get_spki_hash(load_pem_x509_certificate(cert_path.getContent())), - "ascii", - ), - ), - scheme="pb", + def get_nurl(listening_port: IListeningPort) -> DecodedURL: + return build_nurl( + hostname, + listening_port.getHost().port, + str(server._swissnum, "ascii"), + load_pem_x509_certificate(cert_path.getContent()), ) - return nurl return endpoint.listen(Site(server.get_resource())).addCallback( - lambda listening_port: (build_nurl(listening_port), listening_port) + lambda listening_port: (get_nurl(listening_port), listening_port) ) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 39675336f..12a3cba55 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1084,40 +1084,17 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" - def setUp(self): - self._port_assigner = SameProcessStreamEndpointAssigner() - self._port_assigner.setUp() - self.addCleanup(self._port_assigner.tearDown) - return _SharedMixin.setUp(self) - - @inlineCallbacks def _get_istorage_server(self): - swissnum = b"1234" - http_storage_server = HTTPServer(self.server, swissnum) - - # Listen on randomly assigned port, using self-signed cert: - private_key = generate_private_key() - certificate = generate_certificate(private_key) - _, endpoint_string = self._port_assigner.assign(reactor) - nurl, listening_port = yield listen_tls( - http_storage_server, - "127.0.0.1", - serverFromString(reactor, endpoint_string), - private_key_to_file(FilePath(self.mktemp()), private_key), - cert_to_file(FilePath(self.mktemp()), certificate), - ) - self.addCleanup(listening_port.stopListening) + nurl = self.clients[0].storage_nurls[0] # Create HTTP client with non-persistent connections, so we don't leak # state across tests: - returnValue( - _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, persistent=False) - ) + client: IStorageServer = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl(nurl, reactor, persistent=False) ) + self.assertTrue(IStorageServer.providedBy(client)) - # Eventually should also: - # self.assertTrue(IStorageServer.providedBy(client)) + return succeed(client) class FoolscapSharedAPIsTests( From 981b693402929dbe24f4f48cc5a42bb7b95b3285 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 15:25:22 -0400 Subject: [PATCH 728/916] Make HTTPS protocols work with the protocol switcher magic. --- src/allmydata/protocol_switch.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 21d896793..d3e68f860 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -138,6 +138,13 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): protocol = self.https_factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) + + # Update the factory so it knows we're transforming to a new + # protocol object (we'll do that next) + value = self.https_factory.protocols.pop(protocol) + self.https_factory.protocols[self] = value + + # Transform self into the TLS protocol 🪄 self.__class__ = protocol.__class__ self.__dict__ = protocol.__dict__ From 757e8c418c62864246e360be05c9cb25654d9c2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:51:26 -0400 Subject: [PATCH 729/916] Fix typos. --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 09b523c87..3dac376ff 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -654,7 +654,7 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. -If the response reads beyond the end fo the data, the response may be shorter than then requested range. +If the response reads beyond the end of the data, the response may be shorter than the requested range. The resulting ``Content-Range`` header will be consistent with the returned data. If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. @@ -759,7 +759,7 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. -If the response reads beyond the end fo the data, the response may be shorter than then requested range. +If the response reads beyond the end of the data, the response may be shorter than the requested range. The resulting ``Content-Range`` header will be consistent with the returned data. If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. From 5cd9ccfc6ae78be55eca1931402fd512d6199787 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:52:56 -0400 Subject: [PATCH 730/916] Slightly nicer handling for bad edge cases. --- src/allmydata/storage/http_client.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 236ec970f..a464d445a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -474,8 +474,16 @@ def read_share_chunk( if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( - response.headers.getRawHeaders("content-range")[0] + response.headers.getRawHeaders("content-range")[0] or "" ) + if ( + content_range is None + or content_range.stop is None + or content_range.start is None + ): + raise ValueError( + "Content-Range was missing, invalid, or in format we don't support" + ) supposed_length = content_range.stop - content_range.start if supposed_length > length: raise ValueError("Server sent more than we asked for?!") From f671b47a6decb0f53b49697340012119286d0f91 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:53:12 -0400 Subject: [PATCH 731/916] Fix typo. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a464d445a..da272240d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -504,7 +504,7 @@ def read_share_chunk( else: # Technically HTTP allows sending an OK with full body under these # circumstances, but the server is not designed to do that so we ignore - # than possibility for now... + # that possibility for now... raise ClientException(response.code) From 36b96a8776a39d77f017725f2ca4ccb9e4f8cf5c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:53:28 -0400 Subject: [PATCH 732/916] Fix typo. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index da272240d..a2dc5379f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -494,7 +494,7 @@ def read_share_chunk( actual_length = body.tell() if actual_length != supposed_length: # Most likely a mutable that got changed out from under us, but - # concievably could be a bug... + # conceivably could be a bug... raise ValueError( f"Length of response sent from server ({actual_length}) " + f"didn't match Content-Range header ({supposed_length})" From 2b3a8ddeece0e11d095e8c350a2fef66f4deee20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:55:00 -0400 Subject: [PATCH 733/916] Docstring. --- src/allmydata/storage/mutable.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 9a99979e9..51c3a3c8b 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -413,6 +413,9 @@ class MutableShareFile(object): return datav def get_length(self): + """ + Return the length of the data in the share. + """ f = open(self.home, 'rb') data_length = self._read_data_length(f) f.close() From be963e2324c52801998796f0ba2cd931a4bf556f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:55:33 -0400 Subject: [PATCH 734/916] Docstrings. --- src/allmydata/storage/immutable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 2c65304b8..0338af41c 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -204,6 +204,9 @@ class ShareFile(object): self._data_offset = 0xc def get_length(self): + """ + Return the length of the data in the share, if we're reading. + """ return self._length def unlink(self): @@ -549,9 +552,6 @@ class BucketReader(object): self.shnum, reason) - def get_length(self): - return self._share_file.get_length() - @implementer(RIBucketReader) class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 From 83f9c0788b1fca812048fe4cafa912f97d664501 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:56:18 -0400 Subject: [PATCH 735/916] Use more direct API. --- src/allmydata/storage/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 88b650bb9..2bf99d74c 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -798,8 +798,7 @@ class StorageServer(service.MultiService): """Returns the length (in bytes) of an immutable.""" si_dir = storage_index_to_dir(storage_index) path = os.path.join(self.sharedir, si_dir, str(share_number)) - bucket = BucketReader(self, path, storage_index, share_number) - return bucket.get_length() + return ShareFile(path).get_length() def get_mutable_share_length(self, storage_index: bytes, share_number: int) -> int: """Returns the length (in bytes) of a mutable.""" From 94e0568653a2fc49e33ba4be8c1f86b82aa48737 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:57:32 -0400 Subject: [PATCH 736/916] Actually we do need it. --- src/allmydata/storage/immutable.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0338af41c..f7f5aebce 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -552,6 +552,12 @@ class BucketReader(object): self.shnum, reason) + def get_length(self): + """ + Return the length of the data in the share. + """ + return self._share_file.get_length() + @implementer(RIBucketReader) class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 From c14463ac6df6c1d0b4f3a44dd5548de8128c214f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Jul 2022 09:52:40 -0400 Subject: [PATCH 737/916] News file. --- newsfragments/3909.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3909.minor diff --git a/newsfragments/3909.minor b/newsfragments/3909.minor new file mode 100644 index 000000000..e69de29bb From 921e3a771248c24b54e77c9c8d44cc488d572906 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Jul 2022 09:55:03 -0400 Subject: [PATCH 738/916] Don't use broken version of werkzeug. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d07031cd9..c3ee4eb90 100644 --- a/setup.py +++ b/setup.py @@ -133,7 +133,8 @@ install_requires = [ # HTTP server and client "klein", - "werkzeug", + # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 + "werkzeug != 2.2.0", "treq", "cbor2", "pycddl", From 822b652d99296a1d767f88446b2825e09caf8b65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 09:57:18 -0400 Subject: [PATCH 739/916] Improve factoring. --- src/allmydata/client.py | 21 +++++----------- src/allmydata/protocol_switch.py | 30 ++++++++++++++++++++--- src/allmydata/test/test_istorageserver.py | 2 +- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 3318bbfa4..9938ec076 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -37,7 +37,6 @@ import allmydata from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix from allmydata.storage.server import StorageServer, FoolscapStorageServer -from allmydata.storage.http_server import build_nurl from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper @@ -660,10 +659,10 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_web(webport) # strports string # TODO this may be the wrong location for now? but as temporary measure - # it allows us to get NURLs for testing in test_istorageserver.py Will - # eventually get fixed one way or another in + # it allows us to get NURLs for testing in test_istorageserver.py. This + # will eventually get fixed one way or another in # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901 - self.storage_nurls = [] + self.storage_nurls = set() def init_stats_provider(self): self.stats_provider = StatsProvider(self) @@ -826,17 +825,9 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) - for location_hint in self.tub.locationHints: - if location_hint.startswith("tcp:"): - _, hostname, port = location_hint.split(":") - port = int(port) - self.storage_nurls.append( - build_nurl( - hostname, port, swissnum, self.tub.myCertificate.original.to_cryptography() - ) - ) - + self.storage_nurls.update( + self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + ) announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index d3e68f860..f1fa6e061 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -12,6 +12,8 @@ relevant information for a storage server once it becomes available later in the configuration process. """ +from __future__ import annotations + from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall from twisted.internet.ssl import CertificateOptions @@ -19,10 +21,11 @@ from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.internet import reactor +from hyperlink import DecodedURL from foolscap.negotiate import Negotiation from foolscap.api import Tub -from .storage.http_server import HTTPServer +from .storage.http_server import HTTPServer, build_nurl from .storage.server import StorageServer @@ -45,7 +48,9 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): since these are created by Foolscap's ``Tub``, by setting this to be the tub's ``negotiationClass``. - Do not use directly; this needs to be subclassed per ``Tub``. + Do not use directly, use ``support_foolscap_and_https(tub)`` instead. The + way this class works is that a new subclass is created for a specific + ``Tub`` instance. """ # These will be set by support_foolscap_and_https() and add_storage_server(). @@ -61,10 +66,14 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): _timeout: IDelayedCall @classmethod - def add_storage_server(cls, storage_server: StorageServer, swissnum): + def add_storage_server( + cls, storage_server: StorageServer, swissnum: bytes + ) -> set[DecodedURL]: """ Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. + + Returns the resulting NURLs. """ # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. @@ -80,6 +89,21 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Site(cls.http_storage_server.get_resource()), ) + storage_nurls = set() + for location_hint in cls.tub.locationHints: + if location_hint.startswith("tcp:"): + _, hostname, port = location_hint.split(":") + port = int(port) + storage_nurls.add( + build_nurl( + hostname, + port, + str(swissnum, "ascii"), + cls.tub.myCertificate.original.to_cryptography(), + ) + ) + return storage_nurls + def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) self._buffer: bytes = b"" diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 12a3cba55..90159f1f8 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1085,7 +1085,7 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def _get_istorage_server(self): - nurl = self.clients[0].storage_nurls[0] + nurl = list(self.clients[0].storage_nurls)[0] # Create HTTP client with non-persistent connections, so we don't leak # state across tests: From 34518f9d0dcb8fcb91382beb1ead726b811c5dff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:01:09 -0400 Subject: [PATCH 740/916] Fix lints. --- src/allmydata/storage/http_server.py | 4 ++-- src/allmydata/test/test_istorageserver.py | 15 ++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 98611e833..ca8917694 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -10,7 +10,7 @@ from base64 import b64decode import binascii from tempfile import TemporaryFile -from cryptography.x509 import Certificate +from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer from klein import Klein from twisted.web import http @@ -866,7 +866,7 @@ class _TLSEndpointWrapper(object): def build_nurl( - hostname: str, port: int, swissnum: str, certificate: Certificate + hostname: str, port: int, swissnum: str, certificate: CryptoCertificate ) -> DecodedURL: """ Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509 diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 90159f1f8..3328ea598 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -18,21 +18,14 @@ from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor -from twisted.internet.endpoints import serverFromString -from twisted.python.filepath import FilePath from foolscap.api import Referenceable, RemoteException -from allmydata.interfaces import IStorageServer # really, IStorageClient +# A better name for this would be IStorageClient... +from allmydata.interfaces import IStorageServer + from .common_system import SystemTestMixin -from .common import AsyncTestCase, SameProcessStreamEndpointAssigner -from .certs import ( - generate_certificate, - generate_private_key, - private_key_to_file, - cert_to_file, -) +from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage.http_server import HTTPServer, listen_tls from allmydata.storage.http_client import StorageClient from allmydata.storage_client import _HTTPStorageServer From 1cd2185be75e3d2e35307c43e59ab803ebb52ab3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:12:24 -0400 Subject: [PATCH 741/916] More cleanups. --- src/allmydata/protocol_switch.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index f1fa6e061..2e9d404c5 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -48,21 +48,20 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): since these are created by Foolscap's ``Tub``, by setting this to be the tub's ``negotiationClass``. - Do not use directly, use ``support_foolscap_and_https(tub)`` instead. The - way this class works is that a new subclass is created for a specific - ``Tub`` instance. + Do not instantiate directly, use ``support_foolscap_and_https(tub)`` + instead. The way this class works is that a new subclass is created for a + specific ``Tub`` instance. """ - # These will be set by support_foolscap_and_https() and add_storage_server(). + # These are class attributes; they will be set by + # support_foolscap_and_https() and add_storage_server(). - # The HTTP storage server API we're exposing. - http_storage_server: HTTPServer - # The Twisted HTTPS protocol factory wrapping the storage server API: + # The Twisted HTTPS protocol factory wrapping the storage server HTTP API: https_factory: TLSMemoryBIOFactory # The tub that created us: tub: Tub - # This will be created by the instance in connectionMade(): + # This is an instance attribute; it will be set in connectionMade(). _timeout: IDelayedCall @classmethod @@ -70,11 +69,17 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): cls, storage_server: StorageServer, swissnum: bytes ) -> set[DecodedURL]: """ - Add the various storage server-related attributes needed by a - ``Tub``-specific ``_FoolscapOrHttps`` subclass. + Update a ``_FoolscapOrHttps`` subclass for a specific ``Tub`` instance + with the class attributes it requires for a specific storage server. Returns the resulting NURLs. """ + # We need to be a subclass: + assert cls != _FoolscapOrHttps + # The tub instance must already be set: + assert hasattr(cls, "tub") + assert isinstance(cls.tub, Tub) + # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. certificate_options = CertificateOptions( @@ -82,11 +87,11 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): certificate=cls.tub.myCertificate.original, ) - cls.http_storage_server = HTTPServer(storage_server, swissnum) + http_storage_server = HTTPServer(storage_server, swissnum) cls.https_factory = TLSMemoryBIOFactory( certificate_options, False, - Site(cls.http_storage_server.get_resource()), + Site(http_storage_server.get_resource()), ) storage_nurls = set() From 533d2a7ac9576734825822f0621d8db58cb36606 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:15:23 -0400 Subject: [PATCH 742/916] Note Tor and I2P support. --- src/allmydata/protocol_switch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2e9d404c5..5ab4761c6 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -107,6 +107,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): cls.tub.myCertificate.original.to_cryptography(), ) ) + # TODO this is probably where we'll have to support Tor and I2P? + # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9 + # for discussion (there will be separate tickets added for those at + # some point.) return storage_nurls def __init__(self, *args, **kwargs): From d4c73f19fe6eaea7bba25c51e632aef441d7549e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:42:56 -0400 Subject: [PATCH 743/916] A unittest for the metaclass. --- src/allmydata/protocol_switch.py | 10 +++-- src/allmydata/test/test_protocol_switch.py | 43 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 src/allmydata/test/test_protocol_switch.py diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 5ab4761c6..5143cab6a 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -31,13 +31,15 @@ from .storage.server import StorageServer class _PretendToBeNegotiation(type): """ - Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a ``Negotiation`` - instance, since Foolscap has some ``assert isinstance(protocol, - Negotiation`` checks. + Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a + ``Negotiation`` instance, since Foolscap does some checks like + ``assert isinstance(protocol, tub.negotiationClass)`` in its internals, + and sometimes that ``protocol`` is a ``_FoolscapOrHttps`` instance, but + sometimes it's a ``Negotiation`` instance. """ def __instancecheck__(self, instance): - return (instance.__class__ == self) or isinstance(instance, Negotiation) + return issubclass(instance.__class__, self) or isinstance(instance, Negotiation) class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): diff --git a/src/allmydata/test/test_protocol_switch.py b/src/allmydata/test/test_protocol_switch.py new file mode 100644 index 000000000..4906896dc --- /dev/null +++ b/src/allmydata/test/test_protocol_switch.py @@ -0,0 +1,43 @@ +""" +Unit tests for ``allmydata.protocol_switch``. + +By its nature, most of the testing needs to be end-to-end; essentially any test +that uses real Foolscap (``test_system.py``, integration tests) ensures +Foolscap still works. ``test_istorageserver.py`` tests the HTTP support. +""" + +from foolscap.negotiate import Negotiation + +from .common import TestCase +from ..protocol_switch import _PretendToBeNegotiation + + +class UtilityTests(TestCase): + """Tests for utilities in the protocol switch code.""" + + def test_metaclass(self): + """ + A class that has the ``_PretendToBeNegotiation`` metaclass will support + ``isinstance()``'s normal semantics on its own instances, but will also + indicate that ``Negotiation`` instances are its instances. + """ + + class Parent(metaclass=_PretendToBeNegotiation): + pass + + class Child(Parent): + pass + + class Other: + pass + + p = Parent() + self.assertIsInstance(p, Parent) + self.assertIsInstance(Negotiation(), Parent) + self.assertNotIsInstance(Other(), Parent) + + c = Child() + self.assertIsInstance(c, Child) + self.assertIsInstance(c, Parent) + self.assertIsInstance(Negotiation(), Child) + self.assertNotIsInstance(Other(), Child) From 8b3280bf319c68ba1eb957ee9048718070267c8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:51:17 -0400 Subject: [PATCH 744/916] Simplify more. --- src/allmydata/protocol_switch.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 5143cab6a..158df32b5 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -63,9 +63,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # The tub that created us: tub: Tub - # This is an instance attribute; it will be set in connectionMade(). - _timeout: IDelayedCall - @classmethod def add_storage_server( cls, storage_server: StorageServer, swissnum: bytes @@ -117,7 +114,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) - self._buffer: bytes = b"" def __setattr__(self, name, value): if name in {"_foolscap", "_buffer", "transport", "__class__", "_timeout"}: @@ -139,12 +135,15 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # After creation, a Negotiation instance either has initClient() or # initServer() called. Since this is a client, we're never going to do # HTTP, so we can immediately become a Negotiation instance. - assert not self._buffer + assert not hasattr(self, "_buffer") self._convert_to_negotiation() return self.initClient(*args, **kwargs) def connectionMade(self): - self._timeout = reactor.callLater(30, self.transport.abortConnection) + self._buffer: bytes = b"" + self._timeout: IDelayedCall = reactor.callLater( + 30, self.transport.abortConnection + ) def dataReceived(self, data: bytes) -> None: """Handle incoming data. From 709f139c85e00f452ca5dacb308fd3494eb1be4a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 15:51:30 -0400 Subject: [PATCH 745/916] Start refactoring to enable HTTP storage client. --- src/allmydata/storage_client.py | 183 ++++++++++++++++++++++++++------ 1 file changed, 151 insertions(+), 32 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c63bfccff..a058ae828 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -30,6 +30,8 @@ Ported to Python 3. # # 6: implement other sorts of IStorageClient classes: S3, etc +from __future__ import annotations + from six import ensure_text from typing import Union import re, time, hashlib @@ -523,6 +525,45 @@ class IFoolscapStorageServer(Interface): """ +def _parse_announcement(server_id: bytes, furl: bytes, ann: dict) -> tuple[str, bytes, bytes, bytes, bytes]: + """ + Parse the furl and announcement, return: + + (nickname, permutation_seed, tubid, short_description, long_description) + """ + m = re.match(br'pb://(\w+)@', furl) + assert m, furl + tubid_s = m.group(1).lower() + tubid = base32.a2b(tubid_s) + if "permutation-seed-base32" in ann: + seed = ann["permutation-seed-base32"] + if isinstance(seed, str): + seed = seed.encode("utf-8") + ps = base32.a2b(seed) + elif re.search(br'^v0-[0-9a-zA-Z]{52}$', server_id): + ps = base32.a2b(server_id[3:]) + else: + log.msg("unable to parse serverid '%(server_id)s as pubkey, " + "hashing it to get permutation-seed, " + "may not converge with other clients", + server_id=server_id, + facility="tahoe.storage_broker", + level=log.UNUSUAL, umid="qu86tw") + ps = hashlib.sha256(server_id).digest() + permutation_seed = ps + + assert server_id + long_description = server_id + if server_id.startswith(b"v0-"): + # remove v0- prefix from abbreviated name + short_description = server_id[3:3+8] + else: + short_description = server_id[:8] + nickname = ann.get("nickname", "") + + return (nickname, permutation_seed, tubid, short_description, long_description) + + @implementer(IFoolscapStorageServer) @attr.s(frozen=True) class _FoolscapStorage(object): @@ -566,43 +607,13 @@ class _FoolscapStorage(object): The furl will be a Unicode string on Python 3; on Python 2 it will be either a native (bytes) string or a Unicode string. """ - furl = furl.encode("utf-8") - m = re.match(br'pb://(\w+)@', furl) - assert m, furl - tubid_s = m.group(1).lower() - tubid = base32.a2b(tubid_s) - if "permutation-seed-base32" in ann: - seed = ann["permutation-seed-base32"] - if isinstance(seed, str): - seed = seed.encode("utf-8") - ps = base32.a2b(seed) - elif re.search(br'^v0-[0-9a-zA-Z]{52}$', server_id): - ps = base32.a2b(server_id[3:]) - else: - log.msg("unable to parse serverid '%(server_id)s as pubkey, " - "hashing it to get permutation-seed, " - "may not converge with other clients", - server_id=server_id, - facility="tahoe.storage_broker", - level=log.UNUSUAL, umid="qu86tw") - ps = hashlib.sha256(server_id).digest() - permutation_seed = ps - - assert server_id - long_description = server_id - if server_id.startswith(b"v0-"): - # remove v0- prefix from abbreviated name - short_description = server_id[3:3+8] - else: - short_description = server_id[:8] - nickname = ann.get("nickname", "") - + (nickname, permutation_seed, tubid, short_description, long_description) = _parse_announcement(server_id, furl.encode("utf-8"), ann) return cls( nickname=nickname, permutation_seed=permutation_seed, tubid=tubid, storage_server=storage_server, - furl=furl, + furl=furl.encode("utf-8"), short_description=short_description, long_description=long_description, ) @@ -910,6 +921,114 @@ class NativeStorageServer(service.MultiService): # used when the broker wants us to hurry up self._reconnector.reset() + +@implementer(IServer) +class HTTPNativeStorageServer(service.MultiService): + """ + Like ``NativeStorageServer``, but for HTTP clients. + + The notion of being "connected" is less meaningful for HTTP; we just poll + occasionally, and if we've succeeded at last poll, we assume we're + "connected". + """ + + def __init__(self, server_id: bytes, announcement): + service.MultiService.__init__(self) + assert isinstance(server_id, bytes) + self._server_id = server_id + self.announcement = announcement + self._on_status_changed = ObserverList() + furl = announcement["anonymous-storage-FURL"].encode("utf-8") + self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + + def get_permutation_seed(self): + return self._permutation_seed + + def get_name(self): # keep methodname short + return self._name + + def get_longname(self): + return self._longname + + def get_tubid(self): + return self._tubid + + def get_lease_seed(self): + return self._lease_seed + + def get_foolscap_write_enabler_seed(self): + return self._tubid + + def get_nickname(self): + return self._nickname + + def on_status_changed(self, status_changed): + """ + :param status_changed: a callable taking a single arg (the + NativeStorageServer) that is notified when we become connected + """ + return self._on_status_changed.subscribe(status_changed) + + # Special methods used by copy.copy() and copy.deepcopy(). When those are + # used in allmydata.immutable.filenode to copy CheckResults during + # repair, we want it to treat the IServer instances as singletons, and + # not attempt to duplicate them.. + def __copy__(self): + return self + + def __deepcopy__(self, memodict): + return self + + def __repr__(self): + return "" % self.get_name() + + def get_serverid(self): + return self._server_id + + def get_version(self): + pass + + def get_announcement(self): + return self.announcement + + def get_connection_status(self): + pass + + def is_connected(self): + pass + + def get_available_space(self): + # TODO refactor into shared utility with NativeStorageServer + version = self.get_version() + if version is None: + return None + protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) + available_space = protocol_v1_version.get(b'available-space') + if available_space is None: + available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) + return available_space + + def start_connecting(self, trigger_cb): + pass + + def get_rref(self): + # TODO UH + pass + + def get_storage_server(self): + """ + See ``IServer.get_storage_server``. + """ + + def stop_connecting(self): + # used when this descriptor has been superceded by another + pass + + def try_to_connect(self): + # used when the broker wants us to hurry up + pass + + class UnknownServerTypeError(Exception): pass From c3e41588130e9d2c9d65965a9ea06d8f3503bd52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 15:55:14 -0400 Subject: [PATCH 746/916] Remove duplication. --- src/allmydata/storage_client.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a058ae828..e64f63413 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -695,6 +695,16 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): raise AnnouncementNotMatched() +def _available_space_from_version(version): + if version is None: + return None + protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) + available_space = protocol_v1_version.get(b'available-space') + if available_space is None: + available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) + return available_space + + @implementer(IServer) class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. @@ -853,13 +863,7 @@ class NativeStorageServer(service.MultiService): def get_available_space(self): version = self.get_version() - if version is None: - return None - protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) - available_space = protocol_v1_version.get(b'available-space') - if available_space is None: - available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) - return available_space + return _available_space_from_version(version) def start_connecting(self, trigger_cb): self._tub = self._tub_maker(self._handler_overrides) @@ -998,15 +1002,8 @@ class HTTPNativeStorageServer(service.MultiService): pass def get_available_space(self): - # TODO refactor into shared utility with NativeStorageServer version = self.get_version() - if version is None: - return None - protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) - available_space = protocol_v1_version.get(b'available-space') - if available_space is None: - available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) - return available_space + return _available_space_from_version(version) def start_connecting(self, trigger_cb): pass From c3b159a3fd98e63bcc0641c4c2a5dffe5e795a15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:12:57 -0400 Subject: [PATCH 747/916] Continue simplified sketch of HTTPNativeStorageServer. --- src/allmydata/storage_client.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e64f63413..3bcd8e6db 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -45,7 +45,7 @@ from zope.interface import ( implementer, ) from twisted.web import http -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.application import service from twisted.plugin import ( getPlugins, @@ -934,6 +934,9 @@ class HTTPNativeStorageServer(service.MultiService): The notion of being "connected" is less meaningful for HTTP; we just poll occasionally, and if we've succeeded at last poll, we assume we're "connected". + + TODO as first pass, just to get the proof-of-concept going, we will just + assume we're always connected after an initial successful HTTP request. """ def __init__(self, server_id: bytes, announcement): @@ -944,6 +947,13 @@ class HTTPNativeStorageServer(service.MultiService): self._on_status_changed = ObserverList() furl = announcement["anonymous-storage-FURL"].encode("utf-8") self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + self._istorage_server = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl( + announcement["anonymous-storage-NURLs"][0], reactor + ) + ) + self._connection_status = connection_status.ConnectionStatus.unstarted() + self._version = None def get_permutation_seed(self): return self._permutation_seed @@ -984,29 +994,33 @@ class HTTPNativeStorageServer(service.MultiService): return self def __repr__(self): - return "" % self.get_name() + return "" % self.get_name() def get_serverid(self): return self._server_id def get_version(self): - pass + return self._version def get_announcement(self): return self.announcement def get_connection_status(self): - pass + return self._connection_status def is_connected(self): - pass + return self._connection_status.connected def get_available_space(self): version = self.get_version() return _available_space_from_version(version) def start_connecting(self, trigger_cb): - pass + self._istorage_server.get_version().addCallback(self._got_version) + + def _got_version(self, version): + self._version = version + self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) def get_rref(self): # TODO UH @@ -1016,13 +1030,15 @@ class HTTPNativeStorageServer(service.MultiService): """ See ``IServer.get_storage_server``. """ + if self.is_connected(): + return self._istorage_server + else: + return None def stop_connecting(self): - # used when this descriptor has been superceded by another pass def try_to_connect(self): - # used when the broker wants us to hurry up pass From 94be227aaaf7adfefc3457dbc7556b10c7b5f3c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:15:21 -0400 Subject: [PATCH 748/916] Hopefully don't actually need that. --- src/allmydata/storage_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3bcd8e6db..62cc047f2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1022,10 +1022,6 @@ class HTTPNativeStorageServer(service.MultiService): self._version = version self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) - def get_rref(self): - # TODO UH - pass - def get_storage_server(self): """ See ``IServer.get_storage_server``. From 9ad4e844e86302682dfd38e82b7e262231c21ad9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:16:17 -0400 Subject: [PATCH 749/916] Do status change notification. --- src/allmydata/storage_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 62cc047f2..254179559 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -937,6 +937,7 @@ class HTTPNativeStorageServer(service.MultiService): TODO as first pass, just to get the proof-of-concept going, we will just assume we're always connected after an initial successful HTTP request. + Might do polling as follow-up ticket, in which case add link to that here. """ def __init__(self, server_id: bytes, announcement): @@ -1021,6 +1022,7 @@ class HTTPNativeStorageServer(service.MultiService): def _got_version(self, version): self._version = version self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) + self._on_status_changed.notify(self) def get_storage_server(self): """ From f671fb04a18c5a3f20437eac3424db1f97fc5df4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:24:33 -0400 Subject: [PATCH 750/916] A lot closer to working end-to-end. --- src/allmydata/client.py | 6 +++--- src/allmydata/storage_client.py | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 9938ec076..769554b3d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -825,9 +825,9 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.storage_nurls.update( - self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) - ) + nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + self.storage_nurls.update(nurls) + announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 254179559..ec03393a1 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -39,6 +39,7 @@ from os import urandom from configparser import NoSectionError import attr +from hyperlink import DecodedURL from zope.interface import ( Attribute, Interface, @@ -264,6 +265,12 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) + # TODO use constant + if "anonymous-storage-NURLs" in server["ann"]: + print("HTTTTTTTPPPPPPPPPPPPPPPPPPPP") + s = HTTPNativeStorageServer(server_id, server["ann"]) + s.on_status_changed(lambda _: self._got_connection()) + return s handler_overrides = server.get("connections", {}) s = NativeStorageServer( server_id, @@ -950,7 +957,7 @@ class HTTPNativeStorageServer(service.MultiService): self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl( - announcement["anonymous-storage-NURLs"][0], reactor + DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]), reactor ) ) self._connection_status = connection_status.ConnectionStatus.unstarted() From 09d778c2cfb4f888831835903daf6d77205ff5c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:13:09 -0400 Subject: [PATCH 751/916] Allow nodes to disable the HTTPS storage protocol. --- src/allmydata/client.py | 7 ++++--- src/allmydata/node.py | 5 ++++- src/allmydata/storage_client.py | 4 ++-- src/allmydata/test/common_system.py | 20 +++++++++++++------- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 769554b3d..d9fc20e92 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -825,9 +825,10 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) - self.storage_nurls.update(nurls) - announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] + if hasattr(self.tub.negotiationClass, "add_storage_server"): + nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + self.storage_nurls.update(nurls) + announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 597221e9b..0ad68f2b7 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -64,6 +64,7 @@ def _common_valid_config(): "tcp", ), "node": ( + "force_foolscap", "log_gatherer.furl", "nickname", "reveal-ip-address", @@ -709,7 +710,6 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) - support_foolscap_and_https(tub) for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() @@ -907,6 +907,9 @@ def create_main_tub(config, tub_options, handler_overrides=handler_overrides, certFile=certfile, ) + if not config.get_config("node", "force_foolscap", False): + support_foolscap_and_https(tub) + if portlocation is None: log.msg("Tub is not listening") else: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ec03393a1..3c2c7a1b8 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -102,8 +102,8 @@ class StorageClientConfig(object): :ivar preferred_peers: An iterable of the server-ids (``bytes``) of the storage servers where share placement is preferred, in order of - decreasing preference. See the *[client]peers.preferred* - documentation for details. + decreasing preference. See the *[client]peers.preferred* documentation + for details. :ivar dict[unicode, dict[unicode, unicode]] storage_plugins: A mapping from names of ``IFoolscapStoragePlugin`` configured in *tahoe.cfg* to the diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 9851d2b91..75379bbf3 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -698,7 +698,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): return f.read().strip() @inlineCallbacks - def set_up_nodes(self, NUMCLIENTS=5): + def set_up_nodes(self, NUMCLIENTS=5, force_foolscap=False): """ Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All of the nodes are running in this process. @@ -711,6 +711,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): :param int NUMCLIENTS: The number of client nodes to create. + :param bool force_foolscap: Force clients to use Foolscap instead of e.g. + HTTPS when available. + :return: A ``Deferred`` that fires when the nodes have connected to each other. """ @@ -719,16 +722,16 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.introducer = yield self._create_introducer() self.add_service(self.introducer) self.introweb_url = self._get_introducer_web() - yield self._set_up_client_nodes() + yield self._set_up_client_nodes(force_foolscap) @inlineCallbacks - def _set_up_client_nodes(self): + def _set_up_client_nodes(self, force_foolscap): q = self.introducer self.introducer_furl = q.introducer_url self.clients = [] basedirs = [] for i in range(self.numclients): - basedirs.append((yield self._set_up_client_node(i))) + basedirs.append((yield self._set_up_client_node(i, force_foolscap))) # start clients[0], wait for it's tub to be ready (at which point it # will have registered the helper furl). @@ -761,7 +764,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # and the helper-using webport self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() - def _generate_config(self, which, basedir): + def _generate_config(self, which, basedir, force_foolscap=False): config = {} allclients = set(range(self.numclients)) @@ -787,6 +790,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): if which in feature_matrix.get((section, feature), {which}): config.setdefault(section, {})[feature] = value + if force_foolscap: + config.setdefault("node", {})["force_foolscap"] = force_foolscap + setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") @@ -811,14 +817,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): return _render_config(config) - def _set_up_client_node(self, which): + def _set_up_client_node(self, which, force_foolscap): basedir = self.getdir("client%d" % (which,)) fileutil.make_dirs(os.path.join(basedir, "private")) if len(SYSTEM_TEST_CERTS) > (which + 1): f = open(os.path.join(basedir, "private", "node.pem"), "w") f.write(SYSTEM_TEST_CERTS[which + 1]) f.close() - config = self._generate_config(which, basedir) + config = self._generate_config(which, basedir, force_foolscap) fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) return basedir From e8609ac2df01038a7c51952149aaf4566b24e271 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:24:41 -0400 Subject: [PATCH 752/916] test_istorageserver passes with both Foolscap and HTTP again. --- src/allmydata/storage_client.py | 14 +++++----- src/allmydata/test/test_istorageserver.py | 33 ++++++++++++----------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3c2c7a1b8..e2a48e521 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -265,9 +265,8 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) - # TODO use constant - if "anonymous-storage-NURLs" in server["ann"]: - print("HTTTTTTTPPPPPPPPPPPPPPPPPPPP") + # TODO use constant for anonymous-storage-NURLs + if len(server["ann"].get("anonymous-storage-NURLs", [])) > 0: s = HTTPNativeStorageServer(server_id, server["ann"]) s.on_status_changed(lambda _: self._got_connection()) return s @@ -955,10 +954,13 @@ class HTTPNativeStorageServer(service.MultiService): self._on_status_changed = ObserverList() furl = announcement["anonymous-storage-FURL"].encode("utf-8") self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + nurl = DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]) + # Tests don't want persistent HTTPS pool, since that leaves a dirty + # reactor. As a reasonable hack, disabling persistent connnections for + # localhost allows us to have passing tests while not reducing + # performance for real-world usage. self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl( - DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]), reactor - ) + StorageClient.from_nurl(nurl, reactor, nurl.host not in ("localhost", "127.0.0.1")) ) self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 3328ea598..81025d779 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -17,7 +17,6 @@ from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock -from twisted.internet import reactor from foolscap.api import Referenceable, RemoteException # A better name for this would be IStorageClient... @@ -26,8 +25,10 @@ from allmydata.interfaces import IStorageServer from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage.http_client import StorageClient -from allmydata.storage_client import _HTTPStorageServer +from allmydata.storage_client import ( + NativeStorageServer, + HTTPNativeStorageServer, +) # Use random generator with known seed, so results are reproducible if tests @@ -1021,6 +1022,10 @@ class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" SKIP_TESTS = set() # type: Set[str] + FORCE_FOOLSCAP = False + + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) def _get_istorage_server(self): raise NotImplementedError("implement in subclass") @@ -1036,7 +1041,7 @@ class _SharedMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) - yield self.set_up_nodes(1) + yield self.set_up_nodes(1, self.FORCE_FOOLSCAP) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1065,11 +1070,12 @@ class _SharedMixin(SystemTestMixin): class _FoolscapMixin(_SharedMixin): """Run tests on Foolscap version of ``IStorageServer``.""" - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) + FORCE_FOOLSCAP = True def _get_istorage_server(self): - client = self._get_native_server().get_storage_server() + native_server = self._get_native_server() + assert isinstance(native_server, NativeStorageServer) + client = native_server.get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) return succeed(client) @@ -1077,16 +1083,13 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" + FORCE_FOOLSCAP = False + def _get_istorage_server(self): - nurl = list(self.clients[0].storage_nurls)[0] - - # Create HTTP client with non-persistent connections, so we don't leak - # state across tests: - client: IStorageServer = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, persistent=False) - ) + native_server = self._get_native_server() + assert isinstance(native_server, HTTPNativeStorageServer) + client = native_server.get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) - return succeed(client) From 636b8a9e2de2504b347c5662a9b828636c120743 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:28:03 -0400 Subject: [PATCH 753/916] Fix a bytes-vs-str bug. --- newsfragments/3913.minor | 0 src/allmydata/test/test_storage_web.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 newsfragments/3913.minor diff --git a/newsfragments/3913.minor b/newsfragments/3913.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 5984b2892..b47c93849 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -161,7 +161,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn(b"Total buckets: 0 (the number of", s) - self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) + self.failUnless(b"Next crawl in 59 minutes" in s or b"Next crawl in 60 minutes" in s, s) d.addCallback(_check2) return d From 3fbc4d7eea3398075a6986416a585f993549cc65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:45:37 -0400 Subject: [PATCH 754/916] Let's make this a little clearer --- src/allmydata/scripts/tahoe_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 51be32ee3..dfdc97ea5 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -179,7 +179,7 @@ class DaemonizeTheRealService(Service, HookMixin): ) ) else: - self.stderr.write("\nUnknown error\n") + self.stderr.write("\nUnknown error, here's the traceback:\n") reason.printTraceback(self.stderr) reactor.stop() From 42e818f0a702738ceae40d033fa69d43a68e5657 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:47:08 -0400 Subject: [PATCH 755/916] Refer to appropriate attributes, hopefully. --- src/allmydata/storage_client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e2a48e521..87041ff8b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -968,17 +968,18 @@ class HTTPNativeStorageServer(service.MultiService): def get_permutation_seed(self): return self._permutation_seed - def get_name(self): # keep methodname short - return self._name + def get_name(self): + return self._short_description def get_longname(self): - return self._longname + return self._long_description def get_tubid(self): return self._tubid def get_lease_seed(self): - return self._lease_seed + # Apparently this is what Foolscap version above does?! + return self._tubid def get_foolscap_write_enabler_seed(self): return self._tubid From 71b7e9b643930aa2504b1e38a131dd6def208a85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 15 Aug 2022 10:08:50 -0400 Subject: [PATCH 756/916] Support comma-separated multi-location hints. --- src/allmydata/protocol_switch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 158df32b5..89570436c 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -14,6 +14,8 @@ the configuration process. from __future__ import annotations +from itertools import chain + from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall from twisted.internet.ssl import CertificateOptions @@ -94,7 +96,11 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): ) storage_nurls = set() - for location_hint in cls.tub.locationHints: + # Individual hints can be in the form + # "tcp:host:port,tcp:host:port,tcp:host:port". + for location_hint in chain.from_iterable( + hints.split(",") for hints in cls.tub.locationHints + ): if location_hint.startswith("tcp:"): _, hostname, port = location_hint.split(":") port = int(port) From c1bcfab7f80d9a1f3e7b5f2e8c39dd292daedcd9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 15 Aug 2022 11:38:02 -0400 Subject: [PATCH 757/916] Repeatedly poll status of server. --- src/allmydata/storage_client.py | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 87041ff8b..f9a6feb7d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -46,6 +46,7 @@ from zope.interface import ( implementer, ) from twisted.web import http +from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor from twisted.application import service from twisted.plugin import ( @@ -940,10 +941,6 @@ class HTTPNativeStorageServer(service.MultiService): The notion of being "connected" is less meaningful for HTTP; we just poll occasionally, and if we've succeeded at last poll, we assume we're "connected". - - TODO as first pass, just to get the proof-of-concept going, we will just - assume we're always connected after an initial successful HTTP request. - Might do polling as follow-up ticket, in which case add link to that here. """ def __init__(self, server_id: bytes, announcement): @@ -962,8 +959,10 @@ class HTTPNativeStorageServer(service.MultiService): self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor, nurl.host not in ("localhost", "127.0.0.1")) ) + self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None + self._last_connect_time = None def get_permutation_seed(self): return self._permutation_seed @@ -1027,11 +1026,21 @@ class HTTPNativeStorageServer(service.MultiService): return _available_space_from_version(version) def start_connecting(self, trigger_cb): - self._istorage_server.get_version().addCallback(self._got_version) + self._lc = LoopingCall(self._connect) + self._lc.start(1, True) def _got_version(self, version): + self._last_connect_time = time.time() self._version = version - self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) + self._connection_status = connection_status.ConnectionStatus( + True, "connected", [], self._last_connect_time, self._last_connect_time + ) + self._on_status_changed.notify(self) + + def _failed_to_connect(self, reason): + self._connection_status = connection_status.ConnectionStatus( + False, f"failure: {reason}", [], self._last_connect_time, self._last_connect_time + ) self._on_status_changed.notify(self) def get_storage_server(self): @@ -1044,10 +1053,21 @@ class HTTPNativeStorageServer(service.MultiService): return None def stop_connecting(self): - pass + self._lc.stop() def try_to_connect(self): - pass + self._connect() + + def _connect(self): + return self._istorage_server.get_version().addCallbacks( + self._got_version, + self._failed_to_connect + ) + + def stopService(self): + service.MultiService.stopService(self) + self._lc.stop() + self._failed_to_connect("shut down") class UnknownServerTypeError(Exception): From 2e5662aa91cacf6ad36d1ea619ea08ea799591c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Aug 2022 13:11:06 -0400 Subject: [PATCH 758/916] Temporarily enforce requirement that allocated size matches actual size of an immutable. --- src/allmydata/storage/immutable.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index f7f5aebce..6fcca3871 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -419,14 +419,19 @@ class BucketWriter(object): self._already_written.set(True, offset, end) self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") + return self._is_finished() - # Return whether the whole thing has been written. See - # https://github.com/mlenzen/collections-extended/issues/169 and - # https://github.com/mlenzen/collections-extended/issues/172 for why - # it's done this way. + def _is_finished(self): + """ + Return whether the whole thing has been written. + """ return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): + # TODO this can't actually be enabled, because it's not backwards + # compatible. But it's useful for testing, so leaving it on until the + # branch is ready for merge. + assert self._is_finished() precondition(not self.closed) self._timeout.cancel() start = self._clock.seconds() From 556606271dbde1ccfe47ba29cb3767985286a7b4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Aug 2022 13:11:45 -0400 Subject: [PATCH 759/916] News file. --- newsfragments/3915.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3915.minor diff --git a/newsfragments/3915.minor b/newsfragments/3915.minor new file mode 100644 index 000000000..e69de29bb From d50c98a1e95b912c1cbd04d4bf932117176f5ac0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Aug 2022 14:34:40 -0400 Subject: [PATCH 760/916] Calculate URI extension size upfront, instead of hand-waving with a larger value. --- src/allmydata/immutable/encode.py | 18 ++++++++++++++++++ src/allmydata/immutable/layout.py | 16 +++++++--------- src/allmydata/immutable/upload.py | 27 ++++++++++++--------------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 42fc18077..c7887b7ba 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -624,6 +624,7 @@ class Encoder(object): for k in ('crypttext_root_hash', 'crypttext_hash', ): assert k in self.uri_extension_data + self.uri_extension_data uri_extension = uri.pack_extension(self.uri_extension_data) ed = {} for k,v in self.uri_extension_data.items(): @@ -694,3 +695,20 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash + + def get_uri_extension_size(self): + """ + Calculate the size of the URI extension that gets written at the end of + immutables. + + This may be done earlier than actual encoding, so e.g. we might not + know the crypttext hashes, but that's fine for our purposes since we + only care about the length. + """ + params = self.uri_extension_data.copy() + assert params + params["crypttext_hash"] = b"\x00" * 32 + params["crypttext_root_hash"] = b"\x00" * 32 + params["share_root_hash"] = b"\x00" * 32 + uri_extension = uri.pack_extension(params) + return len(uri_extension) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 79c886237..74af09a2b 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -90,7 +90,7 @@ FORCE_V2 = False # set briefly by unit tests to make small-sized V2 shares def make_write_bucket_proxy(rref, server, data_size, block_size, num_segments, - num_share_hashes, uri_extension_size_max): + num_share_hashes, uri_extension_size): # Use layout v1 for small files, so they'll be readable by older versions # (= 2**32 or data_size >= 2**32: @@ -233,8 +232,7 @@ class WriteBucketProxy(object): def put_uri_extension(self, data): offset = self._offsets['uri_extension'] assert isinstance(data, bytes) - precondition(len(data) <= self._uri_extension_size_max, - len(data), self._uri_extension_size_max) + precondition(len(data) == self._uri_extension_size) length = struct.pack(self.fieldstruct, len(data)) return self._write(offset, length+data) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index cb332dfdf..6b9b48f6a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -242,31 +242,26 @@ class UploadResults(object): def get_verifycapstr(self): return self._verifycapstr -# our current uri_extension is 846 bytes for small files, a few bytes -# more for larger ones (since the filesize is encoded in decimal in a -# few places). Ask for a little bit more just in case we need it. If -# the extension changes size, we can change EXTENSION_SIZE to -# allocate a more accurate amount of space. -EXTENSION_SIZE = 1000 -# TODO: actual extensions are closer to 419 bytes, so we can probably lower -# this. def pretty_print_shnum_to_servers(s): return ', '.join([ "sh%s: %s" % (k, '+'.join([idlib.shortnodeid_b2a(x) for x in v])) for k, v in s.items() ]) + class ServerTracker(object): def __init__(self, server, sharesize, blocksize, num_segments, num_share_hashes, storage_index, - bucket_renewal_secret, bucket_cancel_secret): + bucket_renewal_secret, bucket_cancel_secret, + uri_extension_size): self._server = server self.buckets = {} # k: shareid, v: IRemoteBucketWriter self.sharesize = sharesize + self.uri_extension_size = uri_extension_size wbp = layout.make_write_bucket_proxy(None, None, sharesize, blocksize, num_segments, num_share_hashes, - EXTENSION_SIZE) + uri_extension_size) self.wbp_class = wbp.__class__ # to create more of them self.allocated_size = wbp.get_allocated_size() self.blocksize = blocksize @@ -314,7 +309,7 @@ class ServerTracker(object): self.blocksize, self.num_segments, self.num_share_hashes, - EXTENSION_SIZE) + self.uri_extension_size) b[sharenum] = bp self.buckets.update(b) return (alreadygot, set(b.keys())) @@ -487,7 +482,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): def get_shareholders(self, storage_broker, secret_holder, storage_index, share_size, block_size, num_segments, total_shares, needed_shares, - min_happiness): + min_happiness, uri_extension_size): """ @return: (upload_trackers, already_serverids), where upload_trackers is a set of ServerTracker instances that have agreed to hold @@ -529,7 +524,8 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): # figure out how much space to ask for wbp = layout.make_write_bucket_proxy(None, None, share_size, 0, num_segments, - num_share_hashes, EXTENSION_SIZE) + num_share_hashes, + uri_extension_size) allocated_size = wbp.get_allocated_size() # decide upon the renewal/cancel secrets, to include them in the @@ -554,7 +550,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): def _create_server_tracker(server, renew, cancel): return ServerTracker( server, share_size, block_size, num_segments, num_share_hashes, - storage_index, renew, cancel, + storage_index, renew, cancel, uri_extension_size ) readonly_trackers, write_trackers = self._create_trackers( @@ -1326,7 +1322,8 @@ class CHKUploader(object): d = server_selector.get_shareholders(storage_broker, secret_holder, storage_index, share_size, block_size, - num_segments, n, k, desired) + num_segments, n, k, desired, + encoder.get_uri_extension_size()) def _done(res): self._server_selection_elapsed = time.time() - server_selection_started return res From 7aa97336a0fe7b8bda7de38e2c639b142e50f494 Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Wed, 17 Aug 2022 16:03:03 +0100 Subject: [PATCH 761/916] Refactor FakeWebTest & MemoryConsumerTest classes There are base test classes namely `SyncTestCase` and `AsyncTestCase` which we would like all test classes in this code base to extend. This commit refactors two test classes to use the `SyncTestCase` with the newer assert methods. Signed-off-by: Fon E. Noel NFEBE --- newsfragments/3916.minor | 0 src/allmydata/test/test_consumer.py | 24 +++++++++++++++--------- src/allmydata/test/test_testing.py | 7 ++++--- 3 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3916.minor diff --git a/newsfragments/3916.minor b/newsfragments/3916.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index a689de462..234fc2594 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -14,11 +14,17 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer -from twisted.trial.unittest import TestCase from twisted.internet.interfaces import IPushProducer, IPullProducer from allmydata.util.consumer import MemoryConsumer +from .common import ( + SyncTestCase, +) +from testtools.matchers import ( + Equals, +) + @implementer(IPushProducer) @implementer(IPullProducer) @@ -50,7 +56,7 @@ class Producer(object): self.consumer.unregisterProducer() -class MemoryConsumerTests(TestCase): +class MemoryConsumerTests(SyncTestCase): """Tests for MemoryConsumer.""" def test_push_producer(self): @@ -60,14 +66,14 @@ class MemoryConsumerTests(TestCase): consumer = MemoryConsumer() producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, True) - self.assertEqual(consumer.chunks, [b"abc"]) + self.assertThat(consumer.chunks, Equals([b"abc"])) producer.iterate() producer.iterate() - self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) - self.assertEqual(consumer.done, False) + self.assertThat(consumer.chunks, Equals([b"abc", b"def", b"ghi"])) + self.assertFalse(consumer.done) producer.iterate() - self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) - self.assertEqual(consumer.done, True) + self.assertThat(consumer.chunks, Equals([b"abc", b"def", b"ghi"])) + self.assertTrue(consumer.done) def test_pull_producer(self): """ @@ -76,8 +82,8 @@ class MemoryConsumerTests(TestCase): consumer = MemoryConsumer() producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, False) - self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) - self.assertEqual(consumer.done, True) + self.assertThat(consumer.chunks, Equals([b"abc", b"def", b"ghi"])) + self.assertTrue(consumer.done) # download_to_data() is effectively tested by some of the filenode tests, e.g. diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 527b235bd..3715d1aca 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -46,9 +46,10 @@ from hypothesis.strategies import ( binary, ) -from testtools import ( - TestCase, +from .common import ( + SyncTestCase, ) + from testtools.matchers import ( Always, Equals, @@ -61,7 +62,7 @@ from testtools.twistedsupport import ( ) -class FakeWebTest(TestCase): +class FakeWebTest(SyncTestCase): """ Test the WebUI verified-fakes infrastucture """ From c9084a2a45fb16cc90d4f6043017cbc57ba463a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 12:49:06 -0400 Subject: [PATCH 762/916] Disable assertion we can't, sadly, enable. --- src/allmydata/storage/immutable.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 6fcca3871..a02fd3bb2 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -428,10 +428,9 @@ class BucketWriter(object): return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): - # TODO this can't actually be enabled, because it's not backwards - # compatible. But it's useful for testing, so leaving it on until the - # branch is ready for merge. - assert self._is_finished() + # This can't actually be enabled, because it's not backwards compatible + # with old Foolscap clients. + # assert self._is_finished() precondition(not self.closed) self._timeout.cancel() start = self._clock.seconds() From 9d03c476d196c78d7ca5ac36e57c3f1b6c1434b0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 12:49:45 -0400 Subject: [PATCH 763/916] Make sure we write all the bytes we say we're sending. --- src/allmydata/immutable/layout.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 74af09a2b..30ab985a8 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -118,6 +118,7 @@ class WriteBucketProxy(object): self._data_size = data_size self._block_size = block_size self._num_segments = num_segments + self._written_bytes = 0 effective_segments = mathutil.next_power_of_k(num_segments,2) self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE @@ -194,6 +195,11 @@ class WriteBucketProxy(object): return self._write(offset, data) def put_crypttext_hashes(self, hashes): + # plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and + # so is not explicitly written, but we need to write everything, so + # fill it in with nulls. + self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + offset = self._offsets['crypttext_hash_tree'] assert isinstance(hashes, list) data = b"".join(hashes) @@ -242,11 +248,12 @@ class WriteBucketProxy(object): # would reduce the foolscap CPU overhead per share, but wouldn't # reduce the number of round trips, so it might not be worth the # effort. - + self._written_bytes += len(data) return self._pipeline.add(len(data), self._rref.callRemote, "write", offset, data) def close(self): + assert self._written_bytes == self.get_allocated_size(), f"{self._written_bytes} != {self.get_allocated_size()}" d = self._pipeline.add(0, self._rref.callRemote, "close") d.addCallback(lambda ign: self._pipeline.flush()) return d From 3464637bbb1de4a739d87a14d95b0a300c326063 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 12:54:26 -0400 Subject: [PATCH 764/916] Fix unit tests. --- src/allmydata/test/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index c3f2a35e1..134609f81 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -463,7 +463,7 @@ class BucketProxy(unittest.TestCase): block_size=10, num_segments=5, num_share_hashes=3, - uri_extension_size_max=500) + uri_extension_size=500) self.failUnless(interfaces.IStorageBucketWriter.providedBy(bp), bp) def _do_test_readwrite(self, name, header_size, wbp_class, rbp_class): @@ -494,7 +494,7 @@ class BucketProxy(unittest.TestCase): block_size=25, num_segments=4, num_share_hashes=3, - uri_extension_size_max=len(uri_extension)) + uri_extension_size=len(uri_extension)) d = bp.put_header() d.addCallback(lambda res: bp.put_block(0, b"a"*25)) From cd81e5a01c82796fd3d69c93fb7f088ad6bf2a3b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 13:13:22 -0400 Subject: [PATCH 765/916] Hint for future debugging. --- src/allmydata/storage/immutable.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index a02fd3bb2..0893513ae 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -397,7 +397,9 @@ class BucketWriter(object): """ Write data at given offset, return whether the upload is complete. """ - # Delay the timeout, since we received data: + # Delay the timeout, since we received data; if we get an + # AlreadyCancelled error, that means there's a bug in the client and + # write() was called after close(). self._timeout.reset(30 * 60) start = self._clock.seconds() precondition(not self.closed) From 92662d802cf0e43a5a5f684faba455de7a6cde53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 13:15:13 -0400 Subject: [PATCH 766/916] Don't drop a Deferred on the ground. --- src/allmydata/immutable/layout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 30ab985a8..de390bda9 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -198,8 +198,11 @@ class WriteBucketProxy(object): # plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and # so is not explicitly written, but we need to write everything, so # fill it in with nulls. - self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + d = self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + d.addCallback(lambda _: self._really_put_crypttext_hashes(hashes)) + return d + def _really_put_crypttext_hashes(self, hashes): offset = self._offsets['crypttext_hash_tree'] assert isinstance(hashes, list) data = b"".join(hashes) From bdb4aac0de1bd03fb8625e156135d4f0964e478c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 13:15:27 -0400 Subject: [PATCH 767/916] Pass in the missing argument. --- src/allmydata/test/test_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 8d5435e88..18192de6c 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -983,7 +983,7 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, num_segments = encoder.get_param("num_segments") d = selector.get_shareholders(broker, sh, storage_index, share_size, block_size, num_segments, - 10, 3, 4) + 10, 3, 4, encoder.get_uri_extension_size()) def _have_shareholders(upload_trackers_and_already_servers): (upload_trackers, already_servers) = upload_trackers_and_already_servers assert servers_to_break <= len(upload_trackers) From 869b15803c506256004fd47d6c72d5a8f61e0267 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 6 Sep 2022 08:46:09 -0400 Subject: [PATCH 768/916] assorted fixes --- docs/proposed/http-storage-node-protocol.rst | 52 +++++++++++--------- docs/specifications/url.rst | 2 + 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3dac376ff..8fe855be3 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -30,12 +30,12 @@ Glossary introducer a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers - fURL + `fURL `_ a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol (the storage service is an example of such an object) - NURL - a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap + `NURL `_ + a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap swissnum a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service @@ -580,24 +580,6 @@ Responses: the response is ``CONFLICT``. At this point the only thing to do is abort the upload and start from scratch (see below). -``PUT /v1/immutable/:storage_index/:share_number/abort`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -This cancels an *in-progress* upload. - -The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - - X-Tahoe-Authorization: upload-secret - -The response code: - -* When the upload is still in progress and therefore the abort has succeeded, - the response is ``OK``. - Future uploads can start from scratch with no pre-existing upload state stored on the server. -* If the uploaded has already finished, the response is 405 (Method Not Allowed) - and no change is made. - - Discussion `````````` @@ -616,6 +598,24 @@ From RFC 7231:: PATCH method defined in [RFC5789]). +``PUT /v1/immutable/:storage_index/:share_number/abort`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +This cancels an *in-progress* upload. + +The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: + + X-Tahoe-Authorization: upload-secret + +The response code: + +* When the upload is still in progress and therefore the abort has succeeded, + the response is ``OK``. + Future uploads can start from scratch with no pre-existing upload state stored on the server. +* If the uploaded has already finished, the response is 405 (Method Not Allowed) + and no change is made. + + ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -625,7 +625,7 @@ corruption. It also includes potentially important details about the share. For example:: - {"reason": u"expected hash abcd, got hash efgh"} + {"reason": "expected hash abcd, got hash efgh"} .. share-type, storage-index, and share-number are inferred from the URL @@ -799,6 +799,7 @@ Immutable Data 200 OK + { "required": [ {"begin": 16, "end": 48 } ] } PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum @@ -807,6 +808,7 @@ Immutable Data 200 OK + { "required": [ {"begin": 32, "end": 48 } ] } PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum @@ -823,6 +825,7 @@ Immutable Data Range: bytes=0-47 200 OK + Content-Range: bytes 0-47/48 #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: @@ -906,9 +909,12 @@ otherwise it will read a byte which won't match `b""`:: #. Download the contents of share number ``3``:: - GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3 Authorization: Tahoe-LAFS nurl-swissnum + Range: bytes=0-16 + 200 OK + Content-Range: bytes 0-15/16 #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 31fb05fad..421ac57f7 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -10,6 +10,8 @@ The intended audience for this document is Tahoe-LAFS maintainers and other deve Background ---------- +.. _fURLs: + Tahoe-LAFS first used Foolscap_ for network communication. Foolscap connection setup takes as an input a Foolscap URL or a *fURL*. A fURL includes three components: From 3b9eea5b8f1278bad158df336a9f617f8dee7895 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 6 Sep 2022 08:46:52 -0400 Subject: [PATCH 769/916] news fragment --- newsfragments/3922.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3922.documentation diff --git a/newsfragments/3922.documentation b/newsfragments/3922.documentation new file mode 100644 index 000000000..d0232dd02 --- /dev/null +++ b/newsfragments/3922.documentation @@ -0,0 +1 @@ +Several minor errors in the Great Black Swamp proposed specification document have been fixed. \ No newline at end of file From 9975fddd88d263a58e146b91b2c22b5b53500a85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 8 Sep 2022 13:42:19 -0400 Subject: [PATCH 770/916] Get rid of garbage. --- src/allmydata/immutable/encode.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index c7887b7ba..34a9c2472 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -624,7 +624,6 @@ class Encoder(object): for k in ('crypttext_root_hash', 'crypttext_hash', ): assert k in self.uri_extension_data - self.uri_extension_data uri_extension = uri.pack_extension(self.uri_extension_data) ed = {} for k,v in self.uri_extension_data.items(): From c82bb5f21c90e293c1507c71aa68fb4768b3abb6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 8 Sep 2022 13:44:22 -0400 Subject: [PATCH 771/916] Use a more meaningful constant. --- src/allmydata/immutable/encode.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 34a9c2472..3c4440486 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -706,8 +706,8 @@ class Encoder(object): """ params = self.uri_extension_data.copy() assert params - params["crypttext_hash"] = b"\x00" * 32 - params["crypttext_root_hash"] = b"\x00" * 32 - params["share_root_hash"] = b"\x00" * 32 + params["crypttext_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE + params["crypttext_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE + params["share_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE uri_extension = uri.pack_extension(params) return len(uri_extension) From 6310774b8267d2deb0620c1766bc7f93694a3803 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 8 Sep 2022 17:50:58 +0000 Subject: [PATCH 772/916] Add documentation on OpenMetrics statistics endpoint. references ticket:3786 --- docs/stats.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/stats.rst b/docs/stats.rst index 50642d816..c7d69e0d2 100644 --- a/docs/stats.rst +++ b/docs/stats.rst @@ -264,3 +264,18 @@ the "tahoe-conf" file for notes about configuration and installing these plugins into a Munin environment. .. _Munin: http://munin-monitoring.org/ + + +Scraping Stats Values in OpenMetrics Format +=========================================== + +Time Series DataBase (TSDB) software like Prometheus_ and VictoriaMetrics_ can +parse statistics from the e.g. http://localhost:3456/statistics?t=openmetrics +URL in OpenMetrics_ format. Software like Grafana_ can then be used to graph +and alert on these numbers. You can find a pre-configured dashboard for +Grafana at https://grafana.com/grafana/dashboards/16894-tahoe-lafs/. + +.. _OpenMetrics: https://openmetrics.io/ +.. _Prometheus: https://prometheus.io/ +.. _VictoriaMetrics: https://victoriametrics.com/ +.. _Grafana: https://grafana.com/ From ae21ab74a2afb8a0db234e76e2d09e76df0f5958 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 8 Sep 2022 18:05:59 +0000 Subject: [PATCH 773/916] Add newsfragment for the added documentation. --- newsfragments/3786.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3786.minor diff --git a/newsfragments/3786.minor b/newsfragments/3786.minor new file mode 100644 index 000000000..ecd1a2c4e --- /dev/null +++ b/newsfragments/3786.minor @@ -0,0 +1 @@ +Added re-structured text documentation for the OpenMetrics format statistics endpoint. From 373a5328293693612123e7e47be4c01e5de3746b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 09:36:56 -0400 Subject: [PATCH 774/916] Detect corrupted UEB length more consistently. --- src/allmydata/immutable/layout.py | 8 ++++---- src/allmydata/test/test_repairer.py | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index de390bda9..6679fc94c 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -495,10 +495,10 @@ class ReadBucketProxy(object): if len(data) != self._fieldsize: raise LayoutInvalid("not enough bytes to encode URI length -- should be %d bytes long, not %d " % (self._fieldsize, len(data),)) length = struct.unpack(self._fieldstruct, data)[0] - if length >= 2**31: - # URI extension blocks are around 419 bytes long, so this - # must be corrupted. Anyway, the foolscap interface schema - # for "read" will not allow >= 2**31 bytes length. + if length >= 2000: + # URI extension blocks are around 419 bytes long; in previous + # versions of the code 1000 was used as a default catchall. So + # 2000 or more must be corrupted. raise RidiculouslyLargeURIExtensionBlock(length) return self._read(offset+self._fieldsize, length) diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index f9b93af72..8545b1cf4 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -251,6 +251,12 @@ class Verifier(GridTestMixin, unittest.TestCase, RepairTestMixin): self.judge_invisible_corruption) def test_corrupt_ueb(self): + # Note that in some rare situations this might fail, specifically if + # the length of the UEB is corrupted to be a value that is bigger than + # the size but less than 2000, it might not get caught... But that's + # mostly because in that case it doesn't meaningfully corrupt it. See + # _get_uri_extension_the_old_way() in layout.py for where the 2000 + # number comes from. self.basedir = "repairer/Verifier/corrupt_ueb" return self._help_test_verify(common._corrupt_uri_extension, self.judge_invisible_corruption) From 8d5f08771a4f73f09611500c39627334f7273fc9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 09:45:46 -0400 Subject: [PATCH 775/916] Minimal check on parameters' contents. --- src/allmydata/immutable/encode.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 3c4440486..874492785 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -705,9 +705,13 @@ class Encoder(object): only care about the length. """ params = self.uri_extension_data.copy() - assert params params["crypttext_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE params["crypttext_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE params["share_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE + assert params.keys() == { + "codec_name", "codec_params", "size", "segment_size", "num_segments", + "needed_shares", "total_shares", "tail_codec_params", + "crypttext_hash", "crypttext_root_hash", "share_root_hash" + }, params.keys() uri_extension = uri.pack_extension(params) return len(uri_extension) From 00972ba3c6d8b9b83eb8c069ec1c9fa5768aaed3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 09:59:36 -0400 Subject: [PATCH 776/916] Match latest GBS spec. --- docs/specifications/url.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 31fb05fad..a9e37a0ec 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -103,11 +103,8 @@ Version 1 The hash component of a version 1 NURL differs in three ways from the prior version. -1. The hash function used is SHA3-224 instead of SHA1. - The security of SHA1 `continues to be eroded`_. - Contrariwise SHA3 is currently the most recent addition to the SHA family by NIST. - The 224 bit instance is chosen to keep the output short and because it offers greater collision resistance than SHA1 was thought to offer even at its inception - (prior to security research showing actual collision resistance is lower). +1. The hash function used is SHA-256, to match RFC 7469. + The security of SHA1 `continues to be eroded`_; Latacora `SHA-2`_. 2. The hash is computed over the certificate's SPKI instead of the whole certificate. This allows certificate re-generation so long as the public key remains the same. This is useful to allow contact information to be updated or extension of validity period. @@ -140,7 +137,8 @@ Examples * ``pb://azEu8vlRpnEeYm0DySQDeNY3Z2iJXHC_bsbaAw@localhost:47877/64i4aokv4ej#v=1`` .. _`continues to be eroded`: https://en.wikipedia.org/wiki/SHA-1#Cryptanalysis_and_validation -.. _`explored by the web community`: https://www.imperialviolet.org/2011/05/04/pinning.html +.. _`SHA-2`: https://latacora.micro.blog/2018/04/03/cryptographic-right-answers.html +.. _`explored by the web community`: https://www.rfc-editor.org/rfc/rfc7469 .. _Foolscap: https://github.com/warner/foolscap .. [1] ``foolscap.furl.decode_furl`` is taken as the canonical definition of the syntax of a fURL. From 1759eacee3c4cbdb72d956bf1df2c35f7fc435bb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:09:25 -0400 Subject: [PATCH 777/916] No need to include NURL. --- docs/proposed/http-storage-node-protocol.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3dac376ff..b601a785b 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -409,8 +409,7 @@ For example:: "tolerates-immutable-read-overrun": true, "delete-mutable-shares-with-zero-length-writev": true, "fills-holes-with-zero-bytes": true, - "prevents-read-past-end-of-share-data": true, - "gbs-anonymous-storage-url": "pb://...#v=1" + "prevents-read-past-end-of-share-data": true }, "application-version": "1.13.0" } From 0d97847ef5c4625d972dd92e29f9a8187f97a6b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:09:50 -0400 Subject: [PATCH 778/916] News file. --- newsfragments/3904.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3904.minor diff --git a/newsfragments/3904.minor b/newsfragments/3904.minor new file mode 100644 index 000000000..e69de29bb From b1aa93e02234bae93efac860a0078d5a1c089d2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:34:59 -0400 Subject: [PATCH 779/916] Switch prefix. --- docs/proposed/http-storage-node-protocol.rst | 48 ++++++++++---------- src/allmydata/storage/http_client.py | 28 ++++++++---- src/allmydata/storage/http_server.py | 27 ++++++----- src/allmydata/test/test_storage_http.py | 8 ++-- 4 files changed, 61 insertions(+), 50 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index b601a785b..ec800367c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -395,7 +395,7 @@ Encoding General ~~~~~~~ -``GET /v1/version`` +``GET /storage/v1/version`` !!!!!!!!!!!!!!!!!!! Retrieve information about the version of the storage server. @@ -414,7 +414,7 @@ For example:: "application-version": "1.13.0" } -``PUT /v1/lease/:storage_index`` +``PUT /storage/v1/lease/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Either renew or create a new lease on the bucket addressed by ``storage_index``. @@ -467,7 +467,7 @@ Immutable Writing ~~~~~~~ -``POST /v1/immutable/:storage_index`` +``POST /storage/v1/immutable/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Initialize an immutable storage index with some buckets. @@ -503,7 +503,7 @@ Handling repeat calls: Discussion `````````` -We considered making this ``POST /v1/immutable`` instead. +We considered making this ``POST /storage/v1/immutable`` instead. The motivation was to keep *storage index* out of the request URL. Request URLs have an elevated chance of being logged by something. We were concerned that having the *storage index* logged may increase some risks. @@ -538,7 +538,7 @@ Rejected designs for upload secrets: it must contain randomness. Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better. -``PATCH /v1/immutable/:storage_index/:share_number`` +``PATCH /storage/v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. @@ -579,7 +579,7 @@ Responses: the response is ``CONFLICT``. At this point the only thing to do is abort the upload and start from scratch (see below). -``PUT /v1/immutable/:storage_index/:share_number/abort`` +``PUT /storage/v1/immutable/:storage_index/:share_number/abort`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! This cancels an *in-progress* upload. @@ -615,7 +615,7 @@ From RFC 7231:: PATCH method defined in [RFC5789]). -``POST /v1/immutable/:storage_index/:share_number/corrupt`` +``POST /storage/v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. The @@ -634,7 +634,7 @@ couldn't be found. Reading ~~~~~~~ -``GET /v1/immutable/:storage_index/shares`` +``GET /storage/v1/immutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a list (semantically, a set) indicating all shares available for the @@ -644,7 +644,7 @@ indicated storage index. For example:: An unknown storage index results in an empty list. -``GET /v1/immutable/:storage_index/:share_number`` +``GET /storage/v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. @@ -685,7 +685,7 @@ Mutable Writing ~~~~~~~ -``POST /v1/mutable/:storage_index/read-test-write`` +``POST /storage/v1/mutable/:storage_index/read-test-write`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! General purpose read-test-and-write operation for mutable storage indexes. @@ -741,7 +741,7 @@ As a result, if there is no data at all, an empty bytestring is returned no matt Reading ~~~~~~~ -``GET /v1/mutable/:storage_index/shares`` +``GET /storage/v1/mutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. @@ -749,10 +749,10 @@ For example (this is shown as list, since it will be list for JSON, but will be [1, 5] -``GET /v1/mutable/:storage_index/:share_number`` +``GET /storage/v1/mutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Read data from the indicated mutable shares, just like ``GET /v1/immutable/:storage_index`` +Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index`` The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. @@ -764,7 +764,7 @@ The resulting ``Content-Range`` header will be consistent with the returned data If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. -``POST /v1/mutable/:storage_index/:share_number/corrupt`` +``POST /storage/v1/mutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. @@ -778,7 +778,7 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: - POST /v1/immutable/AAAAAAAAAAAAAAAA + POST /storage/v1/immutable/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-renew-secret efgh X-Tahoe-Authorization: lease-cancel-secret jjkl @@ -791,7 +791,7 @@ Immutable Data #. Upload the content for immutable share ``7``:: - PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 0-15/48 X-Tahoe-Authorization: upload-secret xyzf @@ -799,7 +799,7 @@ Immutable Data 200 OK - PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 16-31/48 X-Tahoe-Authorization: upload-secret xyzf @@ -807,7 +807,7 @@ Immutable Data 200 OK - PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 32-47/48 X-Tahoe-Authorization: upload-secret xyzf @@ -817,7 +817,7 @@ Immutable Data #. Download the content of the previously uploaded immutable share ``7``:: - GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7 + GET /storage/v1/immutable/AAAAAAAAAAAAAAAA?share=7 Authorization: Tahoe-LAFS nurl-swissnum Range: bytes=0-47 @@ -826,7 +826,7 @@ Immutable Data #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: - PUT /v1/lease/AAAAAAAAAAAAAAAA + PUT /storage/v1/lease/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-cancel-secret jjkl X-Tahoe-Authorization: lease-renew-secret efgh @@ -841,7 +841,7 @@ The special test vector of size 1 but empty bytes will only pass if there is no existing share, otherwise it will read a byte which won't match `b""`:: - POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: write-enabler abcd X-Tahoe-Authorization: lease-cancel-secret efgh @@ -873,7 +873,7 @@ otherwise it will read a byte which won't match `b""`:: #. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: - POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: write-enabler abcd X-Tahoe-Authorization: lease-cancel-secret efgh @@ -905,14 +905,14 @@ otherwise it will read a byte which won't match `b""`:: #. Download the contents of share number ``3``:: - GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 Authorization: Tahoe-LAFS nurl-swissnum #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: - PUT /v1/lease/BBBBBBBBBBBBBBBB + PUT /storage/v1/lease/BBBBBBBBBBBBBBBB Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-cancel-secret efgh X-Tahoe-Authorization: lease-renew-secret ijkl diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a2dc5379f..16d426dda 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -392,7 +392,7 @@ class StorageClientGeneral(object): """ Return the version metadata for the server. """ - url = self._client.relative_url("/v1/version") + url = self._client.relative_url("/storage/v1/version") response = yield self._client.request("GET", url) decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) @@ -408,7 +408,7 @@ class StorageClientGeneral(object): Otherwise a new lease is added. """ url = self._client.relative_url( - "/v1/lease/{}".format(_encode_si(storage_index)) + "/storage/v1/lease/{}".format(_encode_si(storage_index)) ) response = yield self._client.request( "PUT", @@ -457,7 +457,9 @@ def read_share_chunk( always provided by the current callers. """ url = client.relative_url( - "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number) + "/storage/v1/{}/{}/{}".format( + share_type, _encode_si(storage_index), share_number + ) ) response = yield client.request( "GET", @@ -518,7 +520,7 @@ async def advise_corrupt_share( ): assert isinstance(reason, str) url = client.relative_url( - "/v1/{}/{}/{}/corrupt".format( + "/storage/v1/{}/{}/{}/corrupt".format( share_type, _encode_si(storage_index), share_number ) ) @@ -563,7 +565,9 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index)) + url = self._client.relative_url( + "/storage/v1/immutable/" + _encode_si(storage_index) + ) message = {"share-numbers": share_numbers, "allocated-size": allocated_size} response = yield self._client.request( @@ -588,7 +592,9 @@ class StorageClientImmutables(object): ) -> Deferred[None]: """Abort the upload.""" url = self._client.relative_url( - "/v1/immutable/{}/{}/abort".format(_encode_si(storage_index), share_number) + "/storage/v1/immutable/{}/{}/abort".format( + _encode_si(storage_index), share_number + ) ) response = yield self._client.request( "PUT", @@ -620,7 +626,9 @@ class StorageClientImmutables(object): been uploaded. """ url = self._client.relative_url( - "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) + "/storage/v1/immutable/{}/{}".format( + _encode_si(storage_index), share_number + ) ) response = yield self._client.request( "PATCH", @@ -668,7 +676,7 @@ class StorageClientImmutables(object): Return the set of shares for a given storage index. """ url = self._client.relative_url( - "/v1/immutable/{}/shares".format(_encode_si(storage_index)) + "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) ) response = yield self._client.request( "GET", @@ -774,7 +782,7 @@ class StorageClientMutables: are done and if they are valid the writes are done. """ url = self._client.relative_url( - "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) + "/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) ) message = { "test-write-vectors": { @@ -817,7 +825,7 @@ class StorageClientMutables: List the share numbers for a given storage index. """ url = self._client.relative_url( - "/v1/mutable/{}/shares".format(_encode_si(storage_index)) + "/storage/v1/mutable/{}/shares".format(_encode_si(storage_index)) ) response = await self._client.request("GET", url) if response.code == http.OK: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 68d0740b1..2e9b57b13 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -545,7 +545,7 @@ class HTTPServer(object): ##### Generic APIs ##### - @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) + @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) def version(self, request, authorization): """Return version information.""" return self._send_encoded(request, self._storage_server.get_version()) @@ -555,7 +555,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, - "/v1/immutable/", + "/storage/v1/immutable/", methods=["POST"], ) def allocate_buckets(self, request, authorization, storage_index): @@ -591,7 +591,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.UPLOAD}, - "/v1/immutable///abort", + "/storage/v1/immutable///abort", methods=["PUT"], ) def abort_share_upload(self, request, authorization, storage_index, share_number): @@ -622,7 +622,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.UPLOAD}, - "/v1/immutable//", + "/storage/v1/immutable//", methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): @@ -665,7 +665,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable//shares", + "/storage/v1/immutable//shares", methods=["GET"], ) def list_shares(self, request, authorization, storage_index): @@ -678,7 +678,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable//", + "/storage/v1/immutable//", methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): @@ -694,7 +694,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL}, - "/v1/lease/", + "/storage/v1/lease/", methods=["PUT"], ) def add_or_renew_lease(self, request, authorization, storage_index): @@ -715,7 +715,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable///corrupt", + "/storage/v1/immutable///corrupt", methods=["POST"], ) def advise_corrupt_share_immutable( @@ -736,7 +736,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.WRITE_ENABLER}, - "/v1/mutable//read-test-write", + "/storage/v1/mutable//read-test-write", methods=["POST"], ) def mutable_read_test_write(self, request, authorization, storage_index): @@ -771,7 +771,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/mutable//", + "/storage/v1/mutable//", methods=["GET"], ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): @@ -795,7 +795,10 @@ class HTTPServer(object): return read_range(request, read_data, share_length) @_authorized_route( - _app, set(), "/v1/mutable//shares", methods=["GET"] + _app, + set(), + "/storage/v1/mutable//shares", + methods=["GET"], ) def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" @@ -805,7 +808,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/mutable///corrupt", + "/storage/v1/mutable///corrupt", methods=["POST"], ) def advise_corrupt_share_mutable( diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 419052282..4a912cf6c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -255,7 +255,7 @@ class TestApp(object): else: return "BAD: {}".format(authorization) - @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) + @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) def bad_version(self, request, authorization): """Return version result that violates the expected schema.""" request.setHeader("content-type", CBOR_MIME_TYPE) @@ -534,7 +534,7 @@ class GenericHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = urandom(16) url = self.http.client.relative_url( - "/v1/immutable/" + _encode_si(storage_index) + "/storage/v1/immutable/" + _encode_si(storage_index) ) message = {"bad-message": "missing expected keys"} @@ -1418,7 +1418,7 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) ), ) ) @@ -1441,7 +1441,7 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) ), headers=headers, ) From f5b374a7a2ad95232e8cddca3d9d334f4f4b6986 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:56:11 -0400 Subject: [PATCH 780/916] Make sphinx happy. --- docs/proposed/http-storage-node-protocol.rst | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ec800367c..a44408e6c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -396,7 +396,7 @@ General ~~~~~~~ ``GET /storage/v1/version`` -!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve information about the version of the storage server. Information is returned as an encoded mapping. @@ -415,7 +415,7 @@ For example:: } ``PUT /storage/v1/lease/:storage_index`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Either renew or create a new lease on the bucket addressed by ``storage_index``. @@ -468,7 +468,7 @@ Writing ~~~~~~~ ``POST /storage/v1/immutable/:storage_index`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Initialize an immutable storage index with some buckets. The buckets may have share data written to them once. @@ -539,7 +539,7 @@ Rejected designs for upload secrets: Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better. ``PATCH /storage/v1/immutable/:storage_index/:share_number`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. The share number must belong to the storage index. @@ -580,7 +580,7 @@ Responses: At this point the only thing to do is abort the upload and start from scratch (see below). ``PUT /storage/v1/immutable/:storage_index/:share_number/abort`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! This cancels an *in-progress* upload. @@ -616,7 +616,7 @@ From RFC 7231:: ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. The request body includes an human-meaningful text string with details about the @@ -635,7 +635,7 @@ Reading ~~~~~~~ ``GET /storage/v1/immutable/:storage_index/shares`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index. For example:: @@ -645,7 +645,7 @@ indicated storage index. For example:: An unknown storage index results in an empty list. ``GET /storage/v1/immutable/:storage_index/:share_number`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). @@ -686,7 +686,7 @@ Writing ~~~~~~~ ``POST /storage/v1/mutable/:storage_index/read-test-write`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! General purpose read-test-and-write operation for mutable storage indexes. A mutable storage index is also called a "slot" @@ -742,7 +742,7 @@ Reading ~~~~~~~ ``GET /storage/v1/mutable/:storage_index/shares`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. For example (this is shown as list, since it will be list for JSON, but will be set for CBOR):: @@ -765,7 +765,7 @@ If the response to a query is an empty range, the ``NO CONTENT`` (204) response ``POST /storage/v1/mutable/:storage_index/:share_number/corrupt`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. Just like the immutable version. From 4a573ede3461510d6f2aa09f78d2791dea8393b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 11:29:32 -0400 Subject: [PATCH 781/916] Download the actual data we need, instead of relying on bad reading-beyond-the-end semantics. --- src/allmydata/immutable/layout.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 6679fc94c..07b6b8b3b 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -17,8 +17,10 @@ from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE from allmydata.util import mathutil, observer, pipeline, log from allmydata.util.assertutil import precondition +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage.server import si_b2a + class LayoutInvalid(Exception): """ There is something wrong with these bytes so they can't be interpreted as the kind of immutable file that I know how to download.""" @@ -311,8 +313,6 @@ class WriteBucketProxy_v2(WriteBucketProxy): @implementer(IStorageBucketReader) class ReadBucketProxy(object): - MAX_UEB_SIZE = 2000 # actual size is closer to 419, but varies by a few bytes - def __init__(self, rref, server, storage_index): self._rref = rref self._server = server @@ -389,10 +389,15 @@ class ReadBucketProxy(object): self._offsets[field] = offset return self._offsets - def _fetch_sharehashtree_and_ueb(self, offsets): + @async_to_deferred + async def _fetch_sharehashtree_and_ueb(self, offsets): + [ueb_length] = struct.unpack( + await self._read(offsets['share_hashes'], self._fieldsize), + self._fieldstruct + ) sharehashtree_size = offsets['uri_extension'] - offsets['share_hashes'] return self._read(offsets['share_hashes'], - self.MAX_UEB_SIZE+sharehashtree_size) + ueb_length + self._fieldsize +sharehashtree_size) def _parse_sharehashtree_and_ueb(self, data): sharehashtree_size = self._offsets['uri_extension'] - self._offsets['share_hashes'] From 444bc724c54a07ef4e0dddb53706e1e1d16091b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Sep 2022 10:38:29 -0400 Subject: [PATCH 782/916] A better approach to MAX_UEB_SIZE: just delete the code since it's not used in practice. --- src/allmydata/immutable/layout.py | 59 ++++++------------------------- 1 file changed, 10 insertions(+), 49 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 07b6b8b3b..d552d43c4 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -17,7 +17,6 @@ from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE from allmydata.util import mathutil, observer, pipeline, log from allmydata.util.assertutil import precondition -from allmydata.util.deferredutil import async_to_deferred from allmydata.storage.server import si_b2a @@ -340,11 +339,6 @@ class ReadBucketProxy(object): # TODO: for small shares, read the whole bucket in _start() d = self._fetch_header() d.addCallback(self._parse_offsets) - # XXX The following two callbacks implement a slightly faster/nicer - # way to get the ueb and sharehashtree, but it requires that the - # storage server be >= v1.3.0. - # d.addCallback(self._fetch_sharehashtree_and_ueb) - # d.addCallback(self._parse_sharehashtree_and_ueb) def _fail_waiters(f): self._ready.fire(f) def _notify_waiters(result): @@ -389,34 +383,6 @@ class ReadBucketProxy(object): self._offsets[field] = offset return self._offsets - @async_to_deferred - async def _fetch_sharehashtree_and_ueb(self, offsets): - [ueb_length] = struct.unpack( - await self._read(offsets['share_hashes'], self._fieldsize), - self._fieldstruct - ) - sharehashtree_size = offsets['uri_extension'] - offsets['share_hashes'] - return self._read(offsets['share_hashes'], - ueb_length + self._fieldsize +sharehashtree_size) - - def _parse_sharehashtree_and_ueb(self, data): - sharehashtree_size = self._offsets['uri_extension'] - self._offsets['share_hashes'] - if len(data) < sharehashtree_size: - raise LayoutInvalid("share hash tree truncated -- should have at least %d bytes -- not %d" % (sharehashtree_size, len(data))) - if sharehashtree_size % (2+HASH_SIZE) != 0: - raise LayoutInvalid("share hash tree malformed -- should have an even multiple of %d bytes -- not %d" % (2+HASH_SIZE, sharehashtree_size)) - self._share_hashes = [] - for i in range(0, sharehashtree_size, 2+HASH_SIZE): - hashnum = struct.unpack(">H", data[i:i+2])[0] - hashvalue = data[i+2:i+2+HASH_SIZE] - self._share_hashes.append( (hashnum, hashvalue) ) - - i = self._offsets['uri_extension']-self._offsets['share_hashes'] - if len(data) < i+self._fieldsize: - raise LayoutInvalid("not enough bytes to encode URI length -- should be at least %d bytes long, not %d " % (i+self._fieldsize, len(data),)) - length = struct.unpack(self._fieldstruct, data[i:i+self._fieldsize])[0] - self._ueb_data = data[i+self._fieldsize:i+self._fieldsize+length] - def _get_block_data(self, unused, blocknum, blocksize, thisblocksize): offset = self._offsets['data'] + blocknum * blocksize return self._read(offset, thisblocksize) @@ -459,20 +425,18 @@ class ReadBucketProxy(object): else: return defer.succeed([]) - def _get_share_hashes(self, unused=None): - if hasattr(self, '_share_hashes'): - return self._share_hashes - return self._get_share_hashes_the_old_way() - def get_share_hashes(self): d = self._start_if_needed() d.addCallback(self._get_share_hashes) return d - def _get_share_hashes_the_old_way(self): + def _get_share_hashes(self, _ignore): """ Tahoe storage servers < v1.3.0 would return an error if you tried to read past the end of the share, so we need to use the offset and - read just that much.""" + read just that much. + + HTTP-based storage protocol also doesn't like reading past the end. + """ offset = self._offsets['share_hashes'] size = self._offsets['uri_extension'] - offset if size % (2+HASH_SIZE) != 0: @@ -490,10 +454,13 @@ class ReadBucketProxy(object): d.addCallback(_unpack_share_hashes) return d - def _get_uri_extension_the_old_way(self, unused=None): + def _get_uri_extension(self, unused=None): """ Tahoe storage servers < v1.3.0 would return an error if you tried to read past the end of the share, so we need to fetch the UEB size - and then read just that much.""" + and then read just that much. + + HTTP-based storage protocol also doesn't like reading past the end. + """ offset = self._offsets['uri_extension'] d = self._read(offset, self._fieldsize) def _got_length(data): @@ -510,12 +477,6 @@ class ReadBucketProxy(object): d.addCallback(_got_length) return d - def _get_uri_extension(self, unused=None): - if hasattr(self, '_ueb_data'): - return self._ueb_data - else: - return self._get_uri_extension_the_old_way() - def get_uri_extension(self): d = self._start_if_needed() d.addCallback(self._get_uri_extension) From fb532a71ef4da28c91594e8d05695e267b747137 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Sep 2022 22:43:09 -0600 Subject: [PATCH 783/916] own pid-file checks --- setup.py | 3 ++ src/allmydata/scripts/tahoe_run.py | 36 ++++++++++----- src/allmydata/util/pid.py | 72 ++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 src/allmydata/util/pid.py diff --git a/setup.py b/setup.py index c3ee4eb90..bd16a61ce 100644 --- a/setup.py +++ b/setup.py @@ -138,6 +138,9 @@ install_requires = [ "treq", "cbor2", "pycddl", + + # for pid-file support + "psutil", ] setup_requires = [ diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 51be32ee3..21041f1ab 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -19,6 +19,7 @@ import os, sys from allmydata.scripts.common import BasedirOptions from twisted.scripts import twistd from twisted.python import usage +from twisted.python.filepath import FilePath from twisted.python.reflect import namedAny from twisted.internet.defer import maybeDeferred from twisted.application.service import Service @@ -27,6 +28,11 @@ from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin +from allmydata.util.pid import ( + check_pid_process, + cleanup_pidfile, + ProcessInTheWay, +) from allmydata.storage.crawler import ( MigratePickleFileError, ) @@ -35,28 +41,31 @@ from allmydata.node import ( PrivacyError, ) + def get_pidfile(basedir): """ Returns the path to the PID file. :param basedir: the node's base directory :returns: the path to the PID file """ - return os.path.join(basedir, u"twistd.pid") + return os.path.join(basedir, u"running.process") + def get_pid_from_pidfile(pidfile): """ Tries to read and return the PID stored in the node's PID file - (twistd.pid). + :param pidfile: try to read this PID file :returns: A numeric PID on success, ``None`` if PID file absent or inaccessible, ``-1`` if PID file invalid. """ try: with open(pidfile, "r") as f: - pid = f.read() + data = f.read().strip() except EnvironmentError: return None + pid, _ = data.split() try: pid = int(pid) except ValueError: @@ -64,6 +73,7 @@ def get_pid_from_pidfile(pidfile): return pid + def identify_node_type(basedir): """ :return unicode: None or one of: 'client' or 'introducer'. @@ -227,10 +237,8 @@ def run(config, runApp=twistd.runApp): print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - twistd_args = ["--nodaemon", "--rundir", basedir] - if sys.platform != "win32": - pidfile = get_pidfile(basedir) - twistd_args.extend(["--pidfile", pidfile]) + # we turn off Twisted's pid-file to use our own + twistd_args = ["--pidfile", None, "--nodaemon", "--rundir", basedir] twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -246,12 +254,16 @@ def run(config, runApp=twistd.runApp): return 1 twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} - # handle invalid PID file (twistd might not start otherwise) - if sys.platform != "win32" and get_pid_from_pidfile(pidfile) == -1: - print("found invalid PID file in %s - deleting it" % basedir, file=err) - os.remove(pidfile) + # before we try to run, check against our pidfile -- this will + # raise an exception if there appears to be a running process "in + # the way" + pidfile = FilePath(get_pidfile(config['basedir'])) + try: + check_pid_process(pidfile) + except ProcessInTheWay as e: + print("ERROR: {}".format(e)) + return 1 # We always pass --nodaemon so twistd.runApp does not daemonize. - print("running node in %s" % (quoted_basedir,), file=out) runApp(twistd_config) return 0 diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py new file mode 100644 index 000000000..21e30aa87 --- /dev/null +++ b/src/allmydata/util/pid.py @@ -0,0 +1,72 @@ +import os +import psutil + + +class ProcessInTheWay(Exception): + """ + our pidfile points at a running process + """ + + +def check_pid_process(pidfile, find_process=None): + """ + If another instance appears to be running already, raise an + exception. Otherwise, write our PID + start time to the pidfile + and arrange to delete it upon exit. + + :param FilePath pidfile: the file to read/write our PID from. + + :param Callable find_process: None, or a custom way to get a + Process objet (usually for tests) + + :raises ProcessInTheWay: if a running process exists at our PID + """ + find_process = psutil.Process if find_process is None else find_process + # check if we have another instance running already + if pidfile.exists(): + with pidfile.open("r") as f: + content = f.read().decode("utf8").strip() + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + try: + # if any other process is running at that PID, let the + # user decide if this is another magic-older + # instance. Automated programs may use the start-time to + # help decide this (if the PID is merely recycled, the + # start-time won't match). + proc = find_process(pid) + raise ProcessInTheWay( + "A process is already running as PID {}".format(pid) + ) + except psutil.NoSuchProcess: + print( + "'{pidpath}' refers to {pid} that isn't running".format( + pidpath=pidfile.path, + pid=pid, + ) + ) + # nothing is running at that PID so it must be a stale file + pidfile.remove() + + # write our PID + start-time to the pid-file + pid = os.getpid() + starttime = find_process(pid).create_time() + with pidfile.open("w") as f: + f.write("{} {}\n".format(pid, starttime).encode("utf8")) + + +def cleanup_pidfile(pidfile): + """ + Safely remove the given pidfile + """ + + try: + pidfile.remove() + except Exception as e: + print( + "Couldn't remove '{pidfile}': {err}.".format( + pidfile=pidfile.path, + err=e, + ) + ) From 3bfb60c6f426cadc25bb201e6e59165cedd2b490 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 19:57:01 -0600 Subject: [PATCH 784/916] back to context-manager, simplify --- src/allmydata/scripts/tahoe_run.py | 15 +++++++++------ src/allmydata/test/cli/test_run.py | 20 +++++++++++--------- src/allmydata/util/pid.py | 29 +++++++++++++++++++++-------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 21041f1ab..07f5bf72c 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -30,8 +30,8 @@ from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin from allmydata.util.pid import ( check_pid_process, - cleanup_pidfile, ProcessInTheWay, + InvalidPidFile, ) from allmydata.storage.crawler import ( MigratePickleFileError, @@ -237,8 +237,13 @@ def run(config, runApp=twistd.runApp): print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - # we turn off Twisted's pid-file to use our own - twistd_args = ["--pidfile", None, "--nodaemon", "--rundir", basedir] + twistd_args = [ + # turn off Twisted's pid-file to use our own + "--pidfile", None, + # ensure twistd machinery does not daemonize. + "--nodaemon", + "--rundir", basedir, + ] twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -254,9 +259,7 @@ def run(config, runApp=twistd.runApp): return 1 twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} - # before we try to run, check against our pidfile -- this will - # raise an exception if there appears to be a running process "in - # the way" + # our own pid-style file contains PID and process creation time pidfile = FilePath(get_pidfile(config['basedir'])) try: check_pid_process(pidfile) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 28613e8c1..db01eb440 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -159,7 +159,7 @@ class RunTests(SyncTestCase): """ basedir = FilePath(self.mktemp()).asTextMode() basedir.makedirs() - basedir.child(u"twistd.pid").setContent(b"foo") + basedir.child(u"running.process").setContent(b"foo") basedir.child(u"tahoe-client.tac").setContent(b"") config = RunOptions() @@ -168,17 +168,19 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] - runs = [] - result_code = run(config, runApp=runs.append) + class DummyRunner: + runs = [] + _exitSignal = None + + def run(self): + self.runs.append(True) + + result_code = run(config, runner=DummyRunner()) self.assertThat( config.stderr.getvalue(), Contains("found invalid PID file in"), ) self.assertThat( - runs, - HasLength(1), - ) - self.assertThat( - result_code, - Equals(0), + DummyRunner.runs, + Equals([]) ) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 21e30aa87..3b488a2c2 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,5 +1,8 @@ import os import psutil +from contextlib import ( + contextmanager, +) class ProcessInTheWay(Exception): @@ -8,6 +11,13 @@ class ProcessInTheWay(Exception): """ +class InvalidPidFile(Exception): + """ + our pidfile isn't well-formed + """ + + +@contextmanager def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -26,9 +36,16 @@ def check_pid_process(pidfile, find_process=None): if pidfile.exists(): with pidfile.open("r") as f: content = f.read().decode("utf8").strip() - pid, starttime = content.split() - pid = int(pid) - starttime = float(starttime) + try: + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + except ValueError: + raise InvalidPidFile( + "found invalid PID file in {}".format( + pidfile + ) + ) try: # if any other process is running at that PID, let the # user decide if this is another magic-older @@ -55,11 +72,7 @@ def check_pid_process(pidfile, find_process=None): with pidfile.open("w") as f: f.write("{} {}\n".format(pid, starttime).encode("utf8")) - -def cleanup_pidfile(pidfile): - """ - Safely remove the given pidfile - """ + yield # setup completed, await cleanup try: pidfile.remove() From cad162bb8fb2d961c74f457be6e4495b00f0aeed Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 19:59:18 -0600 Subject: [PATCH 785/916] should have pid-file on windows too, now --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index db01eb440..902e4011a 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -151,7 +151,7 @@ class RunTests(SyncTestCase): """ Tests for ``run``. """ - @skipIf(platform.isWindows(), "There are no PID files on Windows.") + def test_non_numeric_pid(self): """ If the pidfile exists but does not contain a numeric value, a complaint to From 0e0ebf6687280d0be5ae6a536a4f9d48958d03b7 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 20:06:32 -0600 Subject: [PATCH 786/916] more testing --- src/allmydata/test/cli/test_run.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 902e4011a..ecc81fe3f 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -20,6 +20,9 @@ from testtools import ( skipIf, ) +from hypothesis.strategies import text +from hypothesis import given + from testtools.matchers import ( Contains, Equals, @@ -44,6 +47,10 @@ from ...scripts.tahoe_run import ( RunOptions, run, ) +from ...util.pid import ( + check_pid_process, + InvalidPidFile, +) from ...scripts.runner import ( parse_options @@ -180,7 +187,18 @@ class RunTests(SyncTestCase): config.stderr.getvalue(), Contains("found invalid PID file in"), ) + # because the pidfile is invalid we shouldn't get to the + # .run() call itself. self.assertThat( DummyRunner.runs, Equals([]) ) + + @given(text()) + def test_pidfile_contents(self, content): + pidfile = FilePath("pidfile") + pidfile.setContent(content.encode("utf8")) + + with self.assertRaises(InvalidPidFile): + with check_pid_process(pidfile): + pass From e6adfc7726cc3e081d18b712e573ef265e49c3ca Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 20:22:07 -0600 Subject: [PATCH 787/916] news --- newsfragments/3926.incompat | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 newsfragments/3926.incompat diff --git a/newsfragments/3926.incompat b/newsfragments/3926.incompat new file mode 100644 index 000000000..3f58b4ba8 --- /dev/null +++ b/newsfragments/3926.incompat @@ -0,0 +1,10 @@ +Record both the PID and the process creation-time + +A new kind of pidfile in `running.process` records both +the PID and the creation-time of the process. This facilitates +automatic discovery of a "stale" pidfile that points to a +currently-running process. If the recorded creation-time matches +the creation-time of the running process, then it is a still-running +`tahoe run` proecss. Otherwise, the file is stale. + +The `twistd.pid` file is no longer present. \ No newline at end of file From 6048d1d9a99e5f88cd423a9524bede823277709f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 21:13:30 -0600 Subject: [PATCH 788/916] in case hypothesis finds the magic --- src/allmydata/test/cli/test_run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index ecc81fe3f..7bf87eea9 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -12,6 +12,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import re from six.moves import ( StringIO, ) @@ -21,7 +22,7 @@ from testtools import ( ) from hypothesis.strategies import text -from hypothesis import given +from hypothesis import given, assume from testtools.matchers import ( Contains, @@ -194,8 +195,11 @@ class RunTests(SyncTestCase): Equals([]) ) + good_file_content_re = re.compile(r"\w[0-9]*\w[0-9]*\w") + @given(text()) def test_pidfile_contents(self, content): + assume(not self.good_file_content_re.match(content)) pidfile = FilePath("pidfile") pidfile.setContent(content.encode("utf8")) From 642b604753dd9b9af2c740e04e65e58bbae00299 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 21:51:56 -0600 Subject: [PATCH 789/916] use stdin-closing for pidfile cleanup too --- src/allmydata/scripts/tahoe_run.py | 1 + src/allmydata/test/cli/test_run.py | 12 +++--------- src/allmydata/util/pid.py | 17 +++++++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 07f5bf72c..20d5c2bf1 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -30,6 +30,7 @@ from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin from allmydata.util.pid import ( check_pid_process, + cleanup_pidfile, ProcessInTheWay, InvalidPidFile, ) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 7bf87eea9..71085fddd 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -176,14 +176,8 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] - class DummyRunner: - runs = [] - _exitSignal = None - - def run(self): - self.runs.append(True) - - result_code = run(config, runner=DummyRunner()) + runs = [] + result_code = run(config, runApp=runs.append) self.assertThat( config.stderr.getvalue(), Contains("found invalid PID file in"), @@ -191,7 +185,7 @@ class RunTests(SyncTestCase): # because the pidfile is invalid we shouldn't get to the # .run() call itself. self.assertThat( - DummyRunner.runs, + runs, Equals([]) ) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 3b488a2c2..3ab955cb3 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,8 +1,5 @@ import os import psutil -from contextlib import ( - contextmanager, -) class ProcessInTheWay(Exception): @@ -17,7 +14,12 @@ class InvalidPidFile(Exception): """ -@contextmanager +class CannotRemovePidFile(Exception): + """ + something went wrong removing the pidfile + """ + + def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -72,12 +74,15 @@ def check_pid_process(pidfile, find_process=None): with pidfile.open("w") as f: f.write("{} {}\n".format(pid, starttime).encode("utf8")) - yield # setup completed, await cleanup +def cleanup_pidfile(pidfile): + """ + Safely clean up a PID-file + """ try: pidfile.remove() except Exception as e: - print( + raise CannotRemovePidFile( "Couldn't remove '{pidfile}': {err}.".format( pidfile=pidfile.path, err=e, From 82c72ddede1dbbe97365877186af27928a996c0b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 21:58:20 -0600 Subject: [PATCH 790/916] cleanup --- src/allmydata/test/cli/test_run.py | 14 ++------------ src/allmydata/util/pid.py | 4 ++-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 71085fddd..ae869e475 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -17,22 +17,14 @@ from six.moves import ( StringIO, ) -from testtools import ( - skipIf, -) - from hypothesis.strategies import text from hypothesis import given, assume from testtools.matchers import ( Contains, Equals, - HasLength, ) -from twisted.python.runtime import ( - platform, -) from twisted.python.filepath import ( FilePath, ) @@ -184,10 +176,8 @@ class RunTests(SyncTestCase): ) # because the pidfile is invalid we shouldn't get to the # .run() call itself. - self.assertThat( - runs, - Equals([]) - ) + self.assertThat(runs, Equals([])) + self.assertThat(result_code, Equals(1)) good_file_content_re = re.compile(r"\w[0-9]*\w[0-9]*\w") diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 3ab955cb3..ff8129bbc 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -50,11 +50,11 @@ def check_pid_process(pidfile, find_process=None): ) try: # if any other process is running at that PID, let the - # user decide if this is another magic-older + # user decide if this is another legitimate # instance. Automated programs may use the start-time to # help decide this (if the PID is merely recycled, the # start-time won't match). - proc = find_process(pid) + find_process(pid) raise ProcessInTheWay( "A process is already running as PID {}".format(pid) ) From 228bbbc2fe791b83af0d495df44882a63456b59f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 22:39:59 -0600 Subject: [PATCH 791/916] new pid-file --- src/allmydata/test/cli_node_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 410796be2..c324d5565 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -134,7 +134,7 @@ class CLINodeAPI(object): @property def twistd_pid_file(self): - return self.basedir.child(u"twistd.pid") + return self.basedir.child(u"running.process") @property def node_url_file(self): From 114d5e1ed8582fa130953227eced0528862ca381 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 23:08:46 -0600 Subject: [PATCH 792/916] pidfile on windows now --- src/allmydata/scripts/tahoe_run.py | 6 +++-- src/allmydata/test/test_runner.py | 36 ++++++++++++------------------ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 20d5c2bf1..72b8e3eca 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -239,12 +239,14 @@ def run(config, runApp=twistd.runApp): return 1 twistd_args = [ - # turn off Twisted's pid-file to use our own - "--pidfile", None, # ensure twistd machinery does not daemonize. "--nodaemon", "--rundir", basedir, ] + if sys.platform != "win32": + # turn off Twisted's pid-file to use our own -- but only on + # windows, because twistd doesn't know about pidfiles there + twistd_args.extend(["--pidfile", None]) twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 3eb6b8a34..9b6357f46 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -418,9 +418,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): tahoe.active() - # We don't keep track of PIDs in files on Windows. - if not platform.isWindows(): - self.assertTrue(tahoe.twistd_pid_file.exists()) + self.assertTrue(tahoe.twistd_pid_file.exists()) self.assertTrue(tahoe.node_url_file.exists()) # rm this so we can detect when the second incarnation is ready @@ -493,9 +491,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # change on restart storage_furl = fileutil.read(tahoe.storage_furl_file.path) - # We don't keep track of PIDs in files on Windows. - if not platform.isWindows(): - self.assertTrue(tahoe.twistd_pid_file.exists()) + self.assertTrue(tahoe.twistd_pid_file.exists()) # rm this so we can detect when the second incarnation is ready tahoe.node_url_file.remove() @@ -513,21 +509,18 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): fileutil.read(tahoe.storage_furl_file.path), ) - if not platform.isWindows(): - self.assertTrue( - tahoe.twistd_pid_file.exists(), - "PID file ({}) didn't exist when we expected it to. " - "These exist: {}".format( - tahoe.twistd_pid_file, - tahoe.twistd_pid_file.parent().listdir(), - ), - ) + self.assertTrue( + tahoe.twistd_pid_file.exists(), + "PID file ({}) didn't exist when we expected it to. " + "These exist: {}".format( + tahoe.twistd_pid_file, + tahoe.twistd_pid_file.parent().listdir(), + ), + ) yield tahoe.stop_and_wait() - if not platform.isWindows(): - # twistd.pid should be gone by now. - self.assertFalse(tahoe.twistd_pid_file.exists()) - + # twistd.pid should be gone by now. + self.assertFalse(tahoe.twistd_pid_file.exists()) def _remove(self, res, file): fileutil.remove(file) @@ -610,9 +603,8 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): ), ) - if not platform.isWindows(): - # It should not be running. - self.assertFalse(tahoe.twistd_pid_file.exists()) + # It should not be running. + self.assertFalse(tahoe.twistd_pid_file.exists()) # Wait for the operation to *complete*. If we got this far it's # because we got the expected message so we can expect the "tahoe ..." From aef2e96139fc0afc610736b181e218dce2aa9b79 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 17 Sep 2022 16:28:25 -0600 Subject: [PATCH 793/916] refactor: dispatch with our reactor, pass to tahoe_run --- src/allmydata/scripts/runner.py | 12 ++++-------- src/allmydata/scripts/tahoe_run.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index a0d8a752b..756c26f2c 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -47,11 +47,6 @@ if _default_nodedir: NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]" -# XXX all this 'dispatch' stuff needs to be unified + fixed up -_control_node_dispatch = { - "run": tahoe_run.run, -} - process_control_commands = [ ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), ] # type: SubCommands @@ -195,6 +190,7 @@ def parse_or_exit(config, argv, stdout, stderr): return config def dispatch(config, + reactor, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr): command = config.subCommand so = config.subOptions @@ -206,8 +202,8 @@ def dispatch(config, if command in create_dispatch: f = create_dispatch[command] - elif command in _control_node_dispatch: - f = _control_node_dispatch[command] + elif command == "run": + f = lambda config: tahoe_run.run(reactor, config) elif command in debug.dispatch: f = debug.dispatch[command] elif command in admin.dispatch: @@ -361,7 +357,7 @@ def _run_with_reactor(reactor, config, argv, stdout, stderr): stderr, ) d.addCallback(_maybe_enable_eliot_logging, reactor) - d.addCallback(dispatch, stdout=stdout, stderr=stderr) + d.addCallback(dispatch, reactor, stdout=stdout, stderr=stderr) def _show_exception(f): # when task.react() notices a non-SystemExit exception, it does # log.err() with the failure and then exits with rc=1. We want this diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 72b8e3eca..dd4561a4b 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -217,7 +217,7 @@ class DaemonizeTahoeNodePlugin(object): return DaemonizeTheRealService(self.nodetype, self.basedir, so) -def run(config, runApp=twistd.runApp): +def run(reactor, config, runApp=twistd.runApp): """ Runs a Tahoe-LAFS node in the foreground. From 8b2cb79070edabd20fb9bdbb41de51458788e50a Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 17 Sep 2022 16:29:03 -0600 Subject: [PATCH 794/916] cleanup via reactor --- src/allmydata/scripts/tahoe_run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index dd4561a4b..a5b833233 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -269,6 +269,11 @@ def run(reactor, config, runApp=twistd.runApp): except ProcessInTheWay as e: print("ERROR: {}".format(e)) return 1 + else: + reactor.addSystemEventTrigger( + "during", "shutdown", + lambda: cleanup_pidfile(pidfile) + ) # We always pass --nodaemon so twistd.runApp does not daemonize. runApp(twistd_config) From 254a994eb53035b70a653b47b2951d6159634a23 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 17 Sep 2022 16:41:17 -0600 Subject: [PATCH 795/916] flake8 --- src/allmydata/scripts/tahoe_run.py | 2 +- src/allmydata/test/test_runner.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index a5b833233..7722fef51 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -266,7 +266,7 @@ def run(reactor, config, runApp=twistd.runApp): pidfile = FilePath(get_pidfile(config['basedir'])) try: check_pid_process(pidfile) - except ProcessInTheWay as e: + except (ProcessInTheWay, InvalidPidFile) as e: print("ERROR: {}".format(e)) return 1 else: diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 9b6357f46..14d0dfb7f 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -47,9 +47,6 @@ from twisted.internet.defer import ( DeferredList, ) from twisted.python.filepath import FilePath -from twisted.python.runtime import ( - platform, -) from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv from allmydata.test import common_util From fe80126e3fcffe56c171188c7cd5847f19bf6f7b Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 18 Sep 2022 22:39:25 -0600 Subject: [PATCH 796/916] fixups --- src/allmydata/scripts/tahoe_run.py | 2 +- src/allmydata/test/cli/test_run.py | 4 +++- src/allmydata/test/common_util.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 7722fef51..6dfa726a3 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -267,7 +267,7 @@ def run(reactor, config, runApp=twistd.runApp): try: check_pid_process(pidfile) except (ProcessInTheWay, InvalidPidFile) as e: - print("ERROR: {}".format(e)) + print("ERROR: {}".format(e), file=err) return 1 else: reactor.addSystemEventTrigger( diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index ae869e475..6358b70dd 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -168,8 +168,10 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] + from twisted.internet import reactor + runs = [] - result_code = run(config, runApp=runs.append) + result_code = run(reactor, config, runApp=runs.append) self.assertThat( config.stderr.getvalue(), Contains("found invalid PID file in"), diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e63c3eef8..b6d352ab1 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -145,6 +145,7 @@ def run_cli_native(verb, *args, **kwargs): ) d.addCallback( runner.dispatch, + reactor, stdin=stdin, stdout=stdout, stderr=stderr, From ef0b2aca1769dbdf11c5eb50b66c186d2ee9e22f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 19 Sep 2022 10:12:11 -0400 Subject: [PATCH 797/916] Adjust NURL spec to new decisions. --- docs/specifications/url.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 31fb05fad..1ce3b2a7f 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -47,27 +47,27 @@ This can be considered to expand to "**N**\ ew URLs" or "Authe\ **N**\ ticating The anticipated use for a **NURL** will still be to establish a TLS connection to a peer. The protocol run over that TLS connection could be Foolscap though it is more likely to be an HTTP-based protocol (such as GBS). +Unlike fURLs, only a single net-loc is included, for consistency with other forms of URLs. +As a result, multiple NURLs may be available for a single server. + Syntax ------ The EBNF for a NURL is as follows:: - nurl = scheme, hash, "@", net-loc-list, "/", swiss-number, [ version1 ] - - scheme = "pb://" + nurl = tcp-nurl | tor-nurl | i2p-nurl + tcp-nurl = "pb://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ] + tor-nurl = "pb+tor://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ] + i2p-nurl = "pb+i2p://", hash, "@", i2p-loc, "/", swiss-number, [ version1 ] hash = unreserved - net-loc-list = net-loc, [ { ",", net-loc } ] - net-loc = tcp-loc | tor-loc | i2p-loc - - tcp-loc = [ "tcp:" ], hostname, [ ":" port ] - tor-loc = "tor:", hostname, [ ":" port ] - i2p-loc = "i2p:", i2p-addr, [ ":" port ] - - i2p-addr = { unreserved }, ".i2p" + tcp-loc = hostname, [ ":" port ] hostname = domain | IPv4address | IPv6address + i2p-loc = i2p-addr, [ ":" port ] + i2p-addr = { unreserved }, ".i2p" + swiss-number = segment version1 = "#v=1" From 4b2725df006ae172f267072a8bcb222b6be6aad9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 20 Sep 2022 10:09:43 -0400 Subject: [PATCH 798/916] Try to prevent leaking timeouts. --- src/allmydata/protocol_switch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 89570436c..a17f3055c 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -151,6 +151,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): 30, self.transport.abortConnection ) + def connectionLost(self, reason): + if self._timeout.active(): + self._timeout.cancel() + def dataReceived(self, data: bytes) -> None: """Handle incoming data. From 81c8e1c57b8b926ebb3a396f653d7149bd4f6577 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:24:02 -0600 Subject: [PATCH 799/916] windows is special --- src/allmydata/test/test_runner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 14d0dfb7f..5d8143558 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -42,6 +42,7 @@ from twisted.trial import unittest from twisted.internet import reactor from twisted.python import usage +from twisted.python.runtime import platform from twisted.internet.defer import ( inlineCallbacks, DeferredList, @@ -516,8 +517,12 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): ) yield tahoe.stop_and_wait() - # twistd.pid should be gone by now. - self.assertFalse(tahoe.twistd_pid_file.exists()) + # twistd.pid should be gone by now -- except on Windows, where + # killing a subprocess immediately exits with no chance for + # any shutdown code (that is, no Twisted shutdown hooks can + # run). + if not platform.isWindows(): + self.assertFalse(tahoe.twistd_pid_file.exists()) def _remove(self, res, file): fileutil.remove(file) From 6db1476dacc99c00a12d88b1c6af6a8aa76f3404 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:44:21 -0600 Subject: [PATCH 800/916] comment typo --- src/allmydata/scripts/tahoe_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 6dfa726a3..721ced376 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -244,7 +244,7 @@ def run(reactor, config, runApp=twistd.runApp): "--rundir", basedir, ] if sys.platform != "win32": - # turn off Twisted's pid-file to use our own -- but only on + # turn off Twisted's pid-file to use our own -- but not on # windows, because twistd doesn't know about pidfiles there twistd_args.extend(["--pidfile", None]) twistd_args.extend(config.twistd_args) From 0eeb11c9cd45598fbe2e5bdccb4f9cf50fe222f3 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:44:51 -0600 Subject: [PATCH 801/916] after shutdown --- src/allmydata/scripts/tahoe_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 721ced376..40c4a6612 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -271,7 +271,7 @@ def run(reactor, config, runApp=twistd.runApp): return 1 else: reactor.addSystemEventTrigger( - "during", "shutdown", + "after", "shutdown", lambda: cleanup_pidfile(pidfile) ) From 77bc83d341794afbd0c6884fb5e0e914dbe90632 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:45:19 -0600 Subject: [PATCH 802/916] incorrectly removed --- src/allmydata/scripts/tahoe_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 40c4a6612..eb4bb0b66 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -276,5 +276,6 @@ def run(reactor, config, runApp=twistd.runApp): ) # We always pass --nodaemon so twistd.runApp does not daemonize. + print("running node in %s" % (quoted_basedir,), file=out) runApp(twistd_config) return 0 From 1f29cc9c29e42a472ce893259f5bdbf2a31c00e0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:50:46 -0600 Subject: [PATCH 803/916] windows special --- src/allmydata/test/test_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 5d8143558..cf6e9f3b5 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -605,8 +605,10 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): ), ) - # It should not be running. - self.assertFalse(tahoe.twistd_pid_file.exists()) + # It should not be running (but windows shutdown can't run + # code so the PID file still exists there). + if not platform.isWindows(): + self.assertFalse(tahoe.twistd_pid_file.exists()) # Wait for the operation to *complete*. If we got this far it's # because we got the expected message so we can expect the "tahoe ..." From 5973196931d2143f68a34d9b01857339582ec5c0 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:00:27 -0600 Subject: [PATCH 804/916] refactor: use filelock and test it --- setup.py | 1 + src/allmydata/test/test_runner.py | 47 ++++++++++++ src/allmydata/util/pid.py | 117 ++++++++++++++++++------------ 3 files changed, 119 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index bd16a61ce..d99831347 100644 --- a/setup.py +++ b/setup.py @@ -141,6 +141,7 @@ install_requires = [ # for pid-file support "psutil", + "filelock", ] setup_requires = [ diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index cf6e9f3b5..5a8311649 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -50,6 +50,11 @@ from twisted.internet.defer import ( from twisted.python.filepath import FilePath from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv +from allmydata.util.pid import ( + check_pid_process, + _pidfile_to_lockpath, + ProcessInTheWay, +) from allmydata.test import common_util import allmydata from allmydata.scripts.runner import ( @@ -617,3 +622,45 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # What's left is a perfect indicator that the process has exited and # we won't get blamed for leaving the reactor dirty. yield client_running + + +class PidFileLocking(SyncTestCase): + """ + Direct tests for allmydata.util.pid functions + """ + + def test_locking(self): + """ + Fail to create a pidfile if another process has the lock already. + """ + # this can't just be "our" process because the locking library + # allows the same process to acquire a lock multiple times. + pidfile = FilePath("foo") + lockfile = _pidfile_to_lockpath(pidfile) + + with open("code.py", "w") as f: + f.write( + "\n".join([ + "import filelock, time", + "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), + " print('.', flush=True)", + " time.sleep(5)", + ]) + ) + proc = Popen( + [sys.executable, "code.py"], + stdout=PIPE, + stderr=PIPE, + start_new_session=True, + ) + # make sure our subprocess has had time to acquire the lock + # for sure (from the "." it prints) + self.assertThat( + proc.stdout.read(1), + Equals(b".") + ) + + # we should not be able to acuire this corresponding lock as well + with self.assertRaises(ProcessInTheWay): + check_pid_process(pidfile) + proc.terminate() diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index ff8129bbc..e256615d6 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,6 +1,10 @@ import os import psutil +# the docs are a little misleading, but this is either WindowsFileLock +# or UnixFileLock depending upon the platform we're currently on +from filelock import FileLock, Timeout + class ProcessInTheWay(Exception): """ @@ -20,6 +24,14 @@ class CannotRemovePidFile(Exception): """ +def _pidfile_to_lockpath(pidfile): + """ + internal helper. + :returns FilePath: a path to use for file-locking the given pidfile + """ + return pidfile.sibling("{}.lock".format(pidfile.basename())) + + def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -34,57 +46,70 @@ def check_pid_process(pidfile, find_process=None): :raises ProcessInTheWay: if a running process exists at our PID """ find_process = psutil.Process if find_process is None else find_process - # check if we have another instance running already - if pidfile.exists(): - with pidfile.open("r") as f: - content = f.read().decode("utf8").strip() - try: - pid, starttime = content.split() - pid = int(pid) - starttime = float(starttime) - except ValueError: - raise InvalidPidFile( - "found invalid PID file in {}".format( - pidfile - ) - ) - try: - # if any other process is running at that PID, let the - # user decide if this is another legitimate - # instance. Automated programs may use the start-time to - # help decide this (if the PID is merely recycled, the - # start-time won't match). - find_process(pid) - raise ProcessInTheWay( - "A process is already running as PID {}".format(pid) - ) - except psutil.NoSuchProcess: - print( - "'{pidpath}' refers to {pid} that isn't running".format( - pidpath=pidfile.path, - pid=pid, - ) - ) - # nothing is running at that PID so it must be a stale file - pidfile.remove() + lock_path = _pidfile_to_lockpath(pidfile) - # write our PID + start-time to the pid-file - pid = os.getpid() - starttime = find_process(pid).create_time() - with pidfile.open("w") as f: - f.write("{} {}\n".format(pid, starttime).encode("utf8")) + try: + # a short timeout is fine, this lock should only be active + # while someone is reading or deleting the pidfile .. and + # facilitates testing the locking itself. + with FileLock(lock_path.path, timeout=2): + # check if we have another instance running already + if pidfile.exists(): + with pidfile.open("r") as f: + content = f.read().decode("utf8").strip() + try: + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + except ValueError: + raise InvalidPidFile( + "found invalid PID file in {}".format( + pidfile + ) + ) + try: + # if any other process is running at that PID, let the + # user decide if this is another legitimate + # instance. Automated programs may use the start-time to + # help decide this (if the PID is merely recycled, the + # start-time won't match). + find_process(pid) + raise ProcessInTheWay( + "A process is already running as PID {}".format(pid) + ) + except psutil.NoSuchProcess: + print( + "'{pidpath}' refers to {pid} that isn't running".format( + pidpath=pidfile.path, + pid=pid, + ) + ) + # nothing is running at that PID so it must be a stale file + pidfile.remove() + + # write our PID + start-time to the pid-file + pid = os.getpid() + starttime = find_process(pid).create_time() + with pidfile.open("w") as f: + f.write("{} {}\n".format(pid, starttime).encode("utf8")) + except Timeout: + raise ProcessInTheWay( + "Another process is still locking {}".format(pidfile.path) + ) def cleanup_pidfile(pidfile): """ Safely clean up a PID-file """ - try: - pidfile.remove() - except Exception as e: - raise CannotRemovePidFile( - "Couldn't remove '{pidfile}': {err}.".format( - pidfile=pidfile.path, - err=e, + lock_path = _pidfile_to_lockpath(pidfile) + with FileLock(lock_path.path): + try: + pidfile.remove() + except Exception as e: + raise CannotRemovePidFile( + "Couldn't remove '{pidfile}': {err}.".format( + pidfile=pidfile.path, + err=e, + ) ) - ) From ea39e4ca6902daad125596f5e1e2b81989e9cb6b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:01:28 -0600 Subject: [PATCH 805/916] docstring --- src/allmydata/test/cli/test_run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 6358b70dd..551164d3c 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -185,6 +185,9 @@ class RunTests(SyncTestCase): @given(text()) def test_pidfile_contents(self, content): + """ + invalid contents for a pidfile raise errors + """ assume(not self.good_file_content_re.match(content)) pidfile = FilePath("pidfile") pidfile.setContent(content.encode("utf8")) From 56775dde192c90b48fa85cfcb4a2651f5b264791 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:05:30 -0600 Subject: [PATCH 806/916] refactor: parsing in a function --- src/allmydata/scripts/tahoe_run.py | 6 +++--- src/allmydata/util/pid.py | 34 +++++++++++++++++++----------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index eb4bb0b66..4d17492d4 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -29,6 +29,7 @@ from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_pat from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin from allmydata.util.pid import ( + parse_pidfile, check_pid_process, cleanup_pidfile, ProcessInTheWay, @@ -66,10 +67,9 @@ def get_pid_from_pidfile(pidfile): except EnvironmentError: return None - pid, _ = data.split() try: - pid = int(pid) - except ValueError: + pid, _ = parse_pidfile(pidfile) + except InvalidPidFile: return -1 return pid diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index e256615d6..1cb2cc45a 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -32,6 +32,27 @@ def _pidfile_to_lockpath(pidfile): return pidfile.sibling("{}.lock".format(pidfile.basename())) +def parse_pidfile(pidfile): + """ + :param FilePath pidfile: + :returns tuple: 2-tuple of pid, creation-time as int, float + :raises InvalidPidFile: on error + """ + with pidfile.open("r") as f: + content = f.read().decode("utf8").strip() + try: + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + except ValueError: + raise InvalidPidFile( + "found invalid PID file in {}".format( + pidfile + ) + ) + return pid, startime + + def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -55,18 +76,7 @@ def check_pid_process(pidfile, find_process=None): with FileLock(lock_path.path, timeout=2): # check if we have another instance running already if pidfile.exists(): - with pidfile.open("r") as f: - content = f.read().decode("utf8").strip() - try: - pid, starttime = content.split() - pid = int(pid) - starttime = float(starttime) - except ValueError: - raise InvalidPidFile( - "found invalid PID file in {}".format( - pidfile - ) - ) + pid, starttime = parse_pidfile(pidfile) try: # if any other process is running at that PID, let the # user decide if this is another legitimate From 390c8c52da3f9f0ea21ad789caf7c8f6ae9bbc74 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:23:30 -0600 Subject: [PATCH 807/916] formatting + typo --- newsfragments/3926.incompat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newsfragments/3926.incompat b/newsfragments/3926.incompat index 3f58b4ba8..674ad289c 100644 --- a/newsfragments/3926.incompat +++ b/newsfragments/3926.incompat @@ -1,10 +1,10 @@ -Record both the PID and the process creation-time +Record both the PID and the process creation-time: -A new kind of pidfile in `running.process` records both +a new kind of pidfile in `running.process` records both the PID and the creation-time of the process. This facilitates automatic discovery of a "stale" pidfile that points to a currently-running process. If the recorded creation-time matches the creation-time of the running process, then it is a still-running -`tahoe run` proecss. Otherwise, the file is stale. +`tahoe run` process. Otherwise, the file is stale. The `twistd.pid` file is no longer present. \ No newline at end of file From e111694b3e33e54db974acbd057d74380c6de4ce Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:28:09 -0600 Subject: [PATCH 808/916] get rid of find_process= --- src/allmydata/util/pid.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 1cb2cc45a..d681d819e 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -53,7 +53,7 @@ def parse_pidfile(pidfile): return pid, startime -def check_pid_process(pidfile, find_process=None): +def check_pid_process(pidfile): """ If another instance appears to be running already, raise an exception. Otherwise, write our PID + start time to the pidfile @@ -61,12 +61,8 @@ def check_pid_process(pidfile, find_process=None): :param FilePath pidfile: the file to read/write our PID from. - :param Callable find_process: None, or a custom way to get a - Process objet (usually for tests) - :raises ProcessInTheWay: if a running process exists at our PID """ - find_process = psutil.Process if find_process is None else find_process lock_path = _pidfile_to_lockpath(pidfile) try: @@ -83,7 +79,7 @@ def check_pid_process(pidfile, find_process=None): # instance. Automated programs may use the start-time to # help decide this (if the PID is merely recycled, the # start-time won't match). - find_process(pid) + psutil.Process(pid) raise ProcessInTheWay( "A process is already running as PID {}".format(pid) ) @@ -98,8 +94,7 @@ def check_pid_process(pidfile, find_process=None): pidfile.remove() # write our PID + start-time to the pid-file - pid = os.getpid() - starttime = find_process(pid).create_time() + starttime = psutil.Process().create_time() with pidfile.open("w") as f: f.write("{} {}\n".format(pid, starttime).encode("utf8")) except Timeout: From 0a09d23525fc4be928fa288c1301327d6eaccf32 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:29:40 -0600 Subject: [PATCH 809/916] more docstring --- src/allmydata/util/pid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index d681d819e..f965d72ab 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -105,7 +105,8 @@ def check_pid_process(pidfile): def cleanup_pidfile(pidfile): """ - Safely clean up a PID-file + Remove the pidfile specified (respecting locks). If anything at + all goes wrong, `CannotRemovePidFile` is raised. """ lock_path = _pidfile_to_lockpath(pidfile) with FileLock(lock_path.path): From 6eebbda7c6c06732932c50a96ba1a5315c9d35f4 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:07:29 -0600 Subject: [PATCH 810/916] documentation, example code --- docs/check_running.py | 47 +++++++++++++++++++++++++++++++++++++++++++ docs/running.rst | 29 ++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 docs/check_running.py diff --git a/docs/check_running.py b/docs/check_running.py new file mode 100644 index 000000000..ecc55da34 --- /dev/null +++ b/docs/check_running.py @@ -0,0 +1,47 @@ + +import psutil +import filelock + + +def can_spawn_tahoe(pidfile): + """ + Determine if we can spawn a Tahoe-LAFS for the given pidfile. That + pidfile may be deleted if it is stale. + + :param pathlib.Path pidfile: the file to check, that is the Path + to "running.process" in a Tahoe-LAFS configuration directory + + :returns bool: True if we can spawn `tahoe run` here + """ + lockpath = pidfile.parent / (pidfile.name + ".lock") + with filelock.FileLock(lockpath): + try: + with pidfile.open("r") as f: + pid, create_time = f.read().strip().split(" ", 1) + except FileNotFoundError: + return True + + # somewhat interesting: we have a pidfile + pid = int(pid) + create_time = float(create_time) + + try: + proc = psutil.Process(pid) + # most interesting case: there _is_ a process running at the + # recorded PID -- but did it just happen to get that PID, or + # is it the very same one that wrote the file? + if create_time == proc.create_time(): + # _not_ stale! another intance is still running against + # this configuration + return False + + except psutil.NoSuchProcess: + pass + + # the file is stale + pidfile.unlink() + return True + + +from pathlib import Path +print("can spawn?", can_spawn_tahoe(Path("running.process"))) diff --git a/docs/running.rst b/docs/running.rst index 406c8200b..2cff59928 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -124,6 +124,35 @@ Tahoe-LAFS. .. _magic wormhole: https://magic-wormhole.io/ +Multiple Instances +------------------ + +Running multiple instances against the same configuration directory isn't supported. +This will lead to undefined behavior and could corrupt the configuration state. + +We attempt to avoid this situation with a "pidfile"-style file in the config directory called ``running.process``. +There may be a parallel file called ``running.process.lock`` in existence. + +The ``.lock`` file exists to make sure only one process modifies ``running.process`` at once. +The lock file is managed by the `lockfile `_ library. +If you wish to make use of ``running.process`` for any reason you should also lock it and follow the semantics of lockfile. + +If ``running.process` exists it file contains the PID and the creation-time of the process. +When no such file exists, there is no other process running on this configuration. +If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config. +To tell the difference, determine if the PID in the file exists currently. +If it does, check the creation-time of the process versus the one in the file. +If these match, there is another process currently running. +Otherwise, the file is stale -- it should be removed before starting Tahoe-LAFS. + +Some example Python code to check the above situations: + +.. literalinclude:: check_running.py + + + + + A note about small grids ------------------------ From 930f4029f370313222b5c0872754f1db16434029 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:07:46 -0600 Subject: [PATCH 811/916] properly write pid, create-time --- src/allmydata/util/pid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index f965d72ab..1a833f285 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -94,9 +94,9 @@ def check_pid_process(pidfile): pidfile.remove() # write our PID + start-time to the pid-file - starttime = psutil.Process().create_time() + proc = psutil.Process() with pidfile.open("w") as f: - f.write("{} {}\n".format(pid, starttime).encode("utf8")) + f.write("{} {}\n".format(proc.pid, proc.create_time()).encode("utf8")) except Timeout: raise ProcessInTheWay( "Another process is still locking {}".format(pidfile.path) From 8474ecf83d46a25a473269f4f7907a5eb6e6e552 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:15:07 -0600 Subject: [PATCH 812/916] typo --- src/allmydata/util/pid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 1a833f285..c13dc32f3 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -50,7 +50,7 @@ def parse_pidfile(pidfile): pidfile ) ) - return pid, startime + return pid, starttime def check_pid_process(pidfile): From fedea9696412d7397f58c11ea04a9148c55f8fd8 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:26:14 -0600 Subject: [PATCH 813/916] less state --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 551164d3c..e84f52096 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -168,7 +168,7 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] - from twisted.internet import reactor + reactor = MemoryReactor() runs = [] result_code = run(reactor, config, runApp=runs.append) From 8d8b0e6f01cdd8ab1f64593eda772a8b7db6c3d6 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:40:25 -0600 Subject: [PATCH 814/916] cleanup --- src/allmydata/scripts/tahoe_run.py | 6 +----- src/allmydata/util/pid.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 4d17492d4..e22e8c307 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -62,13 +62,9 @@ def get_pid_from_pidfile(pidfile): inaccessible, ``-1`` if PID file invalid. """ try: - with open(pidfile, "r") as f: - data = f.read().strip() + pid, _ = parse_pidfile(pidfile) except EnvironmentError: return None - - try: - pid, _ = parse_pidfile(pidfile) except InvalidPidFile: return -1 diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index c13dc32f3..f12c201d1 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,4 +1,3 @@ -import os import psutil # the docs are a little misleading, but this is either WindowsFileLock From 4f5a1ac37222e51974bcb0a28b5ec9e0e6c0e944 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 23:36:23 -0600 Subject: [PATCH 815/916] naming? --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 5a8311649..962dffd1a 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -638,7 +638,7 @@ class PidFileLocking(SyncTestCase): pidfile = FilePath("foo") lockfile = _pidfile_to_lockpath(pidfile) - with open("code.py", "w") as f: + with open("other_lock.py", "w") as f: f.write( "\n".join([ "import filelock, time", @@ -648,7 +648,7 @@ class PidFileLocking(SyncTestCase): ]) ) proc = Popen( - [sys.executable, "code.py"], + [sys.executable, "other_lock.py"], stdout=PIPE, stderr=PIPE, start_new_session=True, From 8ebe331c358789f3af8dd9d64607ee63404077d7 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Sep 2022 00:11:20 -0600 Subject: [PATCH 816/916] maybe a newline helps --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 962dffd1a..3d8180c7a 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -643,7 +643,7 @@ class PidFileLocking(SyncTestCase): "\n".join([ "import filelock, time", "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), - " print('.', flush=True)", + " print('.\n', flush=True)", " time.sleep(5)", ]) ) @@ -657,7 +657,7 @@ class PidFileLocking(SyncTestCase): # for sure (from the "." it prints) self.assertThat( proc.stdout.read(1), - Equals(b".") + Equals(b".\n") ) # we should not be able to acuire this corresponding lock as well From a182a2507987213b519bcb22c6f49eec0004830c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Sep 2022 21:43:20 -0600 Subject: [PATCH 817/916] backslashes --- src/allmydata/test/test_runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 3d8180c7a..e6b7b746f 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -641,9 +641,10 @@ class PidFileLocking(SyncTestCase): with open("other_lock.py", "w") as f: f.write( "\n".join([ - "import filelock, time", + "import filelock, time, sys", "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), - " print('.\n', flush=True)", + " sys.stdout.write('.\\n')", + " sys.stdout.flush()", " time.sleep(5)", ]) ) @@ -656,7 +657,7 @@ class PidFileLocking(SyncTestCase): # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) self.assertThat( - proc.stdout.read(1), + proc.stdout.read(2), Equals(b".\n") ) From 62b92585c62c44694e5db7a8769a772b2d712a07 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Sep 2022 23:57:19 -0600 Subject: [PATCH 818/916] simplify --- src/allmydata/test/test_runner.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index e6b7b746f..5431fbaa9 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -656,12 +656,9 @@ class PidFileLocking(SyncTestCase): ) # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) - self.assertThat( - proc.stdout.read(2), - Equals(b".\n") - ) + proc.stdout.read(2), - # we should not be able to acuire this corresponding lock as well + # acquiring the same lock should fail; it is locked by the subprocess with self.assertRaises(ProcessInTheWay): check_pid_process(pidfile) proc.terminate() From 7fdeb8797e8164f1b0fd15ddda4108417545e00d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 23 Sep 2022 00:26:39 -0600 Subject: [PATCH 819/916] hardcoding bad --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 5431fbaa9..f414ed8b3 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -635,7 +635,7 @@ class PidFileLocking(SyncTestCase): """ # this can't just be "our" process because the locking library # allows the same process to acquire a lock multiple times. - pidfile = FilePath("foo") + pidfile = FilePath(self.mktemp()) lockfile = _pidfile_to_lockpath(pidfile) with open("other_lock.py", "w") as f: From f2cfd96b5e3af0fe82a7bf1ef770cad08d3969cd Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 23 Sep 2022 01:04:58 -0600 Subject: [PATCH 820/916] typo, longer timeout --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f414ed8b3..b80891642 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -645,7 +645,7 @@ class PidFileLocking(SyncTestCase): "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), " sys.stdout.write('.\\n')", " sys.stdout.flush()", - " time.sleep(5)", + " time.sleep(10)", ]) ) proc = Popen( @@ -656,7 +656,7 @@ class PidFileLocking(SyncTestCase): ) # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) - proc.stdout.read(2), + proc.stdout.read(2) # acquiring the same lock should fail; it is locked by the subprocess with self.assertRaises(ProcessInTheWay): From 8991509f8c82642d75e3070ad7ae02bfe061977d Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 00:16:40 -0600 Subject: [PATCH 821/916] blackslashes.... --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index b80891642..f8211ec02 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -642,7 +642,7 @@ class PidFileLocking(SyncTestCase): f.write( "\n".join([ "import filelock, time, sys", - "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), + "with filelock.FileLock(r'{}', timeout=1):".format(lockfile.path), " sys.stdout.write('.\\n')", " sys.stdout.flush()", " time.sleep(10)", From d42c00ae9293dd18c9f1efd22e86984c4725f222 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 00:46:30 -0600 Subject: [PATCH 822/916] do all checks with lock --- docs/check_running.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/check_running.py b/docs/check_running.py index ecc55da34..55aae0015 100644 --- a/docs/check_running.py +++ b/docs/check_running.py @@ -21,22 +21,22 @@ def can_spawn_tahoe(pidfile): except FileNotFoundError: return True - # somewhat interesting: we have a pidfile - pid = int(pid) - create_time = float(create_time) + # somewhat interesting: we have a pidfile + pid = int(pid) + create_time = float(create_time) - try: - proc = psutil.Process(pid) - # most interesting case: there _is_ a process running at the - # recorded PID -- but did it just happen to get that PID, or - # is it the very same one that wrote the file? - if create_time == proc.create_time(): - # _not_ stale! another intance is still running against - # this configuration - return False + try: + proc = psutil.Process(pid) + # most interesting case: there _is_ a process running at the + # recorded PID -- but did it just happen to get that PID, or + # is it the very same one that wrote the file? + if create_time == proc.create_time(): + # _not_ stale! another intance is still running against + # this configuration + return False - except psutil.NoSuchProcess: - pass + except psutil.NoSuchProcess: + pass # the file is stale pidfile.unlink() From d16d233872df95b8e3876e3aa32e0fdb30cc9f98 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 00:47:58 -0600 Subject: [PATCH 823/916] wording --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index 2cff59928..b487f4ae3 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -128,7 +128,7 @@ Multiple Instances ------------------ Running multiple instances against the same configuration directory isn't supported. -This will lead to undefined behavior and could corrupt the configuration state. +This will lead to undefined behavior and could corrupt the configuration or state. We attempt to avoid this situation with a "pidfile"-style file in the config directory called ``running.process``. There may be a parallel file called ``running.process.lock`` in existence. From 4919b6d9066a10028e6548800a589929c3c094d9 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 09:34:36 -0600 Subject: [PATCH 824/916] typo Co-authored-by: Jean-Paul Calderone --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index b487f4ae3..29df15a3c 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -137,7 +137,7 @@ The ``.lock`` file exists to make sure only one process modifies ``running.proce The lock file is managed by the `lockfile `_ library. If you wish to make use of ``running.process`` for any reason you should also lock it and follow the semantics of lockfile. -If ``running.process` exists it file contains the PID and the creation-time of the process. +If ``running.process`` exists then it contains the PID and the creation-time of the process. When no such file exists, there is no other process running on this configuration. If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config. To tell the difference, determine if the PID in the file exists currently. From 7aae2f78575541799002645e85a3aeab9f8706c2 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 09:34:54 -0600 Subject: [PATCH 825/916] Clarify Co-authored-by: Jean-Paul Calderone --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index 29df15a3c..263448735 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -142,7 +142,7 @@ When no such file exists, there is no other process running on this configuratio If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config. To tell the difference, determine if the PID in the file exists currently. If it does, check the creation-time of the process versus the one in the file. -If these match, there is another process currently running. +If these match, there is another process currently running and using this config. Otherwise, the file is stale -- it should be removed before starting Tahoe-LAFS. Some example Python code to check the above situations: From a7398e13f7c82707738c3862cb085d7e2a055bb2 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 09:35:17 -0600 Subject: [PATCH 826/916] Update docs/check_running.py Co-authored-by: Jean-Paul Calderone --- docs/check_running.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/check_running.py b/docs/check_running.py index 55aae0015..2705f1721 100644 --- a/docs/check_running.py +++ b/docs/check_running.py @@ -38,9 +38,9 @@ def can_spawn_tahoe(pidfile): except psutil.NoSuchProcess: pass - # the file is stale - pidfile.unlink() - return True + # the file is stale + pidfile.unlink() + return True from pathlib import Path From ca522a5293c8f2e38e9b8d2071fc3865907f4177 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 10:07:44 -0600 Subject: [PATCH 827/916] sys.argv not inline --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f8211ec02..00c87ce08 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -642,14 +642,14 @@ class PidFileLocking(SyncTestCase): f.write( "\n".join([ "import filelock, time, sys", - "with filelock.FileLock(r'{}', timeout=1):".format(lockfile.path), + "with filelock.FileLock(sys.argv[1], timeout=1):", " sys.stdout.write('.\\n')", " sys.stdout.flush()", " time.sleep(10)", ]) ) proc = Popen( - [sys.executable, "other_lock.py"], + [sys.executable, "other_lock.py", lockfile.path], stdout=PIPE, stderr=PIPE, start_new_session=True, From bef71978b6e7181598fc30f1b94d56b0b7e6a7c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 10:08:13 -0600 Subject: [PATCH 828/916] don't need start_new_session --- src/allmydata/test/test_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 00c87ce08..74e3f803e 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -652,7 +652,6 @@ class PidFileLocking(SyncTestCase): [sys.executable, "other_lock.py", lockfile.path], stdout=PIPE, stderr=PIPE, - start_new_session=True, ) # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) From 2a3b110d53146d86709a8e161d90e65d6a07f0fe Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 30 Sep 2022 16:48:23 -0600 Subject: [PATCH 829/916] simple build automation --- Makefile | 44 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 4 ++++ 2 files changed, 48 insertions(+) diff --git a/Makefile b/Makefile index 5cbd863a3..6dd2b743b 100644 --- a/Makefile +++ b/Makefile @@ -224,3 +224,47 @@ src/allmydata/_version.py: .tox/create-venvs.log: tox.ini setup.py tox --notest -p all | tee -a "$(@)" + + +# Make a new release. TODO: +# - clean checkout necessary? garbage in tarball? +release: + @echo "Is checkout clean?" + git diff-files --quiet + git diff-index --quiet --cached HEAD -- + + @echo "Install required build software" + python3 -m pip install --editable .[build] + + @echo "Test README" + python3 setup.py check -r -s + + @echo "Update NEWS" + python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag` + git add -u + git commit -m "update NEWS for release" + + @echo "Bump version and create tag" + python3 misc/build_helpers/update-version.py + + @echo "Build and sign wheel" + python3 setup.py bdist_wheel + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl + ls dist/*`git describe --abbrev=0`* + + @echo "Build and sign source-dist" + python3 setup.py sdist + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz + ls dist/*`git describe --abbrev=0`* + +release-test: + gpg --verify dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc + gpg --verify dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc + virtualenv testmf_venv + testmf_venv/bin/pip install dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl + testmf_venv/bin/tahoe-lafs --version +# ... + rm -rf testmf_venv + +release-upload: + twine upload dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc diff --git a/setup.py b/setup.py index d99831347..ffe23a7b5 100644 --- a/setup.py +++ b/setup.py @@ -380,6 +380,10 @@ setup(name="tahoe-lafs", # also set in __init__.py # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some # discussion. ':sys_platform=="win32"': ["pywin32 != 226"], + "build": [ + "dulwich", + "gpg", + ], "test": [ "flake8", # Pin a specific pyflakes so we don't have different folks From 4b708d87bd0bd6d07517a113c065e0f0329b8d34 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 30 Sep 2022 16:53:48 -0600 Subject: [PATCH 830/916] wip --- Makefile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 6dd2b743b..66f1819ad 100644 --- a/Makefile +++ b/Makefile @@ -239,6 +239,9 @@ release: @echo "Test README" python3 setup.py check -r -s +# XXX make branch, based on a ticket (provided how?) +# XXX or, specify that "make release" must run on such a branch "XXXX.tahoe-release" + @echo "Update NEWS" python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag` git add -u @@ -249,22 +252,22 @@ release: @echo "Build and sign wheel" python3 setup.py bdist_wheel - gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl - ls dist/*`git describe --abbrev=0`* + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl + ls dist/*`git describe | cut -b 12-`* @echo "Build and sign source-dist" python3 setup.py sdist - gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz - ls dist/*`git describe --abbrev=0`* + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz + ls dist/*`git describe | cut -b 12-`* release-test: - gpg --verify dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc - gpg --verify dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc + gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc + gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc virtualenv testmf_venv - testmf_venv/bin/pip install dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl + testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl testmf_venv/bin/tahoe-lafs --version # ... rm -rf testmf_venv release-upload: - twine upload dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc + twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc From 4137d6ebb7b73de4782e0d332485684cf3585376 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 30 Sep 2022 17:20:19 -0600 Subject: [PATCH 831/916] proper smoke-test --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 66f1819ad..5ad676e86 100644 --- a/Makefile +++ b/Makefile @@ -260,13 +260,13 @@ release: gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz ls dist/*`git describe | cut -b 12-`* +# basically just a bare-minimum smoke-test that it installs and runs release-test: gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc virtualenv testmf_venv testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl - testmf_venv/bin/tahoe-lafs --version -# ... + testmf_venv/bin/tahoe --version rm -rf testmf_venv release-upload: From 923f456d6e9f53ecb6db67c73a999f72027b2655 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 1 Oct 2022 14:47:19 -0600 Subject: [PATCH 832/916] all upload steps --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 5ad676e86..b68b788ca 100644 --- a/Makefile +++ b/Makefile @@ -270,4 +270,6 @@ release-test: rm -rf testmf_venv release-upload: + scp dist/*`git describe | cut -b 12-`* meejah@tahoe-lafs.org:/home/source/downloads + git push origin_push tahoe-lafs-`git describe | cut -b 12-` twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc From c711b5b0a9c825d6e1bfb3a30437da380a63b422 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 13:33:05 -0600 Subject: [PATCH 833/916] clean docs --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index b68b788ca..8b34fd0e1 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,9 @@ release: git diff-files --quiet git diff-index --quiet --cached HEAD -- + @echo "Clean docs build area" + rm -rf docs/_build/ + @echo "Install required build software" python3 -m pip install --editable .[build] From 3d3dc187646f0ba2203f73eecf2c927147400884 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 14:34:42 -0600 Subject: [PATCH 834/916] better instructions --- Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8b34fd0e1..c501ba3c5 100644 --- a/Makefile +++ b/Makefile @@ -226,8 +226,16 @@ src/allmydata/_version.py: tox --notest -p all | tee -a "$(@)" -# Make a new release. TODO: -# - clean checkout necessary? garbage in tarball? +# to make a new release: +# - create a ticket for the release in Trac +# - ensure local copy is up-to-date +# - create a branch like "XXXX.release" from up-to-date master +# - in the branch, run "make release" +# - run "make release-test" +# - perform any other sanity-checks on the release +# - run "make release-upload" +# Note that several commands below hard-code "meejah"; if you are +# someone else please adjust them. release: @echo "Is checkout clean?" git diff-files --quiet From a22be070b8ff9b4e05af9f28e61d209f64fcdeb2 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 14:51:29 -0600 Subject: [PATCH 835/916] version-updating script --- misc/build_helpers/update-version.py | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 misc/build_helpers/update-version.py diff --git a/misc/build_helpers/update-version.py b/misc/build_helpers/update-version.py new file mode 100644 index 000000000..38baf7c7c --- /dev/null +++ b/misc/build_helpers/update-version.py @@ -0,0 +1,96 @@ +# +# this updates the (tagged) version of the software +# +# Any "options" are hard-coded in here (e.g. the GnuPG key to use) +# + +author = "meejah " + + +import sys +import time +import itertools +from datetime import datetime +from packaging.version import Version + +from dulwich.repo import Repo +from dulwich.porcelain import ( + tag_list, + tag_create, + status, +) + +from twisted.internet.task import ( + react, +) +from twisted.internet.defer import ( + ensureDeferred, +) + + +def existing_tags(git): + versions = sorted( + Version(v.decode("utf8").lstrip("tahoe-lafs-")) + for v in tag_list(git) + if v.startswith(b"tahoe-lafs-") + ) + return versions + + +def create_new_version(git): + versions = existing_tags(git) + biggest = versions[-1] + + return Version( + "{}.{}.{}".format( + biggest.major, + biggest.minor + 1, + 0, + ) + ) + + +async def main(reactor): + git = Repo(".") + + st = status(git) + if any(st.staged.values()) or st.unstaged: + print("unclean checkout; aborting") + raise SystemExit(1) + + v = create_new_version(git) + if "--no-tag" in sys.argv: + print(v) + return + + print("Existing tags: {}".format("\n".join(str(x) for x in existing_tags(git)))) + print("New tag will be {}".format(v)) + + # the "tag time" is seconds from the epoch .. we quantize these to + # the start of the day in question, in UTC. + now = datetime.now() + s = now.utctimetuple() + ts = int( + time.mktime( + time.struct_time((s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0)) + ) + ) + tag_create( + repo=git, + tag="tahoe-lafs-{}".format(str(v)).encode("utf8"), + author=author.encode("utf8"), + message="Release {}".format(v).encode("utf8"), + annotated=True, + objectish=b"HEAD", + sign=author.encode("utf8"), + tag_time=ts, + tag_timezone=0, + ) + + print("Tag created locally, it is not pushed") + print("To push it run something like:") + print(" git push origin {}".format(v)) + + +if __name__ == "__main__": + react(lambda r: ensureDeferred(main(r))) From 1af48672e39ab6430583dbd38fcfe4fa61821d09 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 14:53:03 -0600 Subject: [PATCH 836/916] correct notes --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c501ba3c5..c02184a36 100644 --- a/Makefile +++ b/Makefile @@ -250,14 +250,13 @@ release: @echo "Test README" python3 setup.py check -r -s -# XXX make branch, based on a ticket (provided how?) -# XXX or, specify that "make release" must run on such a branch "XXXX.tahoe-release" - @echo "Update NEWS" python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag` git add -u git commit -m "update NEWS for release" +# note that this always bumps the "middle" number, e.g. from 1.17.1 -> 1.18.0 +# and produces a tag into the Git repository @echo "Bump version and create tag" python3 misc/build_helpers/update-version.py From 6bb46a832bfde84714d35625a256265e93688684 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 18:52:57 -0600 Subject: [PATCH 837/916] flake8 --- misc/build_helpers/update-version.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misc/build_helpers/update-version.py b/misc/build_helpers/update-version.py index 38baf7c7c..75b22edae 100644 --- a/misc/build_helpers/update-version.py +++ b/misc/build_helpers/update-version.py @@ -9,7 +9,6 @@ author = "meejah " import sys import time -import itertools from datetime import datetime from packaging.version import Version From 402d80710caa5aa50f0a5bef79ae979a75dc3594 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 19:03:10 -0600 Subject: [PATCH 838/916] news --- newsfragments/3846.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3846.feature diff --git a/newsfragments/3846.feature b/newsfragments/3846.feature new file mode 100644 index 000000000..fd321eaf0 --- /dev/null +++ b/newsfragments/3846.feature @@ -0,0 +1 @@ +"make" based release automation From e8e43d2100f1e8dfc6bd421dffd3824e57b903d0 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 19:05:16 -0600 Subject: [PATCH 839/916] update NEWS for release --- NEWS.rst | 41 +++++++++++++++++++++++++++++++++++++ newsfragments/3327.minor | 0 newsfragments/3526.minor | 1 - newsfragments/3697.minor | 1 - newsfragments/3709.minor | 0 newsfragments/3786.minor | 1 - newsfragments/3788.minor | 0 newsfragments/3802.minor | 0 newsfragments/3816.minor | 0 newsfragments/3828.feature | 8 -------- newsfragments/3846.feature | 1 - newsfragments/3855.minor | 0 newsfragments/3858.minor | 0 newsfragments/3859.minor | 0 newsfragments/3860.minor | 0 newsfragments/3865.incompat | 1 - newsfragments/3867.minor | 0 newsfragments/3868.minor | 0 newsfragments/3871.minor | 0 newsfragments/3872.minor | 0 newsfragments/3873.incompat | 1 - newsfragments/3875.minor | 0 newsfragments/3876.minor | 0 newsfragments/3877.minor | 0 newsfragments/3879.incompat | 1 - newsfragments/3881.minor | 0 newsfragments/3882.minor | 0 newsfragments/3883.minor | 0 newsfragments/3889.minor | 0 newsfragments/3890.minor | 0 newsfragments/3891.minor | 0 newsfragments/3893.minor | 0 newsfragments/3895.minor | 0 newsfragments/3896.minor | 0 newsfragments/3898.minor | 0 newsfragments/3900.minor | 0 newsfragments/3909.minor | 0 newsfragments/3913.minor | 0 newsfragments/3915.minor | 0 newsfragments/3916.minor | 0 newsfragments/3926.incompat | 10 --------- 41 files changed, 41 insertions(+), 25 deletions(-) delete mode 100644 newsfragments/3327.minor delete mode 100644 newsfragments/3526.minor delete mode 100644 newsfragments/3697.minor delete mode 100644 newsfragments/3709.minor delete mode 100644 newsfragments/3786.minor delete mode 100644 newsfragments/3788.minor delete mode 100644 newsfragments/3802.minor delete mode 100644 newsfragments/3816.minor delete mode 100644 newsfragments/3828.feature delete mode 100644 newsfragments/3846.feature delete mode 100644 newsfragments/3855.minor delete mode 100644 newsfragments/3858.minor delete mode 100644 newsfragments/3859.minor delete mode 100644 newsfragments/3860.minor delete mode 100644 newsfragments/3865.incompat delete mode 100644 newsfragments/3867.minor delete mode 100644 newsfragments/3868.minor delete mode 100644 newsfragments/3871.minor delete mode 100644 newsfragments/3872.minor delete mode 100644 newsfragments/3873.incompat delete mode 100644 newsfragments/3875.minor delete mode 100644 newsfragments/3876.minor delete mode 100644 newsfragments/3877.minor delete mode 100644 newsfragments/3879.incompat delete mode 100644 newsfragments/3881.minor delete mode 100644 newsfragments/3882.minor delete mode 100644 newsfragments/3883.minor delete mode 100644 newsfragments/3889.minor delete mode 100644 newsfragments/3890.minor delete mode 100644 newsfragments/3891.minor delete mode 100644 newsfragments/3893.minor delete mode 100644 newsfragments/3895.minor delete mode 100644 newsfragments/3896.minor delete mode 100644 newsfragments/3898.minor delete mode 100644 newsfragments/3900.minor delete mode 100644 newsfragments/3909.minor delete mode 100644 newsfragments/3913.minor delete mode 100644 newsfragments/3915.minor delete mode 100644 newsfragments/3916.minor delete mode 100644 newsfragments/3926.incompat diff --git a/NEWS.rst b/NEWS.rst index 0f9194cc4..7b1fadb8a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,47 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.18.0 (2022-10-02) +''''''''''''''''''''''''''' + +Backwards Incompatible Changes +------------------------------ + +- Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. (`#3865 `_) +- Python 3.7 or later is now required; Python 2 is no longer supported. (`#3873 `_) +- Share corruption reports stored on disk are now always encoded in UTF-8. (`#3879 `_) +- Record both the PID and the process creation-time: + + a new kind of pidfile in `running.process` records both + the PID and the creation-time of the process. This facilitates + automatic discovery of a "stale" pidfile that points to a + currently-running process. If the recorded creation-time matches + the creation-time of the running process, then it is a still-running + `tahoe run` process. Otherwise, the file is stale. + + The `twistd.pid` file is no longer present. (`#3926 `_) + + +Features +-------- + +- The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification. + + Some code existed to allow tests to shorten this and it's + conceptually possible a modified client produced mutables + with different key-sizes. However, the spec says that they + must be 2048 bits. If you happen to have a capability with + a key-size different from 2048 you may use 1.17.1 or earlier + to read the content. (`#3828 `_) +- "make" based release automation (`#3846 `_) + + +Misc/Other +---------- + +- `#3327 `_, `#3526 `_, `#3697 `_, `#3709 `_, `#3786 `_, `#3788 `_, `#3802 `_, `#3816 `_, `#3855 `_, `#3858 `_, `#3859 `_, `#3860 `_, `#3867 `_, `#3868 `_, `#3871 `_, `#3872 `_, `#3875 `_, `#3876 `_, `#3877 `_, `#3881 `_, `#3882 `_, `#3883 `_, `#3889 `_, `#3890 `_, `#3891 `_, `#3893 `_, `#3895 `_, `#3896 `_, `#3898 `_, `#3900 `_, `#3909 `_, `#3913 `_, `#3915 `_, `#3916 `_ + + Release 1.17.1 (2022-01-07) ''''''''''''''''''''''''''' diff --git a/newsfragments/3327.minor b/newsfragments/3327.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3526.minor b/newsfragments/3526.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3526.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor deleted file mode 100644 index 0977d8a6f..000000000 --- a/newsfragments/3697.minor +++ /dev/null @@ -1 +0,0 @@ -Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only). \ No newline at end of file diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3786.minor b/newsfragments/3786.minor deleted file mode 100644 index ecd1a2c4e..000000000 --- a/newsfragments/3786.minor +++ /dev/null @@ -1 +0,0 @@ -Added re-structured text documentation for the OpenMetrics format statistics endpoint. diff --git a/newsfragments/3788.minor b/newsfragments/3788.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3802.minor b/newsfragments/3802.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3816.minor b/newsfragments/3816.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3828.feature b/newsfragments/3828.feature deleted file mode 100644 index d396439b0..000000000 --- a/newsfragments/3828.feature +++ /dev/null @@ -1,8 +0,0 @@ -The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification. - -Some code existed to allow tests to shorten this and it's -conceptually possible a modified client produced mutables -with different key-sizes. However, the spec says that they -must be 2048 bits. If you happen to have a capability with -a key-size different from 2048 you may use 1.17.1 or earlier -to read the content. diff --git a/newsfragments/3846.feature b/newsfragments/3846.feature deleted file mode 100644 index fd321eaf0..000000000 --- a/newsfragments/3846.feature +++ /dev/null @@ -1 +0,0 @@ -"make" based release automation diff --git a/newsfragments/3855.minor b/newsfragments/3855.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3858.minor b/newsfragments/3858.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3859.minor b/newsfragments/3859.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3860.minor b/newsfragments/3860.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3865.incompat b/newsfragments/3865.incompat deleted file mode 100644 index 59381b269..000000000 --- a/newsfragments/3865.incompat +++ /dev/null @@ -1 +0,0 @@ -Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. \ No newline at end of file diff --git a/newsfragments/3867.minor b/newsfragments/3867.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3868.minor b/newsfragments/3868.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3871.minor b/newsfragments/3871.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3872.minor b/newsfragments/3872.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat deleted file mode 100644 index da8a5fb0e..000000000 --- a/newsfragments/3873.incompat +++ /dev/null @@ -1 +0,0 @@ -Python 3.7 or later is now required; Python 2 is no longer supported. \ No newline at end of file diff --git a/newsfragments/3875.minor b/newsfragments/3875.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3876.minor b/newsfragments/3876.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3877.minor b/newsfragments/3877.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3879.incompat b/newsfragments/3879.incompat deleted file mode 100644 index ca3f24f94..000000000 --- a/newsfragments/3879.incompat +++ /dev/null @@ -1 +0,0 @@ -Share corruption reports stored on disk are now always encoded in UTF-8. \ No newline at end of file diff --git a/newsfragments/3881.minor b/newsfragments/3881.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3882.minor b/newsfragments/3882.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3883.minor b/newsfragments/3883.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3889.minor b/newsfragments/3889.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3890.minor b/newsfragments/3890.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3891.minor b/newsfragments/3891.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3893.minor b/newsfragments/3893.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3895.minor b/newsfragments/3895.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3896.minor b/newsfragments/3896.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3898.minor b/newsfragments/3898.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3900.minor b/newsfragments/3900.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3909.minor b/newsfragments/3909.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3913.minor b/newsfragments/3913.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3915.minor b/newsfragments/3915.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3916.minor b/newsfragments/3916.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3926.incompat b/newsfragments/3926.incompat deleted file mode 100644 index 674ad289c..000000000 --- a/newsfragments/3926.incompat +++ /dev/null @@ -1,10 +0,0 @@ -Record both the PID and the process creation-time: - -a new kind of pidfile in `running.process` records both -the PID and the creation-time of the process. This facilitates -automatic discovery of a "stale" pidfile that points to a -currently-running process. If the recorded creation-time matches -the creation-time of the running process, then it is a still-running -`tahoe run` process. Otherwise, the file is stale. - -The `twistd.pid` file is no longer present. \ No newline at end of file From a53420c1931b4ec9c6a40f5105a44d7d4ac0f846 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 10:49:01 -0400 Subject: [PATCH 840/916] Use known working version of i2pd. --- integration/test_i2p.py | 2 +- newsfragments/3928.minor | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 newsfragments/3928.minor diff --git a/integration/test_i2p.py b/integration/test_i2p.py index f0b06f1e2..97abb40a5 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -55,7 +55,7 @@ def i2p_network(reactor, temp_dir, request): proto, which("docker"), ( - "docker", "run", "-p", "7656:7656", "purplei2p/i2pd", + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.43.0", # Bad URL for reseeds, so it can't talk to other routers. "--reseed.urls", "http://localhost:1/", ), diff --git a/newsfragments/3928.minor b/newsfragments/3928.minor new file mode 100644 index 000000000..e69de29bb From ec15d58e10356016130cb7eaf97681584540a611 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 10:49:08 -0400 Subject: [PATCH 841/916] Actually clean up the container. --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 97abb40a5..15f9d73cf 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request): def cleanup(): try: - proto.transport.signalProcess("KILL") + proto.transport.signalProcess("INT") util.block_with_timeout(proto.exited, reactor) except ProcessExitedAlready: pass From b86f99f0ebaa5d0239d18498f59f412967bf0b27 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:00:34 -0400 Subject: [PATCH 842/916] Make this more accurate given changes in spec. --- docs/specifications/url.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 39a830e5a..fe756208b 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -87,11 +87,13 @@ These differences are separated into distinct versions. Version 0 --------- -A Foolscap fURL is considered the canonical definition of a version 0 NURL. +In theory, a Foolscap fURL with a single netloc is considered the canonical definition of a version 0 NURL. Notably, the hash component is defined as the base32-encoded SHA1 hash of the DER form of an x509v3 certificate. A version 0 NURL is identified by the absence of the ``v=1`` fragment. +In practice, real world fURLs may have more than one netloc, so lack of version fragment will likely just involve dispatching the fURL to a different parser. + Examples ~~~~~~~~ @@ -119,7 +121,7 @@ The hash component of a version 1 NURL differs in three ways from the prior vers *all* certificate fields should be considered within the context of the relationship identified by the SPKI hash. 3. The hash is encoded using urlsafe-base64 (without padding) instead of base32. - This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 224 bit hash. + This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 256 bit hash. A version 1 NURL is identified by the presence of the ``v=1`` fragment. Though the length of the hash string (38 bytes) could also be used to differentiate it from a version 0 NURL, From b0fb72e379bcbfaabb2ae37452d9a68dd481cbea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:02:48 -0400 Subject: [PATCH 843/916] Link to design issue. --- src/allmydata/client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 9938ec076..ac8b03e2f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -591,6 +591,10 @@ def anonymous_storage_enabled(config): @implementer(IStatsProducer) class _Client(node.Node, pollmixin.PollMixin): + """ + This class should be refactored; see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 + """ STOREDIR = 'storage' NODETYPE = "client" @@ -661,7 +665,9 @@ class _Client(node.Node, pollmixin.PollMixin): # TODO this may be the wrong location for now? but as temporary measure # it allows us to get NURLs for testing in test_istorageserver.py. This # will eventually get fixed one way or another in - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901 + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901. See also + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 for the bigger + # picture issue. self.storage_nurls = set() def init_stats_provider(self): From d753bb58da880a00724bd0a9c592803ee7983fca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:05:56 -0400 Subject: [PATCH 844/916] Better type for storage_nurls. --- src/allmydata/client.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ac8b03e2f..a31d05b9c 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,17 +1,9 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - # Don't use future str to prevent leaking future's newbytes into foolscap, which they break. - from past.builtins import unicode as str +from __future__ import annotations +from typing import Optional import os, stat, time, weakref from base64 import urlsafe_b64encode from functools import partial @@ -668,7 +660,7 @@ class _Client(node.Node, pollmixin.PollMixin): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901. See also # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 for the bigger # picture issue. - self.storage_nurls = set() + self.storage_nurls : Optional[set] = None def init_stats_provider(self): self.stats_provider = StatsProvider(self) @@ -831,8 +823,8 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.storage_nurls.update( - self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + self.storage_nurls = self.tub.negotiationClass.add_storage_server( + ss, swissnum.encode("ascii") ) announcement["anonymous-storage-FURL"] = furl From d918135a0d016f579073dac7236a3aadfab76bbf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:10:36 -0400 Subject: [PATCH 845/916] Use parser instead of ad-hoc parser. --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a31d05b9c..417dffed8 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -822,7 +822,7 @@ class _Client(node.Node, pollmixin.PollMixin): if anonymous_storage_enabled(self.config): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) - (_, _, swissnum) = furl.rpartition("/") + (_, _, swissnum) = decode_furl(furl) self.storage_nurls = self.tub.negotiationClass.add_storage_server( ss, swissnum.encode("ascii") ) From 5d53cd4a170cd7315b594102f705fcb9e7eec55e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:16:30 -0400 Subject: [PATCH 846/916] Nicer API. --- src/allmydata/node.py | 8 +++++--- src/allmydata/protocol_switch.py | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 597221e9b..d6cbc9e36 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -55,7 +55,7 @@ from allmydata.util.yamlutil import ( from . import ( __full_version__, ) -from .protocol_switch import support_foolscap_and_https +from .protocol_switch import create_tub_with_https_support def _common_valid_config(): @@ -708,8 +708,10 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han :param dict tub_options: every key-value pair in here will be set in the new Tub via `Tub.setOption` """ - tub = Tub(**kwargs) - support_foolscap_and_https(tub) + # We listen simulataneously for both Foolscap and HTTPS on the same port, + # so we have to create a special Foolscap Tub for that to work: + tub = create_tub_with_https_support(**kwargs) + for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index a17f3055c..2b4ce6da1 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -6,10 +6,11 @@ simple as possible, with no extra configuration needed. Listening on the same port means a user upgrading Tahoe-LAFS will automatically get HTTPS working with no additional changes. -Use ``support_foolscap_and_https()`` to create a new subclass for a ``Tub`` -instance, and then ``add_storage_server()`` on the resulting class to add the -relevant information for a storage server once it becomes available later in -the configuration process. +Use ``create_tub_with_https_support()`` creates a new ``Tub`` that has its +``negotiationClass`` modified to be a new subclass tied to that specific +``Tub`` instance. Calling ``tub.negotiationClass.add_storage_server(...)`` +then adds relevant information for a storage server once it becomes available +later in the configuration process. """ from __future__ import annotations @@ -193,14 +194,17 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): self.__dict__ = protocol.__dict__ -def support_foolscap_and_https(tub: Tub): +def create_tub_with_https_support(**kwargs) -> Tub: """ - Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` + Create a new Tub that also supports HTTPS. + + This involves creating a new protocol switch class for the specific ``Tub`` instance. """ - the_tub = tub + the_tub = Tub(**kwargs) class FoolscapOrHttpForTub(_FoolscapOrHttps): tub = the_tub - tub.negotiationClass = FoolscapOrHttpForTub # type: ignore + the_tub.negotiationClass = FoolscapOrHttpForTub # type: ignore + return the_tub From 3034f35c7b1e1748a5d4a76f73585ced7fc1e2ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:21:54 -0400 Subject: [PATCH 847/916] Document type expectations. --- src/allmydata/storage/http_server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 540675cc7..eefb9b906 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable, Union +from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast from functools import wraps from base64 import b64decode import binascii @@ -19,6 +19,7 @@ from twisted.internet.interfaces import ( IStreamServerEndpoint, IPullProducer, ) +from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.web.server import Site, Request @@ -911,9 +912,10 @@ def listen_tls( endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path) def get_nurl(listening_port: IListeningPort) -> DecodedURL: + address = cast(Union[IPv4Address, IPv6Address], listening_port.getHost()) return build_nurl( hostname, - listening_port.getHost().port, + address.port, str(server._swissnum, "ascii"), load_pem_x509_certificate(cert_path.getContent()), ) From 58247799c1e5f12a2692be0dc72325484a38a6f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:27:19 -0400 Subject: [PATCH 848/916] Fix remaining references to refactored-out-of-existence API. --- src/allmydata/protocol_switch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2b4ce6da1..b0af84c33 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -53,13 +53,13 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): since these are created by Foolscap's ``Tub``, by setting this to be the tub's ``negotiationClass``. - Do not instantiate directly, use ``support_foolscap_and_https(tub)`` + Do not instantiate directly, use ``create_tub_with_https_support(...)`` instead. The way this class works is that a new subclass is created for a specific ``Tub`` instance. """ # These are class attributes; they will be set by - # support_foolscap_and_https() and add_storage_server(). + # create_tub_with_https_support() and add_storage_server(). # The Twisted HTTPS protocol factory wrapping the storage server HTTP API: https_factory: TLSMemoryBIOFactory From 795ec0b2dbc6e5f9d5de23fb64d8148b47025ccc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:52:07 -0400 Subject: [PATCH 849/916] Fix flake8 issue. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index d99831347..c8a9669cb 100644 --- a/setup.py +++ b/setup.py @@ -382,6 +382,9 @@ setup(name="tahoe-lafs", # also set in __init__.py ':sys_platform=="win32"': ["pywin32 != 226"], "test": [ "flake8", + # On Python 3.7, importlib_metadata v5 breaks flake8. + # https://github.com/python/importlib_metadata/issues/407 + "importlib_metadata<5; python_version < '3.8'", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it From a063241609ad918fe5617a8aedb6abaa660d36a5 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 3 Oct 2022 10:18:32 -0600 Subject: [PATCH 850/916] 1.18.0 release-notes --- relnotes.txt | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index e9b298771..dd7cc9429 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.18.0 -The Tahoe-LAFS team is pleased to announce version 1.17.1 of +The Tahoe-LAFS team is pleased to announce version 1.18.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,10 +15,12 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.17.0, released on -December 6, 2021. +The previous stable release of Tahoe-LAFS was v1.17.1, released on +January 7, 2022. -This release fixes two Python3-releated regressions and 4 minor bugs. +This release drops support for Python 2 and for Python 3.6 and earlier. +twistd.pid is no longer used (in favour of one with pid + process creation time). +A collection of minor bugs and issues were also fixed. Please see ``NEWS.rst`` [1] for a complete list of changes. @@ -132,24 +134,23 @@ Of Fame" [13]. ACKNOWLEDGEMENTS -This is the nineteenth release of Tahoe-LAFS to be created -solely as a labor of love by volunteers. Thank you very much -to the team of "hackers in the public interest" who make -Tahoe-LAFS possible. +This is the twentieth release of Tahoe-LAFS to be created solely as a +labor of love by volunteers. Thank you very much to the team of +"hackers in the public interest" who make Tahoe-LAFS possible. meejah on behalf of the Tahoe-LAFS team -January 7, 2022 +October 1, 2022 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.18.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 0e9ab8a0e3b6fe1058a2347270a5c6d3b6dfe060 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 3 Oct 2022 10:18:58 -0600 Subject: [PATCH 851/916] missed release-notes --- newsfragments/3927.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3927.minor diff --git a/newsfragments/3927.minor b/newsfragments/3927.minor new file mode 100644 index 000000000..e69de29bb From c13be0c89b8df744ff46b1b163e4b9138451169c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 09:19:48 -0400 Subject: [PATCH 852/916] Try harder to cleanup. --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 3b41e8308..bacb40290 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -198,6 +198,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.001) + # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that # so I don't know how to test that! See From 8b2884cf3a1ce0d4d17c8483202b48055646b7ed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 09:44:30 -0400 Subject: [PATCH 853/916] Make changes work again. --- src/allmydata/node.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 6747a3c77..7d33d220a 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -698,7 +698,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider): def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers, - handler_overrides={}, **kwargs): + handler_overrides={}, force_foolscap=False, **kwargs): """ Create a Tub with the right options and handlers. It will be ephemeral unless the caller provides certFile= in kwargs @@ -708,10 +708,16 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han :param dict tub_options: every key-value pair in here will be set in the new Tub via `Tub.setOption` + + :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS + storage protocol. """ - # We listen simulataneously for both Foolscap and HTTPS on the same port, + # We listen simultaneously for both Foolscap and HTTPS on the same port, # so we have to create a special Foolscap Tub for that to work: - tub = create_tub_with_https_support(**kwargs) + if force_foolscap: + tub = Tub(**kwargs) + else: + tub = create_tub_with_https_support(**kwargs) for (name, value) in list(tub_options.items()): tub.setOption(name, value) @@ -907,11 +913,10 @@ def create_main_tub(config, tub_options, tub_options, default_connection_handlers, foolscap_connection_handlers, + force_foolscap=config.get_config("node", "force_foolscap", False), handler_overrides=handler_overrides, certFile=certfile, ) - if not config.get_config("node", "force_foolscap", False): - support_foolscap_and_https(tub) if portlocation is None: log.msg("Tub is not listening") From fd07c092edf9e0367a0f2c6d770273a4ba1f6a52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 10:30:07 -0400 Subject: [PATCH 854/916] close() is called while writes are still happening. --- src/allmydata/storage_client.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f9a6feb7d..4ab818b9c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1211,7 +1211,7 @@ class _HTTPBucketWriter(object): storage_index = attr.ib(type=bytes) share_number = attr.ib(type=int) upload_secret = attr.ib(type=bytes) - finished = attr.ib(type=bool, default=False) + finished = attr.ib(type=defer.Deferred[bool], factory=defer.Deferred) def abort(self): return self.client.abort_upload(self.storage_index, self.share_number, @@ -1223,14 +1223,13 @@ class _HTTPBucketWriter(object): self.storage_index, self.share_number, self.upload_secret, offset, data ) if result.finished: - self.finished = True + self.finished.callback(True) defer.returnValue(None) def close(self): - # A no-op in HTTP protocol. - if not self.finished: - return defer.fail(RuntimeError("You didn't finish writing?!")) - return defer.succeed(None) + # We're not _really_ closed until all writes have succeeded and we + # finished writing all the data. + return self.finished From 1294baa82e71e1d4cd8c63fc2c3f6e3041062505 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 10:30:27 -0400 Subject: [PATCH 855/916] LoopingCall may already have been stopped. --- src/allmydata/storage_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 4ab818b9c..a7d5edb11 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1066,7 +1066,8 @@ class HTTPNativeStorageServer(service.MultiService): def stopService(self): service.MultiService.stopService(self) - self._lc.stop() + if self._lc.running: + self._lc.stop() self._failed_to_connect("shut down") From ea1d2486115b848ec5a8409eae328792e5d2a338 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 10:51:43 -0400 Subject: [PATCH 856/916] These objects get stored in a context where they need to be hashed, sometimes. --- src/allmydata/storage/http_client.py | 11 +++++------ src/allmydata/storage_client.py | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 16d426dda..1fe9a99fd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -276,7 +276,7 @@ class _StorageClientHTTPSPolicy: ) -@define +@define(hash=True) class StorageClient(object): """ Low-level HTTP client that talks to the HTTP storage server. @@ -286,7 +286,7 @@ class StorageClient(object): # ``StorageClient.from_nurl()``. _base_url: DecodedURL _swissnum: bytes - _treq: Union[treq, StubTreq, HTTPClient] + _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) @classmethod def from_nurl( @@ -379,13 +379,12 @@ class StorageClient(object): return self._treq.request(method, url, headers=headers, **kwargs) +@define(hash=True) class StorageClientGeneral(object): """ High-level HTTP APIs that aren't immutable- or mutable-specific. """ - - def __init__(self, client): # type: (StorageClient) -> None - self._client = client + _client : StorageClient @inlineCallbacks def get_version(self): @@ -534,7 +533,7 @@ async def advise_corrupt_share( ) -@define +@define(hash=True) class StorageClientImmutables(object): """ APIs for interacting with immutables. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a7d5edb11..3b08f0b25 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1187,7 +1187,7 @@ class _StorageServer(object): -@attr.s +@attr.s(hash=True) class _FakeRemoteReference(object): """ Emulate a Foolscap RemoteReference, calling a local object instead. @@ -1203,7 +1203,6 @@ class _FakeRemoteReference(object): raise RemoteException(e.args) -@attr.s class _HTTPBucketWriter(object): """ Emulate a ``RIBucketWriter``, but use HTTP protocol underneath. @@ -1234,7 +1233,7 @@ class _HTTPBucketWriter(object): -@attr.s +@attr.s(hash=True) class _HTTPBucketReader(object): """ Emulate a ``RIBucketReader``, but use HTTP protocol underneath. From 8190eea48924a095bf8c681fc3a7b9960d7ed839 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 11:02:36 -0400 Subject: [PATCH 857/916] Fix bug introduced in previous commit. --- src/allmydata/storage_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3b08f0b25..6d59b4f7d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1203,6 +1203,7 @@ class _FakeRemoteReference(object): raise RemoteException(e.args) +@attr.s class _HTTPBucketWriter(object): """ Emulate a ``RIBucketWriter``, but use HTTP protocol underneath. From 8b0ddf406e2863d0991f287032efbb203a15c8c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 11:17:19 -0400 Subject: [PATCH 858/916] Make HTTP and Foolscap match in another edge case. --- src/allmydata/storage_client.py | 15 ++++++++++++-- src/allmydata/test/test_istorageserver.py | 24 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6d59b4f7d..51b1eabca 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -45,6 +45,7 @@ from zope.interface import ( Interface, implementer, ) +from twisted.python.failure import Failure from twisted.web import http from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor @@ -1233,6 +1234,16 @@ class _HTTPBucketWriter(object): return self.finished +def _ignore_404(failure: Failure) -> Union[Failure, None]: + """ + Useful for advise_corrupt_share(), since it swallows unknown share numbers + in Foolscap. + """ + if failure.check(HTTPClientException) and failure.value.code == http.NOT_FOUND: + return None + else: + return failure + @attr.s(hash=True) class _HTTPBucketReader(object): @@ -1252,7 +1263,7 @@ class _HTTPBucketReader(object): return self.client.advise_corrupt_share( self.storage_index, self.share_number, str(reason, "utf-8", errors="backslashreplace") - ) + ).addErrback(_ignore_404) # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @@ -1352,7 +1363,7 @@ class _HTTPStorageServer(object): raise ValueError("Unknown share type") return client.advise_corrupt_share( storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") - ) + ).addErrback(_ignore_404) @defer.inlineCallbacks def slot_readv(self, storage_index, shares, readv): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 81025d779..a0370bdb6 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -440,6 +440,17 @@ class IStorageServerImmutableAPIsTestsMixin(object): b"immutable", storage_index, 0, b"ono" ) + @inlineCallbacks + def test_advise_corrupt_share_unknown_share_number(self): + """ + Calling ``advise_corrupt_share()`` on an immutable share, with an + unknown share number, does not result in error. + """ + storage_index, _, _ = yield self.create_share() + yield self.storage_client.advise_corrupt_share( + b"immutable", storage_index, 999, b"ono" + ) + @inlineCallbacks def test_allocate_buckets_creates_lease(self): """ @@ -909,6 +920,19 @@ class IStorageServerMutableAPIsTestsMixin(object): b"mutable", storage_index, 0, b"ono" ) + @inlineCallbacks + def test_advise_corrupt_share_unknown_share_number(self): + """ + Calling ``advise_corrupt_share()`` on a mutable share with an unknown + share number does not result in error (other behavior is opaque at this + level of abstraction). + """ + secrets, storage_index = yield self.create_slot() + + yield self.storage_client.advise_corrupt_share( + b"mutable", storage_index, 999, b"ono" + ) + @inlineCallbacks def test_STARAW_create_lease(self): """ From 0d23237b11aea61241a75e4d19c6df394b9de0b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Oct 2022 13:44:49 -0400 Subject: [PATCH 859/916] Some progress towards passing test_rref. --- src/allmydata/storage/http_client.py | 44 +++++++++++++++++++++---- src/allmydata/storage_client.py | 16 +++++---- src/allmydata/test/common_system.py | 2 ++ src/allmydata/test/test_storage_http.py | 5 +++ src/allmydata/test/test_system.py | 9 +++++ 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1fe9a99fd..2589d4e41 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,7 +20,7 @@ from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed -from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator, IReactorTime from twisted.internet.ssl import CertificateOptions from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer @@ -282,15 +282,32 @@ class StorageClient(object): Low-level HTTP client that talks to the HTTP storage server. """ + # If True, we're doing unit testing. + TEST_MODE = False + + @classmethod + def start_test_mode(cls): + """Switch to testing mode. + + In testing mode we disable persistent HTTP queries and have shorter + timeouts, to make certain tests work, but don't change the actual + semantic work being done—given a fast server, everything would work the + same. + """ + cls.TEST_MODE = True + # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use # ``StorageClient.from_nurl()``. _base_url: DecodedURL _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) + _clock: IReactorTime @classmethod def from_nurl( - cls, nurl: DecodedURL, reactor, persistent: bool = True + cls, + nurl: DecodedURL, + reactor, ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. @@ -302,16 +319,23 @@ class StorageClient(object): swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") + if cls.TEST_MODE: + pool = HTTPConnectionPool(reactor, persistent=False) + pool.retryAutomatically = False + pool.maxPersistentPerHost = 0 + else: + pool = HTTPConnectionPool(reactor) + treq_client = HTTPClient( Agent( reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), - pool=HTTPConnectionPool(reactor, persistent=persistent), + pool=pool, ) ) https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) - return cls(https_url, swissnum, treq_client) + return cls(https_url, swissnum, treq_client, reactor) def relative_url(self, path): """Get a URL relative to the base URL.""" @@ -376,7 +400,14 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return self._treq.request(method, url, headers=headers, **kwargs) + result = self._treq.request(method, url, headers=headers, **kwargs) + + # If we're in test mode, we want an aggressive timeout, e.g. for + # test_rref in test_system.py. + if self.TEST_MODE: + result.addTimeout(1, self._clock) + + return result @define(hash=True) @@ -384,7 +415,8 @@ class StorageClientGeneral(object): """ High-level HTTP APIs that aren't immutable- or mutable-specific. """ - _client : StorageClient + + _client: StorageClient @inlineCallbacks def get_version(self): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 51b1eabca..d492ee4cf 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -951,14 +951,18 @@ class HTTPNativeStorageServer(service.MultiService): self.announcement = announcement self._on_status_changed = ObserverList() furl = announcement["anonymous-storage-FURL"].encode("utf-8") - self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + ( + self._nickname, + self._permutation_seed, + self._tubid, + self._short_description, + self._long_description + ) = _parse_announcement(server_id, furl, announcement) + # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 nurl = DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]) - # Tests don't want persistent HTTPS pool, since that leaves a dirty - # reactor. As a reasonable hack, disabling persistent connnections for - # localhost allows us to have passing tests while not reducing - # performance for real-world usage. self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, nurl.host not in ("localhost", "127.0.0.1")) + StorageClient.from_nurl(nurl, reactor) ) self._connection_status = connection_status.ConnectionStatus.unstarted() diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 75379bbf3..ef4b65529 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -28,6 +28,7 @@ from foolscap.api import flushEventualQueue from allmydata import client from allmydata.introducer.server import create_introducer from allmydata.util import fileutil, log, pollmixin +from allmydata.storage import http_client from twisted.python.filepath import ( FilePath, @@ -645,6 +646,7 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): + http_client.StorageClient.start_test_mode() self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4a912cf6c..819c94f83 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -291,6 +291,7 @@ class CustomHTTPServerTests(SyncTestCase): def setUp(self): super(CustomHTTPServerTests, self).setUp() + StorageClient.start_test_mode() # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -298,6 +299,7 @@ class CustomHTTPServerTests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=StubTreq(self._http_server._app.resource()), + clock=Clock() ) def test_authorization_enforcement(self): @@ -375,6 +377,7 @@ class HttpTestFixture(Fixture): """ def _setUp(self): + StorageClient.start_test_mode() self.clock = Clock() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in @@ -396,6 +399,7 @@ class HttpTestFixture(Fixture): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=self.treq, + clock=self.clock, ) def result_of_with_flush(self, d): @@ -480,6 +484,7 @@ class GenericHTTPAPITests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), b"something wrong", treq=StubTreq(self.http.http_server.get_resource()), + clock=self.http.clock, ) ) with assert_fails_with_http_code(self, http.UNAUTHORIZED): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d859a0e00..d94b4d163 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1796,6 +1796,15 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): class Connections(SystemTestMixin, unittest.TestCase): def test_rref(self): + # The way the listening port is created is via + # SameProcessStreamEndpointAssigner (allmydata.test.common), which then + # makes an endpoint string parsed by AdoptedServerPort. The latter does + # dup(fd), which results in the filedescriptor staying alive _until the + # test ends_. That means that when we disown the service, we still have + # the listening port there on the OS level! Just the resulting + # connections aren't handled. So this test relies on aggressive + # timeouts in the HTTP client and presumably some equivalent in + # Foolscap, since connection refused does _not_ happen. self.basedir = "system/Connections/rref" d = self.set_up_nodes(2) def _start(ign): From b80a215ae1dc80a3760049bec864fe227eee1654 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Oct 2022 13:56:28 -0400 Subject: [PATCH 860/916] test_rref passes now. --- src/allmydata/storage_client.py | 8 ++++---- src/allmydata/test/common_system.py | 2 ++ src/allmydata/test/test_system.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index d492ee4cf..6f2106f87 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1052,10 +1052,9 @@ class HTTPNativeStorageServer(service.MultiService): """ See ``IServer.get_storage_server``. """ - if self.is_connected(): - return self._istorage_server - else: + if self._connection_status.summary == "unstarted": return None + return self._istorage_server def stop_connecting(self): self._lc.stop() @@ -1070,10 +1069,11 @@ class HTTPNativeStorageServer(service.MultiService): ) def stopService(self): - service.MultiService.stopService(self) + result = service.MultiService.stopService(self) if self._lc.running: self._lc.stop() self._failed_to_connect("shut down") + return result class UnknownServerTypeError(Exception): diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index ef4b65529..ee345a0c0 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -21,6 +21,7 @@ from functools import partial from twisted.internet import reactor from twisted.internet import defer from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import deferLater from twisted.application import service from foolscap.api import flushEventualQueue @@ -658,6 +659,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): log.msg("shutting down SystemTest services") d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) + d.addBoth(lambda x: deferLater(reactor, 0.01, lambda: x)) return d def getdir(self, subdir): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d94b4d163..c6d2c6bb7 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1821,9 +1821,10 @@ class Connections(SystemTestMixin, unittest.TestCase): # now shut down the server d.addCallback(lambda ign: self.clients[1].disownServiceParent()) + # and wait for the client to notice def _poll(): - return len(self.c0.storage_broker.get_connected_servers()) < 2 + return len(self.c0.storage_broker.get_connected_servers()) == 1 d.addCallback(lambda ign: self.poll(_poll)) def _down(ign): From 0f31e3cd4b054b17076ffeaa73cc412bc63191b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Oct 2022 14:41:59 -0400 Subject: [PATCH 861/916] Leave HTTP off by default for now. --- src/allmydata/node.py | 8 ++++++-- src/allmydata/test/common_system.py | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 7d33d220a..f572cf7d9 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -908,12 +908,16 @@ def create_main_tub(config, tub_options, # FIXME? "node.pem" was the CERTFILE option/thing certfile = config.get_private_path("node.pem") - tub = create_tub( tub_options, default_connection_handlers, foolscap_connection_handlers, - force_foolscap=config.get_config("node", "force_foolscap", False), + # TODO eventually we will want the default to be False, but for now we + # don't want to enable HTTP by default. + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3934 + force_foolscap=config.get_config( + "node", "force_foolscap", default=True, boolean=True + ), handler_overrides=handler_overrides, certFile=certfile, ) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index ee345a0c0..edeea5689 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -794,13 +794,13 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): if which in feature_matrix.get((section, feature), {which}): config.setdefault(section, {})[feature] = value - if force_foolscap: - config.setdefault("node", {})["force_foolscap"] = force_foolscap + #config.setdefault("node", {})["force_foolscap"] = force_foolscap setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) + setnode("force_foolscap", str(force_foolscap)) tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) setnode("tub.port", tub_port_endpoint) @@ -818,7 +818,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): " furl: %s\n") % self.introducer_furl iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") fileutil.write(iyaml_fn, iyaml) - return _render_config(config) def _set_up_client_node(self, which, force_foolscap): From 42d38433436a0f7650704fd45383688f4eeb9ac1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 09:16:59 -0400 Subject: [PATCH 862/916] Run test_system with both Foolscap and HTTP storage protocols, plus some resulting cleanups. --- src/allmydata/test/common_system.py | 37 +++++++------ src/allmydata/test/test_istorageserver.py | 65 +++++++++-------------- src/allmydata/test/test_system.py | 17 +++++- 3 files changed, 63 insertions(+), 56 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index edeea5689..96ab4e093 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -5,16 +5,7 @@ in ``allmydata.test.test_system``. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes since it causes issues on (so far unported) modules on Python 2. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 - +from typing import Optional import os from functools import partial @@ -30,6 +21,10 @@ from allmydata import client from allmydata.introducer.server import create_introducer from allmydata.util import fileutil, log, pollmixin from allmydata.storage import http_client +from allmydata.storage_client import ( + NativeStorageServer, + HTTPNativeStorageServer, +) from twisted.python.filepath import ( FilePath, @@ -646,6 +641,11 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): + # If set to True, use Foolscap for storage protocol. If set to False, HTTP + # will be used when possible. If set to None, this suggests a bug in the + # test code. + FORCE_FOOLSCAP_FOR_STORAGE : Optional[bool] = None + def setUp(self): http_client.StorageClient.start_test_mode() self.port_assigner = SameProcessStreamEndpointAssigner() @@ -702,7 +702,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): return f.read().strip() @inlineCallbacks - def set_up_nodes(self, NUMCLIENTS=5, force_foolscap=False): + def set_up_nodes(self, NUMCLIENTS=5): """ Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All of the nodes are running in this process. @@ -715,18 +715,25 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): :param int NUMCLIENTS: The number of client nodes to create. - :param bool force_foolscap: Force clients to use Foolscap instead of e.g. - HTTPS when available. - :return: A ``Deferred`` that fires when the nodes have connected to each other. """ + self.assertIn( + self.FORCE_FOOLSCAP_FOR_STORAGE, (True, False), + "You forgot to set FORCE_FOOLSCAP_FOR_STORAGE on {}".format(self.__class__) + ) self.numclients = NUMCLIENTS self.introducer = yield self._create_introducer() self.add_service(self.introducer) self.introweb_url = self._get_introducer_web() - yield self._set_up_client_nodes(force_foolscap) + yield self._set_up_client_nodes(self.FORCE_FOOLSCAP_FOR_STORAGE) + native_server = next(iter(self.clients[0].storage_broker.get_known_servers())) + if self.FORCE_FOOLSCAP_FOR_STORAGE: + expected_storage_server_class = NativeStorageServer + else: + expected_storage_server_class = HTTPNativeStorageServer + self.assertIsInstance(native_server, expected_storage_server_class) @inlineCallbacks def _set_up_client_nodes(self, force_foolscap): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a0370bdb6..a488622c7 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1046,13 +1046,12 @@ class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" SKIP_TESTS = set() # type: Set[str] - FORCE_FOOLSCAP = False - - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) def _get_istorage_server(self): - raise NotImplementedError("implement in subclass") + native_server = next(iter(self.clients[0].storage_broker.get_known_servers())) + client = native_server.get_storage_server() + self.assertTrue(IStorageServer.providedBy(client)) + return client @inlineCallbacks def setUp(self): @@ -1065,7 +1064,7 @@ class _SharedMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) - yield self.set_up_nodes(1, self.FORCE_FOOLSCAP) + yield self.set_up_nodes(1) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1075,7 +1074,7 @@ class _SharedMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock - self.storage_client = yield self._get_istorage_server() + self.storage_client = self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1091,49 +1090,29 @@ class _SharedMixin(SystemTestMixin): yield SystemTestMixin.tearDown(self) -class _FoolscapMixin(_SharedMixin): - """Run tests on Foolscap version of ``IStorageServer``.""" - - FORCE_FOOLSCAP = True - - def _get_istorage_server(self): - native_server = self._get_native_server() - assert isinstance(native_server, NativeStorageServer) - client = native_server.get_storage_server() - self.assertTrue(IStorageServer.providedBy(client)) - return succeed(client) - - -class _HTTPMixin(_SharedMixin): - """Run tests on the HTTP version of ``IStorageServer``.""" - - FORCE_FOOLSCAP = False - - def _get_istorage_server(self): - native_server = self._get_native_server() - assert isinstance(native_server, HTTPNativeStorageServer) - client = native_server.get_storage_server() - self.assertTrue(IStorageServer.providedBy(client)) - return succeed(client) - - class FoolscapSharedAPIsTests( - _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = True + class HTTPSharedAPIsTests( - _HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for shared ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = False + class FoolscapImmutableAPIsTests( - _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = True + def test_disconnection(self): """ If we disconnect in the middle of writing to a bucket, all data is @@ -1156,23 +1135,29 @@ class FoolscapImmutableAPIsTests( """ current = self.storage_client yield self.bounce_client(0) - self.storage_client = self._get_native_server().get_storage_server() + self.storage_client = self._get_istorage_server() assert self.storage_client is not current class HTTPImmutableAPIsTests( - _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = False + class FoolscapMutableAPIsTests( - _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for mutable ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = True + class HTTPMutableAPIsTests( - _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" + + FORCE_FOOLSCAP_FOR_STORAGE = False diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c6d2c6bb7..a83ff9488 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -117,7 +117,8 @@ class CountingDataUploadable(upload.Data): class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): - + """Foolscap integration-y tests.""" + FORCE_FOOLSCAP_FOR_STORAGE = True timeout = 180 def test_connections(self): @@ -1794,6 +1795,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): class Connections(SystemTestMixin, unittest.TestCase): + FORCE_FOOLSCAP_FOR_STORAGE = True def test_rref(self): # The way the listening port is created is via @@ -1834,3 +1836,16 @@ class Connections(SystemTestMixin, unittest.TestCase): self.assertEqual(storage_server, self.s1_storage_server) d.addCallback(_down) return d + + +class HTTPSystemTest(SystemTest): + """HTTP storage protocol variant of the system tests.""" + + FORCE_FOOLSCAP_FOR_STORAGE = False + + + +class HTTPConnections(Connections): + """HTTP storage protocol variant of the connections tests.""" + FORCE_FOOLSCAP_FOR_STORAGE = False + From e409262e86ff3639187bfa89f438b6e9db071228 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 09:50:07 -0400 Subject: [PATCH 863/916] Fix some flakes. --- src/allmydata/test/test_istorageserver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a488622c7..9e7e7b6e1 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -15,7 +15,7 @@ from typing import Set from random import Random from unittest import SkipTest -from twisted.internet.defer import inlineCallbacks, returnValue, succeed +from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock from foolscap.api import Referenceable, RemoteException @@ -25,10 +25,6 @@ from allmydata.interfaces import IStorageServer from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage_client import ( - NativeStorageServer, - HTTPNativeStorageServer, -) # Use random generator with known seed, so results are reproducible if tests From 0febc8745653992cbb53d98702c92edc24b7a516 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 10:03:06 -0400 Subject: [PATCH 864/916] Don't include reactor in comparison. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2589d4e41..40979d3cb 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -301,7 +301,7 @@ class StorageClient(object): _base_url: DecodedURL _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) - _clock: IReactorTime + _clock: IReactorTime = field(eq=False) @classmethod def from_nurl( From f68c3978f616c5efecc15094aa83c363bd6db58d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 10:18:38 -0400 Subject: [PATCH 865/916] News fragment. --- newsfragments/3783.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3783.minor diff --git a/newsfragments/3783.minor b/newsfragments/3783.minor new file mode 100644 index 000000000..e69de29bb From 1a3e3a86c317c22a79790cc134102f6dc5b368ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 11:27:04 -0400 Subject: [PATCH 866/916] Require latest pycddl, and work around a regression. --- newsfragments/3938.bugfix | 1 + setup.py | 2 +- src/allmydata/storage/http_client.py | 12 ++++++------ src/allmydata/storage/http_server.py | 12 +++++------- 4 files changed, 13 insertions(+), 14 deletions(-) create mode 100644 newsfragments/3938.bugfix diff --git a/newsfragments/3938.bugfix b/newsfragments/3938.bugfix new file mode 100644 index 000000000..c2778cfdf --- /dev/null +++ b/newsfragments/3938.bugfix @@ -0,0 +1 @@ +Work with (and require) newer versions of pycddl. \ No newline at end of file diff --git a/setup.py b/setup.py index 72478767c..768e44e29 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,7 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - "pycddl", + "pycddl >= 0.2", # for pid-file support "psutil", diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 16d426dda..420d3610f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -83,35 +83,35 @@ _SCHEMAS = { "allocate_buckets": Schema( """ response = { - already-have: #6.258([* uint]) - allocated: #6.258([* uint]) + already-have: #6.258([0*256 uint]) + allocated: #6.258([0*256 uint]) } """ ), "immutable_write_share_chunk": Schema( """ response = { - required: [* {begin: uint, end: uint}] + required: [0* {begin: uint, end: uint}] } """ ), "list_shares": Schema( """ - response = #6.258([* uint]) + response = #6.258([0*256 uint]) """ ), "mutable_read_test_write": Schema( """ response = { "success": bool, - "data": {* share_number: [* bstr]} + "data": {0*256 share_number: [0* bstr]} } share_number = uint """ ), "mutable_list_shares": Schema( """ - response = #6.258([* uint]) + response = #6.258([0*256 uint]) """ ), } diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index eefb9b906..3902976ba 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -260,7 +260,7 @@ _SCHEMAS = { "allocate_buckets": Schema( """ request = { - share-numbers: #6.258([*256 uint]) + share-numbers: #6.258([0*256 uint]) allocated-size: uint } """ @@ -276,15 +276,13 @@ _SCHEMAS = { """ request = { "test-write-vectors": { - ; TODO Add length limit here, after - ; https://github.com/anweiss/cddl/issues/128 is fixed - * share_number => { - "test": [*30 {"offset": uint, "size": uint, "specimen": bstr}] - "write": [*30 {"offset": uint, "data": bstr}] + 0*256 share_number : { + "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}] + "write": [0*30 {"offset": uint, "data": bstr}] "new-length": uint / null } } - "read-vector": [*30 {"offset": uint, "size": uint}] + "read-vector": [0*30 {"offset": uint, "size": uint}] } share_number = uint """ From 46fbe3d0283695dc503fabb0b9f8c4ed9401cdcf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 18 Oct 2022 17:32:23 -0400 Subject: [PATCH 867/916] bump pypi-deps-db for new pycddl version --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 79eabe7a1..950151416 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -53,10 +53,10 @@ "homepage": "", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a", - "sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x", + "rev": "5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6", + "sha256": "0pc6mj7rzvmhh303rvj5wf4hrksm4h2rf4fsvqs0ljjdmgxrqm3f", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz", + "url": "https://github.com/DavHau/pypi-deps-db/archive/5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From 48ae729c0de57818d132763aa62e99faffd46556 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Nov 2022 10:18:23 -0400 Subject: [PATCH 868/916] Don't reuse basedir across tests. --- src/allmydata/test/test_system.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index a83ff9488..f03d795ba 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -121,8 +121,13 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): FORCE_FOOLSCAP_FOR_STORAGE = True timeout = 180 + @property + def basedir(self): + return "system/SystemTest/{}-foolscap-{}".format( + self.id().split(".")[-1], self.FORCE_FOOLSCAP_FOR_STORAGE + ) + def test_connections(self): - self.basedir = "system/SystemTest/test_connections" d = self.set_up_nodes() self.extra_node = None d.addCallback(lambda res: self.add_extra_node(self.numclients)) @@ -150,11 +155,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): del test_connections def test_upload_and_download_random_key(self): - self.basedir = "system/SystemTest/test_upload_and_download_random_key" return self._test_upload_and_download(convergence=None) def test_upload_and_download_convergent(self): - self.basedir = "system/SystemTest/test_upload_and_download_convergent" return self._test_upload_and_download(convergence=b"some convergence string") def _test_upload_and_download(self, convergence): @@ -517,7 +520,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_mutable(self): - self.basedir = "system/SystemTest/test_mutable" DATA = b"initial contents go here." # 25 bytes % 3 != 0 DATA_uploadable = MutableData(DATA) NEWDATA = b"new contents yay" @@ -747,7 +749,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # plaintext_hash check. def test_filesystem(self): - self.basedir = "system/SystemTest/test_filesystem" self.data = LARGE_DATA d = self.set_up_nodes() def _new_happy_semantics(ign): @@ -1714,7 +1715,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. - self.basedir = "system/SystemTest/test_filesystem_with_cli_in_subprocess" d = self.set_up_nodes() def _new_happy_semantics(ign): for c in self.clients: @@ -1807,7 +1807,9 @@ class Connections(SystemTestMixin, unittest.TestCase): # connections aren't handled. So this test relies on aggressive # timeouts in the HTTP client and presumably some equivalent in # Foolscap, since connection refused does _not_ happen. - self.basedir = "system/Connections/rref" + self.basedir = "system/Connections/rref-foolscap-{}".format( + self.FORCE_FOOLSCAP_FOR_STORAGE + ) d = self.set_up_nodes(2) def _start(ign): self.c0 = self.clients[0] From e05136c2385d222bd50413054dc8ac2a9d60d243 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Nov 2022 13:13:21 -0400 Subject: [PATCH 869/916] Less aggressive timeout, to try to make tests pass on CI. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index adc3e1525..e520088c3 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -405,7 +405,7 @@ class StorageClient(object): # If we're in test mode, we want an aggressive timeout, e.g. for # test_rref in test_system.py. if self.TEST_MODE: - result.addTimeout(1, self._clock) + result.addTimeout(5, self._clock) return result From db59eb12c092264f357c59afc3586dcb8259d0f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Nov 2022 15:22:36 -0400 Subject: [PATCH 870/916] Increase timeout. --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 764651c40..854013c32 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -52,7 +52,7 @@ fi # This is primarily aimed at catching hangs on the PyPy job which runs for # about 21 minutes and then gets killed by CircleCI in a way that fails the # job and bypasses our "allowed failure" logic. -TIMEOUT="timeout --kill-after 1m 15m" +TIMEOUT="timeout --kill-after 1m 25m" # Run the test suite as a non-root user. This is the expected usage some # small areas of the test suite assume non-root privileges (such as unreadable From 262d9d85b97cb064da47c82bab22e62b48db6cd4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Nov 2022 14:14:21 -0400 Subject: [PATCH 871/916] Switch to using persistent connections in tests too. --- src/allmydata/storage/http_client.py | 34 +++++++++++++++------------- src/allmydata/test/common_system.py | 10 +++++++- src/allmydata/test/test_system.py | 3 +++ 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e520088c3..96820d4a5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -282,19 +282,20 @@ class StorageClient(object): Low-level HTTP client that talks to the HTTP storage server. """ - # If True, we're doing unit testing. - TEST_MODE = False + # If set, we're doing unit testing and we should call this with + # HTTPConnectionPool we create. + TEST_MODE_REGISTER_HTTP_POOL = None @classmethod - def start_test_mode(cls): + def start_test_mode(cls, callback): """Switch to testing mode. - In testing mode we disable persistent HTTP queries and have shorter - timeouts, to make certain tests work, but don't change the actual - semantic work being done—given a fast server, everything would work the - same. + In testing mode we register the pool with test system using the given + callback so it can Do Things, most notably killing off idle HTTP + connections at test shutdown and, in some tests, in the midddle of the + test. """ - cls.TEST_MODE = True + cls.TEST_MODE_REGISTER_HTTP_POOL = callback # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use # ``StorageClient.from_nurl()``. @@ -318,13 +319,10 @@ class StorageClient(object): assert nurl.scheme == "pb" swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") + pool = HTTPConnectionPool(reactor) - if cls.TEST_MODE: - pool = HTTPConnectionPool(reactor, persistent=False) - pool.retryAutomatically = False - pool.maxPersistentPerHost = 0 - else: - pool = HTTPConnectionPool(reactor) + if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: + cls.TEST_MODE_REGISTER_HTTP_POOL(pool) treq_client = HTTPClient( Agent( @@ -403,8 +401,12 @@ class StorageClient(object): result = self._treq.request(method, url, headers=headers, **kwargs) # If we're in test mode, we want an aggressive timeout, e.g. for - # test_rref in test_system.py. - if self.TEST_MODE: + # test_rref in test_system.py. Unfortunately, test_rref results in the + # socket still listening(!), only without an HTTP server, due to limits + # in the relevant socket-binding test setup code. As a result, we don't + # get connection refused, the client will successfully connect. So we + # want a timeout so we notice that. + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: result.addTimeout(5, self._clock) return result diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 96ab4e093..f47aad3b6 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -647,7 +647,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): FORCE_FOOLSCAP_FOR_STORAGE : Optional[bool] = None def setUp(self): - http_client.StorageClient.start_test_mode() + self._http_client_pools = [] + http_client.StorageClient.start_test_mode(self._http_client_pools.append) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) @@ -655,10 +656,17 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.sparent = service.MultiService() self.sparent.startService() + def close_idle_http_connections(self): + """Close all HTTP client connections that are just hanging around.""" + return defer.gatherResults( + [pool.closeCachedConnections() for pool in self._http_client_pools] + ) + def tearDown(self): log.msg("shutting down SystemTest services") d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) + d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) d.addBoth(lambda x: deferLater(reactor, 0.01, lambda: x)) return d diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index f03d795ba..670ac5868 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1826,6 +1826,9 @@ class Connections(SystemTestMixin, unittest.TestCase): # now shut down the server d.addCallback(lambda ign: self.clients[1].disownServiceParent()) + # kill any persistent http connections that might continue to work + d.addCallback(lambda ign: self.close_idle_http_connections()) + # and wait for the client to notice def _poll(): return len(self.c0.storage_broker.get_connected_servers()) == 1 From 8bebb09edd2026a77dd6f8081a1fe7c0069071b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Nov 2022 14:38:59 -0400 Subject: [PATCH 872/916] Less test-specific way to make test_rref pass. --- src/allmydata/storage/http_client.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 96820d4a5..7fcf8114c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -398,18 +398,7 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - result = self._treq.request(method, url, headers=headers, **kwargs) - - # If we're in test mode, we want an aggressive timeout, e.g. for - # test_rref in test_system.py. Unfortunately, test_rref results in the - # socket still listening(!), only without an HTTP server, due to limits - # in the relevant socket-binding test setup code. As a result, we don't - # get connection refused, the client will successfully connect. So we - # want a timeout so we notice that. - if self.TEST_MODE_REGISTER_HTTP_POOL is not None: - result.addTimeout(5, self._clock) - - return result + return self._treq.request(method, url, headers=headers, **kwargs) @define(hash=True) @@ -426,7 +415,12 @@ class StorageClientGeneral(object): Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - response = yield self._client.request("GET", url) + result = self._client.request("GET", url) + # 1. Getting the version should never take particularly long. + # 2. Clients rely on the version command for liveness checks of servers. + result.addTimeout(5, self._client._clock) + + response = yield result decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) From 1e50e96e2456910598862e64f7585a6dd47d59f2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Nov 2022 15:04:41 -0400 Subject: [PATCH 873/916] Update to new test API. --- src/allmydata/test/test_storage_http.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 819c94f83..25c21e03f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -291,7 +291,9 @@ class CustomHTTPServerTests(SyncTestCase): def setUp(self): super(CustomHTTPServerTests, self).setUp() - StorageClient.start_test_mode() + StorageClient.start_test_mode( + lambda pool: self.addCleanup(pool.closeCachedConnections) + ) # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -299,7 +301,7 @@ class CustomHTTPServerTests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=StubTreq(self._http_server._app.resource()), - clock=Clock() + clock=Clock(), ) def test_authorization_enforcement(self): @@ -377,7 +379,9 @@ class HttpTestFixture(Fixture): """ def _setUp(self): - StorageClient.start_test_mode() + StorageClient.start_test_mode( + lambda pool: self.addCleanup(pool.closeCachedConnections) + ) self.clock = Clock() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in @@ -1446,7 +1450,9 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format( + self.KIND, _encode_si(storage_index) + ) ), headers=headers, ) From 414b4635569145ed277bfe0e0e540d62430e0bb8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 09:23:04 -0500 Subject: [PATCH 874/916] Use built-in treq timeout feature. --- src/allmydata/storage/http_client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7fcf8114c..d6121aba2 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -415,12 +415,10 @@ class StorageClientGeneral(object): Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - result = self._client.request("GET", url) # 1. Getting the version should never take particularly long. # 2. Clients rely on the version command for liveness checks of servers. - result.addTimeout(5, self._client._clock) - - response = yield result + # Thus, a short timeout. + response = yield self._client.request("GET", url, timeout=5) decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) From c4772482ef19d5e1aeed99f01e38fab52a14786d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:19:00 -0500 Subject: [PATCH 875/916] WIP --- src/allmydata/storage/http_client.py | 33 +++++++++++++--- src/allmydata/test/test_storage_http.py | 51 ++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 420d3610f..0bf68fdd3 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,8 +20,13 @@ from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed -from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.interfaces import ( + IOpenSSLClientConnectionCreator, + IReactorTime, + IDelayedCall, +) from twisted.internet.ssl import CertificateOptions +from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -124,16 +129,20 @@ class _LengthLimitedCollector: """ remaining_length: int + timeout_on_silence: IDelayedCall f: BytesIO = field(factory=BytesIO) def __call__(self, data: bytes): + self.timeout_on_silence.reset(60) self.remaining_length -= len(data) if self.remaining_length < 0: raise ValueError("Response length was too long") self.f.write(data) -def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[BinaryIO]: +def limited_content( + response, max_length: int = 30 * 1024 * 1024, clock: IReactorTime = reactor +) -> Deferred[BinaryIO]: """ Like ``treq.content()``, but limit data read from the response to a set length. If the response is longer than the max allowed length, the result @@ -142,11 +151,16 @@ def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[Bi A potentially useful future improvement would be using a temporary file to store the content; since filesystem buffering means that would use memory for small responses and disk for large responses. + + This will time out if no data is received for 60 seconds; so long as a + trickle of data continues to arrive, it will continue to run. """ - collector = _LengthLimitedCollector(max_length) + d = succeed(None) + timeout = clock.callLater(60, d.cancel) + collector = _LengthLimitedCollector(max_length, timeout) + # Make really sure everything gets called in Deferred context, treq might # call collector directly... - d = succeed(None) d.addCallback(lambda _: treq.collect(response, collector)) def done(_): @@ -307,6 +321,8 @@ class StorageClient(object): reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), pool=HTTPConnectionPool(reactor, persistent=persistent), + # TCP-level connection timeout + connectTimeout=5, ) ) @@ -337,6 +353,7 @@ class StorageClient(object): write_enabler_secret=None, headers=None, message_to_serialize=None, + timeout: Union[int, float] = 60, **kwargs, ): """ @@ -376,7 +393,9 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return self._treq.request(method, url, headers=headers, **kwargs) + return self._treq.request( + method, url, headers=headers, timeout=timeout, **kwargs + ) class StorageClientGeneral(object): @@ -461,6 +480,9 @@ def read_share_chunk( share_type, _encode_si(storage_index), share_number ) ) + # The default timeout is for getting the response, so it doesn't include + # the time it takes to download the body... so we will will deal with that + # later. response = yield client.request( "GET", url, @@ -469,6 +491,7 @@ def read_share_chunk( # but Range constructor does that the conversion for us. {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), + unbuffered=True, # Don't buffer the response in memory. ) if response.code == http.NO_CONTENT: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4a912cf6c..54a26da09 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -31,6 +31,8 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock, Cooperator +from twisted.internet.interfaces import IReactorTime +from twisted.internet.defer import CancelledError, Deferred from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -245,6 +247,7 @@ def gen_bytes(length: int) -> bytes: class TestApp(object): """HTTP API for testing purposes.""" + clock: IReactorTime _app = Klein() _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using @@ -266,6 +269,17 @@ class TestApp(object): """Return bytes to the given length using ``gen_bytes()``.""" return gen_bytes(length) + @_authorized_route(_app, set(), "/slowly_never_finish_result", methods=["GET"]) + def slowly_never_finish_result(self, request, authorization): + """ + Send data immediately, after 59 seconds, after another 59 seconds, and then + never again, without finishing the response. + """ + request.write(b"a") + self.clock.callLater(59, request.write, b"b") + self.clock.callLater(59 + 59, request.write, b"c") + return Deferred() + def result_of(d): """ @@ -299,6 +313,10 @@ class CustomHTTPServerTests(SyncTestCase): SWISSNUM_FOR_TEST, treq=StubTreq(self._http_server._app.resource()), ) + # We're using a Treq private API to get the reactor, alas, but only in + # a test, so not going to worry about it too much. This would be fixed + # if https://github.com/twisted/treq/issues/226 were ever fixed. + self._http_server.clock = self.client._treq._agent._memoryReactor def test_authorization_enforcement(self): """ @@ -367,6 +385,35 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(ValueError): result_of(limited_content(response, too_short)) + def test_limited_content_silence_causes_timeout(self): + """ + ``http_client.limited_content() times out if it receives no data for 60 + seconds. + """ + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/slowly_never_finish_result", + ) + ) + + body_deferred = limited_content(response, 4, self._http_server.clock) + result = [] + error = [] + body_deferred.addCallbacks(result.append, error.append) + + for i in range(59 + 59 + 60): + self.assertEqual((result, error), ([], [])) + self._http_server.clock.advance(1) + # Push data between in-memory client and in-memory server: + self.client._treq._agent.flush() + + # After 59 (second write) + 59 (third write) + 60 seconds (quiescent + # timeout) the limited_content() response times out. + self.assertTrue(error) + with self.assertRaises(CancelledError): + error[0].raiseException() + class HttpTestFixture(Fixture): """ @@ -1441,7 +1488,9 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format( + self.KIND, _encode_si(storage_index) + ) ), headers=headers, ) From f8b9607fc2c1062609eb3bcf42024ad7e81e729f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:26:11 -0500 Subject: [PATCH 876/916] Finish up limited_content() timeout code. --- src/allmydata/storage/http_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 56f7aa629..c76cd00b9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -164,6 +164,7 @@ def limited_content( d.addCallback(lambda _: treq.collect(response, collector)) def done(_): + timeout.cancel() collector.f.seek(0) return collector.f @@ -539,7 +540,7 @@ def read_share_chunk( raise ValueError("Server sent more than we asked for?!") # It might also send less than we asked for. That's (probably) OK, e.g. # if we went past the end of the file. - body = yield limited_content(response, supposed_length) + body = yield limited_content(response, supposed_length, client._clock) body.seek(0, SEEK_END) actual_length = body.tell() if actual_length != supposed_length: From 2c911eeac1901fbc333d550e6923d225e6ed07cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:28:36 -0500 Subject: [PATCH 877/916] Make sure everything is using the same clock. --- src/allmydata/test/test_storage_http.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 55bc8f79a..3ee955c3b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -311,16 +311,18 @@ class CustomHTTPServerTests(SyncTestCase): # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() + treq = StubTreq(self._http_server._app.resource()) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - clock=Clock(), + treq=treq, + # We're using a Treq private API to get the reactor, alas, but only + # in a test, so not going to worry about it too much. This would be + # fixed if https://github.com/twisted/treq/issues/226 were ever + # fixed. + clock=treq._agent._memoryReactor, ) - # We're using a Treq private API to get the reactor, alas, but only in - # a test, so not going to worry about it too much. This would be fixed - # if https://github.com/twisted/treq/issues/226 were ever fixed. - self._http_server.clock = self.client._treq._agent._memoryReactor + self._http_server.clock = self.client._clock def test_authorization_enforcement(self): """ From afd4f52ff74d4d3f73258ec9ac27e1dea3a928e5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:32:14 -0500 Subject: [PATCH 878/916] News file. --- newsfragments/3940.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3940.minor diff --git a/newsfragments/3940.minor b/newsfragments/3940.minor new file mode 100644 index 000000000..e69de29bb From 65a7945fd9de23ad34c5f17bbf7cfe898243b9e2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:39:45 -0500 Subject: [PATCH 879/916] Don't need a connection timeout since we have request-level timeouts. --- src/allmydata/storage/http_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index c76cd00b9..adf4eb7fa 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -343,8 +343,6 @@ class StorageClient(object): Agent( reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), - # TCP-level connection timeout - connectTimeout=5, pool=pool, ) ) @@ -385,6 +383,8 @@ class StorageClient(object): If ``message_to_serialize`` is set, it will be serialized (by default with CBOR) and set as the request body. + + Default timeout is 60 seconds. """ headers = self._get_headers(headers) @@ -506,9 +506,9 @@ def read_share_chunk( share_type, _encode_si(storage_index), share_number ) ) - # The default timeout is for getting the response, so it doesn't include - # the time it takes to download the body... so we will will deal with that - # later. + # The default 60 second timeout is for getting the response, so it doesn't + # include the time it takes to download the body... so we will will deal + # with that later, via limited_content(). response = yield client.request( "GET", url, From 8d678fe3de4dacdf206e737ef130a91b92004656 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:41:50 -0500 Subject: [PATCH 880/916] Increase timeout. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index f47aad3b6..90990a8ca 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -667,7 +667,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 0.01, lambda: x)) + d.addBoth(lambda x: deferLater(reactor, 0.02, lambda: x)) return d def getdir(self, subdir): From 90f1eb6245d176bc2a9f32098be1971fb0857f51 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Nov 2022 09:24:29 -0500 Subject: [PATCH 881/916] Fix the fURL and NURL links --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- docs/specifications/url.rst | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 8fe855be3..6643c08f2 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -30,11 +30,11 @@ Glossary introducer a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers - `fURL `_ + :ref:`fURLs ` a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol (the storage service is an example of such an object) - `NURL `_ + :ref:`NURLs ` a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap swissnum diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 421ac57f7..efc7ad76c 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -7,11 +7,11 @@ These are not to be confused with the URI-like capabilities Tahoe-LAFS uses to r An attempt is also made to outline the rationale for certain choices about these URLs. The intended audience for this document is Tahoe-LAFS maintainers and other developers interested in interoperating with Tahoe-LAFS or these URLs. +.. _furls: + Background ---------- -.. _fURLs: - Tahoe-LAFS first used Foolscap_ for network communication. Foolscap connection setup takes as an input a Foolscap URL or a *fURL*. A fURL includes three components: @@ -33,6 +33,8 @@ The client's use of the swissnum is what allows the server to authorize the clie .. _`swiss number`: http://wiki.erights.org/wiki/Swiss_number +.. _NURLs: + NURLs ----- From d1287df62990d7c096e1935718c2f048d1a2039d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:02:19 -0500 Subject: [PATCH 882/916] The short timeout should be specific to the storage client's needs. --- src/allmydata/storage/http_client.py | 5 +---- src/allmydata/storage_client.py | 8 ++++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d6121aba2..d468d2436 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -415,10 +415,7 @@ class StorageClientGeneral(object): Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - # 1. Getting the version should never take particularly long. - # 2. Clients rely on the version command for liveness checks of servers. - # Thus, a short timeout. - response = yield self._client.request("GET", url, timeout=5) + response = yield self._client.request("GET", url) decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6f2106f87..140e29607 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -944,12 +944,13 @@ class HTTPNativeStorageServer(service.MultiService): "connected". """ - def __init__(self, server_id: bytes, announcement): + def __init__(self, server_id: bytes, announcement, reactor=reactor): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id self.announcement = announcement self._on_status_changed = ObserverList() + self._reactor = reactor furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( self._nickname, @@ -1063,7 +1064,10 @@ class HTTPNativeStorageServer(service.MultiService): self._connect() def _connect(self): - return self._istorage_server.get_version().addCallbacks( + result = self._istorage_server.get_version() + # Set a short timeout since we're relying on this for server liveness. + result.addTimeout(5, self._reactor) + result.addCallbacks( self._got_version, self._failed_to_connect ) From 6c80ad5290c634a3395a3c5a222a15f6ed9f0abe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:13:50 -0500 Subject: [PATCH 883/916] Not necessary. --- src/allmydata/storage/http_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d468d2436..f0b45742c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -301,8 +301,8 @@ class StorageClient(object): # ``StorageClient.from_nurl()``. _base_url: DecodedURL _swissnum: bytes - _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) - _clock: IReactorTime = field(eq=False) + _treq: Union[treq, StubTreq, HTTPClient] + _clock: IReactorTime @classmethod def from_nurl( From d700163aecda5ff23b772c561b5f9a1992b45f82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:14:29 -0500 Subject: [PATCH 884/916] Remove no-longer-relevant comment. --- src/allmydata/storage/http_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f0b45742c..cc26d4b37 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -312,8 +312,6 @@ class StorageClient(object): ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. - - ``persistent`` indicates whether to use persistent HTTP connections. """ assert nurl.fragment == "v=1" assert nurl.scheme == "pb" From 4aeb62b66c12e5d337d6ebeeb26cea8f3f3ff13d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:16:41 -0500 Subject: [PATCH 885/916] Use a constant. --- src/allmydata/client.py | 2 +- src/allmydata/storage_client.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index aa03015fc..1e28bb98b 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -826,7 +826,7 @@ class _Client(node.Node, pollmixin.PollMixin): if hasattr(self.tub.negotiationClass, "add_storage_server"): nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) self.storage_nurls = nurls - announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] + announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls] announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 140e29607..59d3406f1 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -80,6 +80,8 @@ from allmydata.storage.http_client import ( ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ) +ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs" + # who is responsible for de-duplication? # both? @@ -267,8 +269,7 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) - # TODO use constant for anonymous-storage-NURLs - if len(server["ann"].get("anonymous-storage-NURLs", [])) > 0: + if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: s = HTTPNativeStorageServer(server_id, server["ann"]) s.on_status_changed(lambda _: self._got_connection()) return s @@ -961,7 +962,7 @@ class HTTPNativeStorageServer(service.MultiService): ) = _parse_announcement(server_id, furl, announcement) # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 - nurl = DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]) + nurl = DecodedURL.from_text(announcement[ANONYMOUS_STORAGE_NURLS][0]) self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor) ) From 8e4ac6903298e8081daf4d1947c569d02111d160 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:21:31 -0500 Subject: [PATCH 886/916] Stop test mode when done. --- src/allmydata/storage/http_client.py | 5 +++++ src/allmydata/test/common_system.py | 1 + src/allmydata/test/test_storage_http.py | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cc26d4b37..fed66bb75 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -297,6 +297,11 @@ class StorageClient(object): """ cls.TEST_MODE_REGISTER_HTTP_POOL = callback + @classmethod + def stop_test_mode(cls): + """Stop testing mode.""" + cls.TEST_MODE_REGISTER_HTTP_POOL = None + # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use # ``StorageClient.from_nurl()``. _base_url: DecodedURL diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 90990a8ca..af86440cc 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -649,6 +649,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._http_client_pools.append) + self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 25c21e03f..87a6a2306 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -294,6 +294,7 @@ class CustomHTTPServerTests(SyncTestCase): StorageClient.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) + self.addCleanup(StorageClient.stop_test_mode) # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -382,6 +383,7 @@ class HttpTestFixture(Fixture): StorageClient.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) + self.addCleanup(StorageClient.stop_test_mode) self.clock = Clock() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in From fb52b4d302d6f717a4393a518ddbe8fb773e406c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:22:30 -0500 Subject: [PATCH 887/916] Delete some garbage. --- src/allmydata/test/common_system.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index af86440cc..0c7d7f747 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -810,8 +810,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): if which in feature_matrix.get((section, feature), {which}): config.setdefault(section, {})[feature] = value - #config.setdefault("node", {})["force_foolscap"] = force_foolscap - setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") From f3fc4268309316e9200f251df64b27a7bca5f33e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:36:14 -0500 Subject: [PATCH 888/916] Switch to [storage] force_foolscap. --- src/allmydata/client.py | 1 + src/allmydata/node.py | 3 +-- src/allmydata/test/common_system.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 1e28bb98b..1a158a1aa 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -104,6 +104,7 @@ _client_config = configutil.ValidConfiguration( "reserved_space", "storage_dir", "plugins", + "force_foolscap", ), "sftpd": ( "accounts.file", diff --git a/src/allmydata/node.py b/src/allmydata/node.py index f572cf7d9..8266fe3fb 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -64,7 +64,6 @@ def _common_valid_config(): "tcp", ), "node": ( - "force_foolscap", "log_gatherer.furl", "nickname", "reveal-ip-address", @@ -916,7 +915,7 @@ def create_main_tub(config, tub_options, # don't want to enable HTTP by default. # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3934 force_foolscap=config.get_config( - "node", "force_foolscap", default=True, boolean=True + "storage", "force_foolscap", default=True, boolean=True ), handler_overrides=handler_overrides, certFile=certfile, diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 0c7d7f747..d49e7831d 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -814,7 +814,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): sethelper = partial(setconf, config, which, "helper") setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) - setnode("force_foolscap", str(force_foolscap)) + setconf(config, which, "storage", "force_foolscap", str(force_foolscap)) tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) setnode("tub.port", tub_port_endpoint) From 2a5e8e59715ec647387f77b83733d9541886544b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 15:02:15 -0500 Subject: [PATCH 889/916] Better cleanup. --- src/allmydata/storage_client.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 59d3406f1..8e9ad3656 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -970,6 +970,7 @@ class HTTPNativeStorageServer(service.MultiService): self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None self._last_connect_time = None + self._connecting_deferred = None def get_permutation_seed(self): return self._permutation_seed @@ -1060,20 +1061,30 @@ class HTTPNativeStorageServer(service.MultiService): def stop_connecting(self): self._lc.stop() + if self._connecting_deferred is not None: + self._connecting_deferred.cancel() def try_to_connect(self): self._connect() def _connect(self): result = self._istorage_server.get_version() + + def remove_connecting_deferred(result): + self._connecting_deferred = None + return result + # Set a short timeout since we're relying on this for server liveness. - result.addTimeout(5, self._reactor) - result.addCallbacks( + self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth( + remove_connecting_deferred).addCallbacks( self._got_version, self._failed_to_connect ) def stopService(self): + if self._connecting_deferred is not None: + self._connecting_deferred.cancel() + result = service.MultiService.stopService(self) if self._lc.running: self._lc.stop() From a20943e10c7d1f4b30b383138f489e9c9dd1eb85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 09:33:01 -0500 Subject: [PATCH 890/916] As an experiment, see if this fixes failing CI. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index d49e7831d..8bc25aacf 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -649,7 +649,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._http_client_pools.append) - self.addCleanup(http_client.StorageClient.stop_test_mode) + #self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) From 9f5f287473d734932f348d77b89fb81838e5c3d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 09:57:39 -0500 Subject: [PATCH 891/916] Nope, not helpful. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 8bc25aacf..d49e7831d 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -649,7 +649,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._http_client_pools.append) - #self.addCleanup(http_client.StorageClient.stop_test_mode) + self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) From 2ab172ffca9c6faac1751709ce5db5d17e4e28db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 10:26:29 -0500 Subject: [PATCH 892/916] Try to set more aggressive timeouts when testing. --- src/allmydata/test/common_system.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index d49e7831d..e75021248 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -648,7 +648,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] - http_client.StorageClient.start_test_mode(self._http_client_pools.append) + http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool) self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() @@ -657,6 +657,23 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.sparent = service.MultiService() self.sparent.startService() + def _got_new_http_connection_pool(self, pool): + # Register the pool for shutdown later: + self._http_client_pools.append(pool) + # Disable retries: + pool.retryAutomatically = False + # Make a much more aggressive timeout for connections, we're connecting + # locally after all... and also make sure it's lower than the delay we + # add in tearDown, to prevent dirty reactor issues. + getConnection = pool.getConnection + + def getConnectionWithTimeout(*args, **kwargs): + d = getConnection(*args, **kwargs) + d.addTimeout(0.05, reactor) + return d + + pool.getConnection = getConnectionWithTimeout + def close_idle_http_connections(self): """Close all HTTP client connections that are just hanging around.""" return defer.gatherResults( @@ -668,7 +685,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 0.02, lambda: x)) + d.addBoth(lambda x: deferLater(reactor, 0.1, lambda: x)) return d def getdir(self, subdir): From 35317373474c5170d9a15b5b9cd895ceb7222391 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 10:36:11 -0500 Subject: [PATCH 893/916] Make timeouts less aggressive, CI machines are slow? --- src/allmydata/test/common_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index e75021248..8d3019935 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -669,7 +669,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def getConnectionWithTimeout(*args, **kwargs): d = getConnection(*args, **kwargs) - d.addTimeout(0.05, reactor) + d.addTimeout(1, reactor) return d pool.getConnection = getConnectionWithTimeout @@ -685,7 +685,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 0.1, lambda: x)) + d.addBoth(lambda x: deferLater(reactor, 2, lambda: x)) return d def getdir(self, subdir): From 097d918a240ba291ebd6b00108f071362eefcbd3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 13:37:50 -0500 Subject: [PATCH 894/916] Sigh --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index bacb40290..a9421c3e5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -179,6 +179,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From d182a2f1865002cc9a3167c1f585413ac6db4307 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Nov 2022 11:01:12 -0500 Subject: [PATCH 895/916] Add the delay to appropriate test. --- src/allmydata/test/test_storage_https.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index a9421c3e5..01431267f 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -144,6 +144,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_has_wrong_hash(self): """ @@ -179,10 +183,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 9b21f1da90a1b80414959822fec689040db75d40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Nov 2022 11:35:10 -0500 Subject: [PATCH 896/916] Increase how many statuses are stored. --- src/allmydata/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/history.py b/src/allmydata/history.py index b5cfb7318..06a22ab5d 100644 --- a/src/allmydata/history.py +++ b/src/allmydata/history.py @@ -20,7 +20,7 @@ class History(object): MAX_UPLOAD_STATUSES = 10 MAX_MAPUPDATE_STATUSES = 20 MAX_PUBLISH_STATUSES = 20 - MAX_RETRIEVE_STATUSES = 20 + MAX_RETRIEVE_STATUSES = 40 def __init__(self, stats_provider=None): self.stats_provider = stats_provider From 4c0c75a034568c621ca327b00e881075743254c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Nov 2022 13:56:54 -0500 Subject: [PATCH 897/916] Fix DelayedCall leak in tests. --- src/allmydata/storage/http_client.py | 38 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 8 ++++-- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4ed37f901..5b4ec9db8 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -26,7 +26,6 @@ from twisted.internet.interfaces import ( IDelayedCall, ) from twisted.internet.ssl import CertificateOptions -from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -141,7 +140,9 @@ class _LengthLimitedCollector: def limited_content( - response, max_length: int = 30 * 1024 * 1024, clock: IReactorTime = reactor + response, + clock: IReactorTime, + max_length: int = 30 * 1024 * 1024, ) -> Deferred[BinaryIO]: """ Like ``treq.content()``, but limit data read from the response to a set @@ -168,11 +169,10 @@ def limited_content( collector.f.seek(0) return collector.f - d.addCallback(done) - return d + return d.addCallback(done) -def _decode_cbor(response, schema: Schema): +def _decode_cbor(response, schema: Schema, clock: IReactorTime): """Given HTTP response, return decoded CBOR body.""" def got_content(f: BinaryIO): @@ -183,7 +183,7 @@ def _decode_cbor(response, schema: Schema): if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) if content_type == CBOR_MIME_TYPE: - return limited_content(response).addCallback(got_content) + return limited_content(response, clock).addCallback(got_content) else: raise ClientException(-1, "Server didn't send CBOR") else: @@ -439,7 +439,9 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/storage/v1/version") response = yield self._client.request("GET", url) - decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) + decoded_response = yield _decode_cbor( + response, _SCHEMAS["get_version"], self._client._clock + ) returnValue(decoded_response) @inlineCallbacks @@ -540,7 +542,7 @@ def read_share_chunk( raise ValueError("Server sent more than we asked for?!") # It might also send less than we asked for. That's (probably) OK, e.g. # if we went past the end of the file. - body = yield limited_content(response, supposed_length, client._clock) + body = yield limited_content(response, client._clock, supposed_length) body.seek(0, SEEK_END) actual_length = body.tell() if actual_length != supposed_length: @@ -627,7 +629,9 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield _decode_cbor(response, _SCHEMAS["allocate_buckets"]) + decoded_response = yield _decode_cbor( + response, _SCHEMAS["allocate_buckets"], self._client._clock + ) returnValue( ImmutableCreateResult( already_have=decoded_response["already-have"], @@ -703,7 +707,9 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield _decode_cbor(response, _SCHEMAS["immutable_write_share_chunk"]) + body = yield _decode_cbor( + response, _SCHEMAS["immutable_write_share_chunk"], self._client._clock + ) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) @@ -732,7 +738,9 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield _decode_cbor(response, _SCHEMAS["list_shares"]) + body = yield _decode_cbor( + response, _SCHEMAS["list_shares"], self._client._clock + ) returnValue(set(body)) else: raise ClientException(response.code) @@ -849,7 +857,9 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) + result = await _decode_cbor( + response, _SCHEMAS["mutable_read_test_write"], self._client._clock + ) return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: raise ClientException(response.code, (await response.content())) @@ -878,7 +888,9 @@ class StorageClientMutables: ) response = await self._client.request("GET", url) if response.code == http.OK: - return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"]) + return await _decode_cbor( + response, _SCHEMAS["mutable_list_shares"], self._client._clock + ) else: raise ClientException(response.code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index fa3532839..4f7174c06 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -371,7 +371,9 @@ class CustomHTTPServerTests(SyncTestCase): ) self.assertEqual( - result_of(limited_content(response, at_least_length)).read(), + result_of( + limited_content(response, self._http_server.clock, at_least_length) + ).read(), gen_bytes(length), ) @@ -390,7 +392,7 @@ class CustomHTTPServerTests(SyncTestCase): ) with self.assertRaises(ValueError): - result_of(limited_content(response, too_short)) + result_of(limited_content(response, self._http_server.clock, too_short)) def test_limited_content_silence_causes_timeout(self): """ @@ -404,7 +406,7 @@ class CustomHTTPServerTests(SyncTestCase): ) ) - body_deferred = limited_content(response, 4, self._http_server.clock) + body_deferred = limited_content(response, self._http_server.clock, 4) result = [] error = [] body_deferred.addCallbacks(result.append, error.append) From 8cfdae2ab4005943689a0713ba5bd8f3b0831d9b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Nov 2022 15:26:02 -0500 Subject: [PATCH 898/916] sigh --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 01431267f..062eb5b0e 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -183,6 +183,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 3a613aee704d0d4231f70e82d46bcaed84960692 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Nov 2022 12:24:50 -0500 Subject: [PATCH 899/916] Try a different approach to timeouts: dynamic, instead of hardcoded. --- src/allmydata/test/common_system.py | 30 +++++++++++++++++++++++- src/allmydata/test/test_storage_https.py | 21 ++++------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 8d3019935..a6b239005 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -20,6 +20,7 @@ from foolscap.api import flushEventualQueue from allmydata import client from allmydata.introducer.server import create_introducer from allmydata.util import fileutil, log, pollmixin +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage import http_client from allmydata.storage_client import ( NativeStorageServer, @@ -639,6 +640,33 @@ def _render_section_values(values): )) +@async_to_deferred +async def spin_until_cleanup_done(value=None, timeout=10): + """ + At the end of the test, spin until either a timeout is hit, or the reactor + has no more DelayedCalls. + + Make sure to register during setUp. + """ + def num_fds(): + if hasattr(reactor, "handles"): + # IOCP! + return len(reactor.handles) + else: + # Normal reactor + return len([r for r in reactor.getReaders() + if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker")] + ) + len(reactor.getWriters()) + + for i in range(timeout * 1000): + # There's a single DelayedCall for AsynchronousDeferredRunTest's + # timeout... + if (len(reactor.getDelayedCalls()) < 2 and num_fds() == 0): + break + await deferLater(reactor, 0.001) + return value + + class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # If set to True, use Foolscap for storage protocol. If set to False, HTTP @@ -685,7 +713,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 2, lambda: x)) + d.addBoth(spin_until_cleanup_done) return d def getdir(self, subdir): diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 062eb5b0e..284c8cda8 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -12,7 +12,6 @@ from cryptography import x509 from twisted.internet.endpoints import serverFromString from twisted.internet import reactor -from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived @@ -30,6 +29,7 @@ from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper from ..util.deferredutil import async_to_deferred +from .common_system import spin_until_cleanup_done class HTTPSNurlTests(SyncTestCase): @@ -87,6 +87,10 @@ class PinningHTTPSValidation(AsyncTestCase): self.addCleanup(self._port_assigner.tearDown) return AsyncTestCase.setUp(self) + def tearDown(self): + AsyncTestCase.tearDown(self) + return spin_until_cleanup_done() + @asynccontextmanager async def listen(self, private_key_path: FilePath, cert_path: FilePath): """ @@ -107,9 +111,6 @@ class PinningHTTPSValidation(AsyncTestCase): yield f"https://127.0.0.1:{listening_port.getHost().port}/" finally: await listening_port.stopListening() - # Make sure all server connections are closed :( No idea why this - # is necessary when it's not for IStorageServer HTTPS tests. - await deferLater(reactor, 0.01) def request(self, url: str, expected_certificate: x509.Certificate): """ @@ -144,10 +145,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_has_wrong_hash(self): """ @@ -183,10 +180,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ @@ -206,10 +199,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.001) - # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that # so I don't know how to test that! See From c80469b50bd6f97d98ab22b48ac4b6481020a1df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Nov 2022 11:55:56 -0500 Subject: [PATCH 900/916] Handle the Windows waker too. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index a6b239005..ca2904b53 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -655,7 +655,7 @@ async def spin_until_cleanup_done(value=None, timeout=10): else: # Normal reactor return len([r for r in reactor.getReaders() - if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker")] + if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker", "_SocketWaker")] ) + len(reactor.getWriters()) for i in range(timeout * 1000): From 62400d29b3819994e6c777f77cd0ec7e3ecb5def Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:36:53 -0500 Subject: [PATCH 901/916] Seems like Ubuntu 22.04 has issues with Tor at the moment --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327014ca..ad055da2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,7 @@ jobs: matrix: os: - windows-latest - - ubuntu-latest + - ubuntu-20.04 python-version: - 3.7 - 3.9 From 4fd92a915bb312a2e2bf4f185112570b8d32d393 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:43:45 -0500 Subject: [PATCH 902/916] Install tor on any ubuntu version. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad055da2f..4e5c9a757 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: steps: - name: Install Tor [Ubuntu] - if: matrix.os == 'ubuntu-latest' + if: ${{ contains(matrix.os, 'ubuntu') }} run: sudo apt install tor # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. From 7f1d7d4f46847ea83a78d85dea649b45d78583dd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:53:07 -0500 Subject: [PATCH 903/916] Better explanation. --- src/allmydata/test/common_system.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index ca2904b53..297046cc5 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -643,10 +643,16 @@ def _render_section_values(values): @async_to_deferred async def spin_until_cleanup_done(value=None, timeout=10): """ - At the end of the test, spin until either a timeout is hit, or the reactor - has no more DelayedCalls. + At the end of the test, spin until the reactor has no more DelayedCalls + and file descriptors (or equivalents) registered. This prevents dirty + reactor errors, while also not hard-coding a fixed amount of time, so it + can finish faster on faster computers. - Make sure to register during setUp. + There is also a timeout: if it takes more than 10 seconds (by default) for + the remaining reactor state to clean itself up, the presumption is that it + will never get cleaned up and the spinning stops. + + Make sure to run as last thing in tearDown. """ def num_fds(): if hasattr(reactor, "handles"): From 6c3e9e670de208e7b5d2dc37d192de2c3d464e80 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:53:11 -0500 Subject: [PATCH 904/916] Link to issue. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e5c9a757..99ac28926 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,8 @@ jobs: matrix: os: - windows-latest + # 22.04 has some issue with Tor at the moment: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - ubuntu-20.04 python-version: - 3.7 From 562111012e4418707ec141c8f488d36ea61325ae Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:18:05 -0600 Subject: [PATCH 905/916] Give GITHUB_TOKEN just enough permissions to run the workflow --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327014ca..588e71747 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,16 @@ on: - "master" pull_request: +# At the start of each workflow run, GitHub creates a unique +# GITHUB_TOKEN secret to use in the workflow. It is a good idea for +# this GITHUB_TOKEN to have the minimum of permissions. See: +# +# - https://docs.github.com/en/actions/security-guides/automatic-token-authentication +# - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +# +permissions: + contents: read + # Control to what degree jobs in this workflow will run concurrently with # other instances of themselves. # From 9bd384ac2db0199c446ebcffefffb01cccf1e2de Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:18:44 -0600 Subject: [PATCH 906/916] Add news fragment --- newsfragments/3944.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3944.minor diff --git a/newsfragments/3944.minor b/newsfragments/3944.minor new file mode 100644 index 000000000..e69de29bb From 5e6189e1159432e30b55340a9230d1ea317971ce Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:25:19 -0600 Subject: [PATCH 907/916] Use newer version of actions/setup-python --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 588e71747..bd757fe08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -208,7 +208,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -268,7 +268,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 23d8d1cb01682a13ad788bdf832513c1cddc63ed Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:28:57 -0600 Subject: [PATCH 908/916] Use action/setup-python@v4's caching feature --- .github/workflows/ci.yml | 48 +++------------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd757fe08..6c608e888 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,25 +76,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - # To use pip caching with GitHub Actions in an OS-independent - # manner, we need `pip cache dir` command, which became - # available since pip v20.1+. At the time of writing this, - # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and - # windows-latest, and pip v20.3.1 for macos-latest. - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - # See https://github.com/actions/cache - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' # caching pip dependencies - name: Install Python packages run: | @@ -211,19 +193,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' # caching pip dependencies - name: Install Python packages run: | @@ -271,19 +241,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' # caching pip dependencies - name: Install Python packages run: | From 15881da348dfa9c9f92836f59175e3582fdab8cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:37:46 -0600 Subject: [PATCH 909/916] Use newer version of actions/checkout --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c608e888..4447e539c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -185,7 +185,7 @@ jobs: args: install tor - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -233,7 +233,7 @@ jobs: steps: - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 From 26d30979c0fc3345c78846aaf37db1a7f83610eb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:38:48 -0600 Subject: [PATCH 910/916] Use newer version of actions/upload-artifact --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4447e539c..64a60bd04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,13 +90,13 @@ jobs: run: python -m tox - name: Upload eliot.log - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: eliot.log path: eliot.log - name: Upload trial log - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: test.log path: _trial_temp/test.log @@ -212,7 +212,7 @@ jobs: run: tox -e integration - name: Upload eliot.log in case of failure - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 if: failure() with: name: integration.eliot.json @@ -259,7 +259,7 @@ jobs: run: dist/Tahoe-LAFS/tahoe --version - name: Upload PyInstaller package - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Tahoe-LAFS-${{ matrix.os }}-Python-${{ matrix.python-version }} path: dist/Tahoe-LAFS-*-*.* From 7715972429c34d4c6a684f184ab5f4ba1613df16 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:40:19 -0600 Subject: [PATCH 911/916] Use newer version of crazy-max/ghaction-chocolatey --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64a60bd04..169e981ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,7 @@ jobs: - name: Install Tor [Windows] if: matrix.os == 'windows-latest' - uses: crazy-max/ghaction-chocolatey@v1 + uses: crazy-max/ghaction-chocolatey@v2 with: args: install tor From 2ab8e3e8d20087521dc4aa7ffb358e3f65a7a6aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:02:56 -0500 Subject: [PATCH 912/916] Cancel timeout on failures too. --- src/allmydata/storage/http_client.py | 7 ++++++- src/allmydata/test/test_storage_http.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5b4ec9db8..73fba9888 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -169,7 +169,12 @@ def limited_content( collector.f.seek(0) return collector.f - return d.addCallback(done) + def failed(f): + if timeout.active(): + timeout.cancel() + return f + + return d.addCallbacks(done, failed) def _decode_cbor(response, schema: Schema, clock: IReactorTime): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4f7174c06..8dbe18545 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -280,6 +280,14 @@ class TestApp(object): self.clock.callLater(59 + 59, request.write, b"c") return Deferred() + @_authorized_route(_app, set(), "/die_unfinished", methods=["GET"]) + def die(self, request, authorization): + """ + Dies half-way. + """ + request.transport.loseConnection() + return Deferred() + def result_of(d): """ @@ -423,6 +431,22 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(CancelledError): error[0].raiseException() + def test_limited_content_cancels_timeout_on_failed_response(self): + """ + If the response fails somehow, the timeout is still cancelled. + """ + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/die", + ) + ) + + d = limited_content(response, self._http_server.clock, 4) + with self.assertRaises(ValueError): + result_of(d) + self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0) + class HttpTestFixture(Fixture): """ From 38d7430c570fd3ff9b2b3ea720706d6d3198fbfa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:03:42 -0500 Subject: [PATCH 913/916] Simplify. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 73fba9888..5abc44bdd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -382,7 +382,7 @@ class StorageClient(object): write_enabler_secret=None, headers=None, message_to_serialize=None, - timeout: Union[int, float] = 60, + timeout: float = 60, **kwargs, ): """ From 0f4dc9129538dbbe8b88073c3a5047462f4209a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:12:08 -0500 Subject: [PATCH 914/916] Refactor so internal attributes needn't leak. --- src/allmydata/storage/http_client.py | 63 +++++++++++++--------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5abc44bdd..79bf061c9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -177,26 +177,6 @@ def limited_content( return d.addCallbacks(done, failed) -def _decode_cbor(response, schema: Schema, clock: IReactorTime): - """Given HTTP response, return decoded CBOR body.""" - - def got_content(f: BinaryIO): - data = f.read() - schema.validate_cbor(data) - return loads(data) - - if response.code > 199 and response.code < 300: - content_type = get_content_type(response.headers) - if content_type == CBOR_MIME_TYPE: - return limited_content(response, clock).addCallback(got_content) - else: - raise ClientException(-1, "Server didn't send CBOR") - else: - return treq.content(response).addCallback( - lambda data: fail(ClientException(response.code, response.phrase, data)) - ) - - @define class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" @@ -428,6 +408,25 @@ class StorageClient(object): method, url, headers=headers, timeout=timeout, **kwargs ) + def decode_cbor(self, response, schema: Schema): + """Given HTTP response, return decoded CBOR body.""" + + def got_content(f: BinaryIO): + data = f.read() + schema.validate_cbor(data) + return loads(data) + + if response.code > 199 and response.code < 300: + content_type = get_content_type(response.headers) + if content_type == CBOR_MIME_TYPE: + return limited_content(response, self._clock).addCallback(got_content) + else: + raise ClientException(-1, "Server didn't send CBOR") + else: + return treq.content(response).addCallback( + lambda data: fail(ClientException(response.code, response.phrase, data)) + ) + @define(hash=True) class StorageClientGeneral(object): @@ -444,8 +443,8 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/storage/v1/version") response = yield self._client.request("GET", url) - decoded_response = yield _decode_cbor( - response, _SCHEMAS["get_version"], self._client._clock + decoded_response = yield self._client.decode_cbor( + response, _SCHEMAS["get_version"] ) returnValue(decoded_response) @@ -634,8 +633,8 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield _decode_cbor( - response, _SCHEMAS["allocate_buckets"], self._client._clock + decoded_response = yield self._client.decode_cbor( + response, _SCHEMAS["allocate_buckets"] ) returnValue( ImmutableCreateResult( @@ -712,8 +711,8 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield _decode_cbor( - response, _SCHEMAS["immutable_write_share_chunk"], self._client._clock + body = yield self._client.decode_cbor( + response, _SCHEMAS["immutable_write_share_chunk"] ) remaining = RangeMap() for chunk in body["required"]: @@ -743,9 +742,7 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield _decode_cbor( - response, _SCHEMAS["list_shares"], self._client._clock - ) + body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"]) returnValue(set(body)) else: raise ClientException(response.code) @@ -862,8 +859,8 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - result = await _decode_cbor( - response, _SCHEMAS["mutable_read_test_write"], self._client._clock + result = await self._client.decode_cbor( + response, _SCHEMAS["mutable_read_test_write"] ) return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: @@ -893,8 +890,8 @@ class StorageClientMutables: ) response = await self._client.request("GET", url) if response.code == http.OK: - return await _decode_cbor( - response, _SCHEMAS["mutable_list_shares"], self._client._clock + return await self._client.decode_cbor( + response, _SCHEMAS["mutable_list_shares"] ) else: raise ClientException(response.code) From 3ba166c2cb939a58fdb16dad06cd0dbd1ad39961 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:20:12 -0500 Subject: [PATCH 915/916] A bit more robust code. --- src/allmydata/test/common_system.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 297046cc5..01966824a 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -659,10 +659,11 @@ async def spin_until_cleanup_done(value=None, timeout=10): # IOCP! return len(reactor.handles) else: - # Normal reactor - return len([r for r in reactor.getReaders() - if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker", "_SocketWaker")] - ) + len(reactor.getWriters()) + # Normal reactor; having internal readers still registered is fine, + # that's not our code. + return len( + set(reactor.getReaders()) - set(reactor._internalReaders) + ) + len(reactor.getWriters()) for i in range(timeout * 1000): # There's a single DelayedCall for AsynchronousDeferredRunTest's From aa80c9ef4748cf10e3b448b298df8b589c35cafd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:21:59 -0500 Subject: [PATCH 916/916] Be more robust. --- src/allmydata/test/test_storage_https.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 284c8cda8..a11b0eed5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -12,6 +12,7 @@ from cryptography import x509 from twisted.internet.endpoints import serverFromString from twisted.internet import reactor +from twisted.internet.defer import maybeDeferred from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived @@ -88,8 +89,8 @@ class PinningHTTPSValidation(AsyncTestCase): return AsyncTestCase.setUp(self) def tearDown(self): - AsyncTestCase.tearDown(self) - return spin_until_cleanup_done() + d = maybeDeferred(AsyncTestCase.tearDown, self) + return d.addCallback(lambda _: spin_until_cleanup_done()) @asynccontextmanager async def listen(self, private_key_path: FilePath, cert_path: FilePath):