tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

Merge branch '2857-deprecations'

This commit is contained in:
Brian Warner 2017-07-23 12:00:03 -05:00
parent 74e7ef4b98 ad92b2b9b2
commit 1d1653d361
6 changed files with 99 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
setup.py
View File

@ -285,6 +285,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
"pytest",
"pytest-twisted",
"hypothesis >= 3.6.1",
"treq",
],
"tor": [
"foolscap[tor] >= 0.12.5",

12
src/allmydata/test/common_web.py
View File

@ -1,7 +1,9 @@
import re
import treq
from twisted.internet import defer
from twisted.web import client
from twisted.web.error import Error
from nevow.testutil import FakeRequest
from nevow import inevow, context
@ -81,3 +83,13 @@ class HTTPClientHEADFactory(client.HTTPClientFactory):
class HTTPClientGETFactory(client.HTTPClientFactory):
protocol = MyGetter
@defer.inlineCallbacks
def do_http(method, url, **kwargs):
response = yield treq.request(method, url, persistent=False, **kwargs)
body = yield treq.content(response)
# TODO: replace this with response.fail_for_status when
# https://github.com/twisted/treq/pull/159 has landed
if 400 <= response.code < 600:
raise Error(response.code, response=body)
defer.returnValue(body)

58
src/allmydata/test/test_deepcheck.py
View File

@ -2,6 +2,7 @@
import os, json, urllib
from twisted.trial import unittest
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks, returnValue
from allmydata.immutable import upload
from allmydata.mutable.common import UnrecoverableFileError
from allmydata.mutable.publish import MutableData
@ -11,11 +12,11 @@ from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
IDeepCheckResults, IDeepCheckAndRepairResults
from allmydata.monitor import Monitor, OperationCancelledError
from allmydata.uri import LiteralFileURI
from twisted.web.client import getPage
from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
ShouldFailMixin
from .common_util import StallMixin, run_cli
from .common_web import do_http
from allmydata.test.no_network import GridTestMixin
from .cli.common import CLITestMixin
@ -148,54 +149,47 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
le.args = tuple(le.args + (unit,))
raise
@inlineCallbacks
def web(self, n, method="GET", **kwargs):
# returns (data, url)
url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
+ "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
d = getPage(url, method=method)
d.addCallback(lambda data: (data,url))
return d
data = yield do_http(method, url, browser_like_redirects=True)
returnValue((data,url))
def wait_for_operation(self, ignored, ophandle):
@inlineCallbacks
def wait_for_operation(self, ophandle):
url = self.client_baseurls[0] + "operations/" + ophandle
url += "?t=status&output=JSON"
d = getPage(url)
def _got(res):
try:
data = json.loads(res)
except ValueError:
self.fail("%s: not JSON: '%s'" % (url, res))
if not data["finished"]:
d = self.stall(delay=1.0)
d.addCallback(self.wait_for_operation, ophandle)
return d
return data
d.addCallback(_got)
return d
while True:
body = yield do_http("get", url)
data = json.loads(body)
if data["finished"]:
break
yield self.stall(delay=0.1)
returnValue(data)
def get_operation_results(self, ignored, ophandle, output=None):
@inlineCallbacks
def get_operation_results(self, ophandle, output=None):
url = self.client_baseurls[0] + "operations/" + ophandle
url += "?t=status"
if output:
url += "&output=" + output
d = getPage(url)
def _got(res):
body = yield do_http("get", url)
if output and output.lower() == "json":
try:
return json.loads(res)
except ValueError:
self.fail("%s: not JSON: '%s'" % (url, res))
return res
d.addCallback(_got)
return d
data = json.loads(body)
else:
data = body
returnValue(data)
@inlineCallbacks
def slow_web(self, n, output=None, **kwargs):
# use ophandle=
handle = base32.b2a(os.urandom(4))
d = self.web(n, "POST", ophandle=handle, **kwargs)
d.addCallback(self.wait_for_operation, handle)
d.addCallback(self.get_operation_results, handle, output=output)
return d
yield self.web(n, "POST", ophandle=handle, **kwargs)
yield self.wait_for_operation(handle)
data = yield self.get_operation_results(handle, output=output)
returnValue(data)
class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):

69
src/allmydata/test/test_system.py
View File

@ -28,10 +28,9 @@ from allmydata.mutable.publish import MutableData
from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue
from twisted.python.failure import Failure
from twisted.web.client import getPage
from twisted.web.error import Error
from .common import TEST_RSA_KEY_SIZE
from .common_web import do_http, Error
# TODO: move this to common or common_util
from allmydata.test.test_runner import RunBinTahoeMixin
@ -1326,7 +1325,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d
def _test_introweb(self, res):
d = getPage(self.introweb_url, method="GET", followRedirect=True)
d = do_http("get", self.introweb_url)
def _check(res):
try:
self.failUnless("%s: %s" % (allmydata.__appname__, allmydata.__version__) in res)
@ -1360,11 +1359,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
raise
d.addCallback(_check)
# make sure it serves the CSS too
d.addCallback(lambda res:
getPage(self.introweb_url+"tahoe.css", method="GET"))
d.addCallback(lambda res:
getPage(self.introweb_url + "?t=json",
method="GET", followRedirect=True))
d.addCallback(lambda res: do_http("get", self.introweb_url+"tahoe.css"))
d.addCallback(lambda res: do_http("get", self.introweb_url + "?t=json"))
def _check_json(res):
data = json.loads(res)
try:
@ -1607,14 +1603,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d
def PUT(self, urlpath, data):
url = self.webish_url + urlpath
return getPage(url, method="PUT", postdata=data)
return do_http("put", self.webish_url + urlpath, data=data)
def GET(self, urlpath, followRedirect=False):
url = self.webish_url + urlpath
return getPage(url, method="GET", followRedirect=followRedirect)
def GET(self, urlpath):
return do_http("get", self.webish_url + urlpath)
def POST(self, urlpath, followRedirect=False, use_helper=False, **fields):
def POST(self, urlpath, use_helper=False, **fields):
sepbase = "boogabooga"
sep = "--" + sepbase
form = []
@ -1639,21 +1633,18 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
if fields:
body = "\r\n".join(form) + "\r\n"
headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
return self.POST2(urlpath, body, headers, followRedirect, use_helper)
return self.POST2(urlpath, body, headers, use_helper)
def POST2(self, urlpath, body="", headers={}, followRedirect=False,
use_helper=False):
def POST2(self, urlpath, body="", headers={}, use_helper=False):
if use_helper:
url = self.helper_webish_url + urlpath
else:
url = self.webish_url + urlpath
return getPage(url, method="POST", postdata=body, headers=headers,
followRedirect=followRedirect)
return do_http("post", url, data=body, headers=headers)
def _test_web(self, res):
base = self.webish_url
public = "uri/" + self._root_directory_uri
d = getPage(base)
d = self.GET("")
def _got_welcome(page):
html = page.replace('\n', ' ')
connected_re = r'Connected to <span>%d</span>\s*of <span>%d</span> known storage servers' % (self.numclients, self.numclients)
@ -1669,23 +1660,22 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(self.log, "done with _got_welcome")
# get the welcome page from the node that uses the helper too
d.addCallback(lambda res: getPage(self.helper_webish_url))
d.addCallback(lambda res: do_http("get", self.helper_webish_url))
def _got_welcome_helper(page):
html = page.replace('\n', ' ')
self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), page)
self.failUnlessIn("Not running helper", page)
d.addCallback(_got_welcome_helper)
d.addCallback(lambda res: getPage(base + public))
d.addCallback(lambda res: getPage(base + public + "/subdir1"))
d.addCallback(lambda res: self.GET(public))
d.addCallback(lambda res: self.GET(public + "/subdir1"))
def _got_subdir1(page):
# there ought to be an href for our file
self.failUnlessIn('<td align="right">%d</td>' % len(self.data), page)
self.failUnless(">mydata567</a>" in page)
d.addCallback(_got_subdir1)
d.addCallback(self.log, "done with _got_subdir1")
d.addCallback(lambda res:
getPage(base + public + "/subdir1/mydata567"))
d.addCallback(lambda res: self.GET(public + "/subdir1/mydata567"))
def _got_data(page):
self.failUnlessEqual(page, self.data)
d.addCallback(_got_data)
@ -1693,8 +1683,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# download from a URI embedded in a URL
d.addCallback(self.log, "_get_from_uri")
def _get_from_uri(res):
return getPage(base + "uri/%s?filename=%s"
% (self.uri, "mydata567"))
return self.GET("uri/%s?filename=%s" % (self.uri, "mydata567"))
d.addCallback(_get_from_uri)
def _got_from_uri(page):
self.failUnlessEqual(page, self.data)
@ -1703,18 +1692,18 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# download from a URI embedded in a URL, second form
d.addCallback(self.log, "_get_from_uri2")
def _get_from_uri2(res):
return getPage(base + "uri?uri=%s" % (self.uri,))
return self.GET("uri?uri=%s" % (self.uri,))
d.addCallback(_get_from_uri2)
d.addCallback(_got_from_uri)
# download from a bogus URI, make sure we get a reasonable error
d.addCallback(self.log, "_get_from_bogus_uri", level=log.UNUSUAL)
@defer.inlineCallbacks
def _get_from_bogus_uri(res):
d1 = getPage(base + "uri/%s?filename=%s"
d1 = self.GET("uri/%s?filename=%s"
% (self.mangle_uri(self.uri), "mydata567"))
d1.addBoth(self.shouldFail, Error, "downloading bogus URI",
"410")
return d1
e = yield self.assertFailure(d1, Error)
self.assertEquals(e.status, "410")
d.addCallback(_get_from_bogus_uri)
d.addCallback(self.log, "_got_from_bogus_uri", level=log.UNUSUAL)
@ -1751,7 +1740,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
file=("foo.txt", "data2" * 10000)))
# check that the status page exists
d.addCallback(lambda res: self.GET("status", followRedirect=True))
d.addCallback(lambda res: self.GET("status"))
def _got_status(res):
# find an interesting upload and download to look at. LIT files
# are not interesting.
@ -1790,8 +1779,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(_got_retrieve)
# check that the helper status page exists
d.addCallback(lambda res:
self.GET("helper_status", followRedirect=True))
d.addCallback(lambda res: self.GET("helper_status"))
def _got_helper_status(res):
self.failUnless("Bytes Fetched:" in res)
# touch a couple of files in the helper's working directory to
@ -1811,8 +1799,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
os.utime(encfile, (now, then))
d.addCallback(_got_helper_status)
# and that the json form exists
d.addCallback(lambda res:
self.GET("helper_status?t=json", followRedirect=True))
d.addCallback(lambda res: self.GET("helper_status?t=json"))
def _got_helper_status_json(res):
data = json.loads(res)
self.failUnlessEqual(data["chk_upload_helper.upload_need_upload"],
@ -1829,14 +1816,16 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# and check that client[3] (which uses a helper but does not run one
# itself) doesn't explode when you ask for its status
d.addCallback(lambda res: getPage(self.helper_webish_url + "status/"))
d.addCallback(lambda res: do_http("get",
self.helper_webish_url + "status/"))
def _got_non_helper_status(res):
self.failUnlessIn("Recent and Active Operations", res)
d.addCallback(_got_non_helper_status)
# or for helper status with t=json
d.addCallback(lambda res:
getPage(self.helper_webish_url + "helper_status?t=json"))
do_http("get",
self.helper_webish_url + "helper_status?t=json"))
def _got_non_helper_status_json(res):
data = json.loads(res)
self.failUnlessEqual(data, {})

32
src/allmydata/test/web/test_introducer.py
View File

@ -2,10 +2,10 @@ import os.path
from twisted.trial import unittest
from foolscap.api import fireEventually, flushEventualQueue
from allmydata.util import fileutil
from twisted.internet import defer, reactor
from twisted.internet import defer
from allmydata.introducer import IntroducerNode
from .common import FAVICON_MARKUP
from ..common_web import HTTPClientGETFactory
from ..common_web import do_http
class IntroducerWeb(unittest.TestCase):
def setUp(self):
@ -18,6 +18,7 @@ class IntroducerWeb(unittest.TestCase):
d.addCallback(flushEventualQueue)
return d
@defer.inlineCallbacks
def test_welcome(self):
basedir = "web.IntroducerWeb.test_welcome"
os.mkdir(basedir)
@ -29,31 +30,12 @@ class IntroducerWeb(unittest.TestCase):
self.node = IntroducerNode(basedir)
self.ws = self.node.getServiceNamed("webish")
d = fireEventually(None)
d.addCallback(lambda ign: self.node.startService())
yield fireEventually(None)
self.node.startService()
d.addCallback(lambda ign: self.GET("/"))
def _check(res):
url = "http://localhost:%d/" % self.ws.getPortnum()
res = yield do_http("get", url)
self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
self.failUnlessIn(FAVICON_MARKUP, res)
self.failUnlessIn('Page rendered at', res)
self.failUnlessIn('Tahoe-LAFS code imported from:', res)
d.addCallback(_check)
return d
def GET(self, urlpath, followRedirect=False, return_response=False,
**kwargs):
# if return_response=True, this fires with (data, statuscode,
# respheaders) instead of just data.
assert not isinstance(urlpath, unicode)
url = self.ws.getURL().rstrip('/') + urlpath
factory = HTTPClientGETFactory(url, method="GET",
followRedirect=followRedirect, **kwargs)
reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
d = factory.deferred
def _got_data(data):
return (data, factory.status, factory.response_headers)
if return_response:
d.addCallback(_got_data)
return factory.deferred

34
src/allmydata/test/web/test_web.py
View File

@ -1,9 +1,11 @@
import os.path, re, urllib, time, cgi
import json
import treq
from twisted.application import service
from twisted.trial import unittest
from twisted.internet import defer, reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import Clock
from twisted.web import client, error, http
from twisted.python import failure, log
@ -27,7 +29,7 @@ from ..common import FakeCHKFileNode, FakeMutableFileNode, \
from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
from allmydata.mutable import servermap, publish, retrieve
from .. import common_util as testutil
from ..common_web import HTTPClientGETFactory, HTTPClientHEADFactory
from ..common_web import HTTPClientGETFactory, do_http, Error
from allmydata.client import Client, SecretHolder
from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP
# create a fake uploader/downloader, and a couple of fake dirnodes, then
@ -489,25 +491,25 @@ class WebMixin(testutil.TimezoneMixin):
d.addCallback(_got_data)
return factory.deferred
def HEAD(self, urlpath, return_response=False, **kwargs):
# this requires some surgery, because twisted.web.client doesn't want
# to give us back the response headers.
factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
reactor.connectTCP("localhost", self.webish_port, factory)
d = factory.deferred
def _got_data(data):
return (data, factory.status, factory.response_headers)
if return_response:
d.addCallback(_got_data)
return factory.deferred
def PUT(self, urlpath, data, **kwargs):
@inlineCallbacks
def HEAD(self, urlpath, return_response=False, headers={}):
url = self.webish_url + urlpath
return client.getPage(url, method="PUT", postdata=data, **kwargs)
response = yield treq.request("head", url, persistent=False,
headers=headers)
if 400 <= response.code < 600:
raise Error(response.code, response="")
resp_headers = {}
for (key, values) in response.headers.getAllRawHeaders():
resp_headers[key.lower()] = values
returnValue( ("", response.code, resp_headers) )
def PUT(self, urlpath, data, headers={}):
url = self.webish_url + urlpath
return do_http("put", url, data=data, headers=headers)
def DELETE(self, urlpath):
url = self.webish_url + urlpath
return client.getPage(url, method="DELETE")
return do_http("delete", url)
def POST(self, urlpath, followRedirect=False, **fields):
sepbase = "boogabooga"