Merge branch '2857-deprecations'

setup.py

@@ -285,6 +285,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "pytest",
               "pytest-twisted",
               "hypothesis >= 3.6.1",
+              "treq",
           ],
           "tor": [
               "foolscap[tor] >= 0.12.5",

src/allmydata/test/common_web.py

@@ -1,7 +1,9 @@
 
 import re
+import treq
 from twisted.internet import defer
 from twisted.web import client
+from twisted.web.error import Error
 from nevow.testutil import FakeRequest
 from nevow import inevow, context
 
@@ -81,3 +83,13 @@ class HTTPClientHEADFactory(client.HTTPClientFactory):
 
 class HTTPClientGETFactory(client.HTTPClientFactory):
     protocol = MyGetter
+
+@defer.inlineCallbacks
+def do_http(method, url, **kwargs):
+    response = yield treq.request(method, url, persistent=False, **kwargs)
+    body = yield treq.content(response)
+    # TODO: replace this with response.fail_for_status when
+    # https://github.com/twisted/treq/pull/159 has landed
+    if 400 <= response.code < 600:
+        raise Error(response.code, response=body)
+    defer.returnValue(body)

src/allmydata/test/test_deepcheck.py

@@ -2,6 +2,7 @@
 import os, json, urllib
 from twisted.trial import unittest
 from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, returnValue
 from allmydata.immutable import upload
 from allmydata.mutable.common import UnrecoverableFileError
 from allmydata.mutable.publish import MutableData
@@ -11,11 +12,11 @@ from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
      IDeepCheckResults, IDeepCheckAndRepairResults
 from allmydata.monitor import Monitor, OperationCancelledError
 from allmydata.uri import LiteralFileURI
-from twisted.web.client import getPage
 
 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
      ShouldFailMixin
 from .common_util import StallMixin, run_cli
+from .common_web import do_http
 from allmydata.test.no_network import GridTestMixin
 from .cli.common import CLITestMixin
 
@@ -148,54 +149,47 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
                 le.args = tuple(le.args + (unit,))
                 raise
 
+    @inlineCallbacks
     def web(self, n, method="GET", **kwargs):
         # returns (data, url)
         url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
-        d = getPage(url, method=method)
+        data = yield do_http(method, url, browser_like_redirects=True)
-        d.addCallback(lambda data: (data,url))
+        returnValue((data,url))
-        return d
 
-    def wait_for_operation(self, ignored, ophandle):
+    @inlineCallbacks
+    def wait_for_operation(self, ophandle):
         url = self.client_baseurls[0] + "operations/" + ophandle
         url += "?t=status&output=JSON"
-        d = getPage(url)
+        while True:
-        def _got(res):
+            body = yield do_http("get", url)
-            try:
+            data = json.loads(body)
-                data = json.loads(res)
+            if data["finished"]:
-            except ValueError:
+                break
-                self.fail("%s: not JSON: '%s'" % (url, res))
+            yield self.stall(delay=0.1)
-            if not data["finished"]:
+        returnValue(data)
-                d = self.stall(delay=1.0)
-                d.addCallback(self.wait_for_operation, ophandle)
-                return d
-            return data
-        d.addCallback(_got)
-        return d
 
-    def get_operation_results(self, ignored, ophandle, output=None):
+    @inlineCallbacks
+    def get_operation_results(self, ophandle, output=None):
         url = self.client_baseurls[0] + "operations/" + ophandle
         url += "?t=status"
         if output:
             url += "&output=" + output
-        d = getPage(url)
+        body = yield do_http("get", url)
-        def _got(res):
+        if output and output.lower() == "json":
-            if output and output.lower() == "json":
+            data = json.loads(body)
-                try:
+        else:
-                    return json.loads(res)
+            data = body
-                except ValueError:
+        returnValue(data)
-                    self.fail("%s: not JSON: '%s'" % (url, res))
-            return res
-        d.addCallback(_got)
-        return d
 
+    @inlineCallbacks
     def slow_web(self, n, output=None, **kwargs):
         # use ophandle=
         handle = base32.b2a(os.urandom(4))
-        d = self.web(n, "POST", ophandle=handle, **kwargs)
+        yield self.web(n, "POST", ophandle=handle, **kwargs)
-        d.addCallback(self.wait_for_operation, handle)
+        yield self.wait_for_operation(handle)
-        d.addCallback(self.get_operation_results, handle, output=output)
+        data = yield self.get_operation_results(handle, output=output)
-        return d
+        returnValue(data)
 
 
 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):

src/allmydata/test/test_system.py

@@ -28,10 +28,9 @@ from allmydata.mutable.publish import MutableData
 
 from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue
 from twisted.python.failure import Failure
-from twisted.web.client import getPage
-from twisted.web.error import Error
 
 from .common import TEST_RSA_KEY_SIZE
+from .common_web import do_http, Error
 
 # TODO: move this to common or common_util
 from allmydata.test.test_runner import RunBinTahoeMixin
@@ -1326,7 +1325,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return d
 
     def _test_introweb(self, res):
-        d = getPage(self.introweb_url, method="GET", followRedirect=True)
+        d = do_http("get", self.introweb_url)
         def _check(res):
             try:
                 self.failUnless("%s: %s" % (allmydata.__appname__, allmydata.__version__) in res)
@@ -1360,11 +1359,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 raise
         d.addCallback(_check)
         # make sure it serves the CSS too
-        d.addCallback(lambda res:
+        d.addCallback(lambda res: do_http("get", self.introweb_url+"tahoe.css"))
-                      getPage(self.introweb_url+"tahoe.css", method="GET"))
+        d.addCallback(lambda res: do_http("get", self.introweb_url + "?t=json"))
-        d.addCallback(lambda res:
-                      getPage(self.introweb_url + "?t=json",
-                              method="GET", followRedirect=True))
         def _check_json(res):
             data = json.loads(res)
             try:
@@ -1607,14 +1603,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return d
 
     def PUT(self, urlpath, data):
-        url = self.webish_url + urlpath
+        return do_http("put", self.webish_url + urlpath, data=data)
-        return getPage(url, method="PUT", postdata=data)
 
-    def GET(self, urlpath, followRedirect=False):
+    def GET(self, urlpath):
-        url = self.webish_url + urlpath
+        return do_http("get", self.webish_url + urlpath)
-        return getPage(url, method="GET", followRedirect=followRedirect)
 
-    def POST(self, urlpath, followRedirect=False, use_helper=False, **fields):
+    def POST(self, urlpath, use_helper=False, **fields):
         sepbase = "boogabooga"
         sep = "--" + sepbase
         form = []
@@ -1639,21 +1633,18 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         if fields:
             body = "\r\n".join(form) + "\r\n"
             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
-        return self.POST2(urlpath, body, headers, followRedirect, use_helper)
+        return self.POST2(urlpath, body, headers, use_helper)
 
-    def POST2(self, urlpath, body="", headers={}, followRedirect=False,
+    def POST2(self, urlpath, body="", headers={}, use_helper=False):
-              use_helper=False):
         if use_helper:
             url = self.helper_webish_url + urlpath
         else:
             url = self.webish_url + urlpath
-        return getPage(url, method="POST", postdata=body, headers=headers,
+        return do_http("post", url, data=body, headers=headers)
-                       followRedirect=followRedirect)
 
     def _test_web(self, res):
-        base = self.webish_url
         public = "uri/" + self._root_directory_uri
-        d = getPage(base)
+        d = self.GET("")
         def _got_welcome(page):
             html = page.replace('\n', ' ')
             connected_re = r'Connected to <span>%d</span>\s*of <span>%d</span> known storage servers' % (self.numclients, self.numclients)
@@ -1669,23 +1660,22 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(self.log, "done with _got_welcome")
 
         # get the welcome page from the node that uses the helper too
-        d.addCallback(lambda res: getPage(self.helper_webish_url))
+        d.addCallback(lambda res: do_http("get", self.helper_webish_url))
         def _got_welcome_helper(page):
             html = page.replace('\n', ' ')
             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), page)
             self.failUnlessIn("Not running helper", page)
         d.addCallback(_got_welcome_helper)
 
-        d.addCallback(lambda res: getPage(base + public))
+        d.addCallback(lambda res: self.GET(public))
-        d.addCallback(lambda res: getPage(base + public + "/subdir1"))
+        d.addCallback(lambda res: self.GET(public + "/subdir1"))
         def _got_subdir1(page):
             # there ought to be an href for our file
             self.failUnlessIn('<td align="right">%d</td>' % len(self.data), page)
             self.failUnless(">mydata567</a>" in page)
         d.addCallback(_got_subdir1)
         d.addCallback(self.log, "done with _got_subdir1")
-        d.addCallback(lambda res:
+        d.addCallback(lambda res: self.GET(public + "/subdir1/mydata567"))
-                      getPage(base + public + "/subdir1/mydata567"))
         def _got_data(page):
             self.failUnlessEqual(page, self.data)
         d.addCallback(_got_data)
@@ -1693,8 +1683,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # download from a URI embedded in a URL
         d.addCallback(self.log, "_get_from_uri")
         def _get_from_uri(res):
-            return getPage(base + "uri/%s?filename=%s"
+            return self.GET("uri/%s?filename=%s" % (self.uri, "mydata567"))
-                           % (self.uri, "mydata567"))
         d.addCallback(_get_from_uri)
         def _got_from_uri(page):
             self.failUnlessEqual(page, self.data)
@@ -1703,18 +1692,18 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # download from a URI embedded in a URL, second form
         d.addCallback(self.log, "_get_from_uri2")
         def _get_from_uri2(res):
-            return getPage(base + "uri?uri=%s" % (self.uri,))
+            return self.GET("uri?uri=%s" % (self.uri,))
         d.addCallback(_get_from_uri2)
         d.addCallback(_got_from_uri)
 
         # download from a bogus URI, make sure we get a reasonable error
         d.addCallback(self.log, "_get_from_bogus_uri", level=log.UNUSUAL)
+        @defer.inlineCallbacks
         def _get_from_bogus_uri(res):
-            d1 = getPage(base + "uri/%s?filename=%s"
+            d1 = self.GET("uri/%s?filename=%s"
-                         % (self.mangle_uri(self.uri), "mydata567"))
+                          % (self.mangle_uri(self.uri), "mydata567"))
-            d1.addBoth(self.shouldFail, Error, "downloading bogus URI",
+            e = yield self.assertFailure(d1, Error)
-                       "410")
+            self.assertEquals(e.status, "410")
-            return d1
         d.addCallback(_get_from_bogus_uri)
         d.addCallback(self.log, "_got_from_bogus_uri", level=log.UNUSUAL)
 
@@ -1751,7 +1740,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                                             file=("foo.txt", "data2" * 10000)))
 
         # check that the status page exists
-        d.addCallback(lambda res: self.GET("status", followRedirect=True))
+        d.addCallback(lambda res: self.GET("status"))
         def _got_status(res):
             # find an interesting upload and download to look at. LIT files
             # are not interesting.
@@ -1790,8 +1779,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(_got_retrieve)
 
         # check that the helper status page exists
-        d.addCallback(lambda res:
+        d.addCallback(lambda res: self.GET("helper_status"))
-                      self.GET("helper_status", followRedirect=True))
         def _got_helper_status(res):
             self.failUnless("Bytes Fetched:" in res)
             # touch a couple of files in the helper's working directory to
@@ -1811,8 +1799,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             os.utime(encfile, (now, then))
         d.addCallback(_got_helper_status)
         # and that the json form exists
-        d.addCallback(lambda res:
+        d.addCallback(lambda res: self.GET("helper_status?t=json"))
-                      self.GET("helper_status?t=json", followRedirect=True))
         def _got_helper_status_json(res):
             data = json.loads(res)
             self.failUnlessEqual(data["chk_upload_helper.upload_need_upload"],
@@ -1829,14 +1816,16 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         # and check that client[3] (which uses a helper but does not run one
         # itself) doesn't explode when you ask for its status
-        d.addCallback(lambda res: getPage(self.helper_webish_url + "status/"))
+        d.addCallback(lambda res: do_http("get",
+                                          self.helper_webish_url + "status/"))
         def _got_non_helper_status(res):
             self.failUnlessIn("Recent and Active Operations", res)
         d.addCallback(_got_non_helper_status)
 
         # or for helper status with t=json
         d.addCallback(lambda res:
-                      getPage(self.helper_webish_url + "helper_status?t=json"))
+                      do_http("get",
+                              self.helper_webish_url + "helper_status?t=json"))
         def _got_non_helper_status_json(res):
             data = json.loads(res)
             self.failUnlessEqual(data, {})

src/allmydata/test/web/test_introducer.py

@@ -2,10 +2,10 @@ import os.path
 from twisted.trial import unittest
 from foolscap.api import fireEventually, flushEventualQueue
 from allmydata.util import fileutil
-from twisted.internet import defer, reactor
+from twisted.internet import defer
 from allmydata.introducer import IntroducerNode
 from .common import FAVICON_MARKUP
-from ..common_web import HTTPClientGETFactory
+from ..common_web import do_http
 
 class IntroducerWeb(unittest.TestCase):
     def setUp(self):
@@ -18,6 +18,7 @@ class IntroducerWeb(unittest.TestCase):
         d.addCallback(flushEventualQueue)
         return d
 
+    @defer.inlineCallbacks
     def test_welcome(self):
         basedir = "web.IntroducerWeb.test_welcome"
         os.mkdir(basedir)
@@ -29,31 +30,12 @@ class IntroducerWeb(unittest.TestCase):
         self.node = IntroducerNode(basedir)
         self.ws = self.node.getServiceNamed("webish")
 
-        d = fireEventually(None)
+        yield fireEventually(None)
-        d.addCallback(lambda ign: self.node.startService())
+        self.node.startService()
-
-        d.addCallback(lambda ign: self.GET("/"))
-        def _check(res):
-            self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
-            self.failUnlessIn(FAVICON_MARKUP, res)
-            self.failUnlessIn('Page rendered at', res)
-            self.failUnlessIn('Tahoe-LAFS code imported from:', res)
-        d.addCallback(_check)
-        return d
-
-    def GET(self, urlpath, followRedirect=False, return_response=False,
-            **kwargs):
-        # if return_response=True, this fires with (data, statuscode,
-        # respheaders) instead of just data.
-        assert not isinstance(urlpath, unicode)
-        url = self.ws.getURL().rstrip('/') + urlpath
-        factory = HTTPClientGETFactory(url, method="GET",
-                                       followRedirect=followRedirect, **kwargs)
-        reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
-        d = factory.deferred
-        def _got_data(data):
-            return (data, factory.status, factory.response_headers)
-        if return_response:
-            d.addCallback(_got_data)
-        return factory.deferred
 
+        url = "http://localhost:%d/" % self.ws.getPortnum()
+        res = yield do_http("get", url)
+        self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
+        self.failUnlessIn(FAVICON_MARKUP, res)
+        self.failUnlessIn('Page rendered at', res)
+        self.failUnlessIn('Tahoe-LAFS code imported from:', res)

src/allmydata/test/web/test_web.py

@@ -1,9 +1,11 @@
 import os.path, re, urllib, time, cgi
 import json
+import treq
 
 from twisted.application import service
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
+from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.internet.task import Clock
 from twisted.web import client, error, http
 from twisted.python import failure, log
@@ -27,7 +29,7 @@ from ..common import FakeCHKFileNode, FakeMutableFileNode, \
 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
 from allmydata.mutable import servermap, publish, retrieve
 from .. import common_util as testutil
-from ..common_web import HTTPClientGETFactory, HTTPClientHEADFactory
+from ..common_web import HTTPClientGETFactory, do_http, Error
 from allmydata.client import Client, SecretHolder
 from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP
 # create a fake uploader/downloader, and a couple of fake dirnodes, then
@@ -489,25 +491,25 @@ class WebMixin(testutil.TimezoneMixin):
             d.addCallback(_got_data)
         return factory.deferred
 
-    def HEAD(self, urlpath, return_response=False, **kwargs):
+    @inlineCallbacks
-        # this requires some surgery, because twisted.web.client doesn't want
+    def HEAD(self, urlpath, return_response=False, headers={}):
-        # to give us back the response headers.
-        factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
-        reactor.connectTCP("localhost", self.webish_port, factory)
-        d = factory.deferred
-        def _got_data(data):
-            return (data, factory.status, factory.response_headers)
-        if return_response:
-            d.addCallback(_got_data)
-        return factory.deferred
-
-    def PUT(self, urlpath, data, **kwargs):
         url = self.webish_url + urlpath
-        return client.getPage(url, method="PUT", postdata=data, **kwargs)
+        response = yield treq.request("head", url, persistent=False,
+                                      headers=headers)
+        if 400 <= response.code < 600:
+            raise Error(response.code, response="")
+        resp_headers = {}
+        for (key, values) in response.headers.getAllRawHeaders():
+            resp_headers[key.lower()] = values
+        returnValue( ("", response.code, resp_headers) )
+
+    def PUT(self, urlpath, data, headers={}):
+        url = self.webish_url + urlpath
+        return do_http("put", url, data=data, headers=headers)
 
     def DELETE(self, urlpath):
         url = self.webish_url + urlpath
-        return client.getPage(url, method="DELETE")
+        return do_http("delete", url)
 
     def POST(self, urlpath, followRedirect=False, **fields):
         sepbase = "boogabooga"