tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

Merge branch '3607.web-python-3-part-1' into 3611.web-python-3-part-2

This commit is contained in:
Itamar Turner-Trauring 2021-02-10 14:33:00 -05:00
parent e5806301d3 e3bb368184
commit 3abbe76d6a
8 changed files with 188 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3607.minor Normal file
View File

13
src/allmydata/dirnode.py
View File

@ -74,6 +74,13 @@ ADD_FILE = ActionType(
u"Add a new file as a child of a directory.",
)
class _OnlyFiles(object):
"""Marker for replacement option of only replacing files."""
ONLY_FILES = _OnlyFiles()
def update_metadata(metadata, new_metadata, now):
"""Updates 'metadata' in-place with the information in 'new_metadata'.
@ -179,7 +186,7 @@ class Adder(object):
if entries is None:
entries = {}
precondition(isinstance(entries, dict), entries)
precondition(overwrite in (True, False, "only-files"), overwrite)
precondition(overwrite in (True, False, ONLY_FILES), overwrite)
# keys of 'entries' may not be normalized.
self.entries = entries
self.overwrite = overwrite
@ -205,7 +212,7 @@ class Adder(object):
if not self.overwrite:
raise ExistingChildError("child %s already exists" % quote_output(name, encoding='utf-8'))
if self.overwrite == "only-files" and IDirectoryNode.providedBy(children[name][0]):
if self.overwrite == ONLY_FILES and IDirectoryNode.providedBy(children[name][0]):
raise ExistingChildError("child %s already exists as a directory" % quote_output(name, encoding='utf-8'))
metadata = children[name][1].copy()
@ -701,7 +708,7 @@ class DirectoryNode(object):
'new_child_namex' and 'current_child_namex' need not be normalized.
The overwrite parameter may be True (overwrite any existing child),
False (error if the new child link already exists), or "only-files"
False (error if the new child link already exists), or ONLY_FILES
(error if the new child link exists and points to a directory).
"""
if self.is_readonly() or new_parent.is_readonly():

4
src/allmydata/test/test_dirnode.py
View File

@ -1978,12 +1978,12 @@ class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
overwrite=False))
d.addCallback(lambda res:
root_node.set_node(u'file1', filenode,
overwrite="only-files"))
overwrite=dirnode.ONLY_FILES))
d.addCallback(lambda res:
self.shouldFail(ExistingChildError, "set_node",
"child 'dir1' already exists",
root_node.set_node, u'dir1', filenode,
overwrite="only-files"))
overwrite=dirnode.ONLY_FILES))
return d
d.addCallback(_test_adder)

11
src/allmydata/test/web/test_util.py
View File

@ -12,17 +12,18 @@ if PY2:
from twisted.trial import unittest
from allmydata.web import status, common
from allmydata.dirnode import ONLY_FILES
from ..common import ShouldFailMixin
from .. import common_util as testutil
class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
def test_parse_replace_arg(self):
self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
"only-files")
self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
self.failUnlessReallyEqual(common.parse_replace_arg(b"true"), True)
self.failUnlessReallyEqual(common.parse_replace_arg(b"false"), False)
self.failUnlessReallyEqual(common.parse_replace_arg(b"only-files"),
ONLY_FILES)
self.failUnlessRaises(common.WebError, common.parse_replace_arg, b"only_fles")
def test_abbreviate_time(self):
self.failUnlessReallyEqual(common.abbreviate_time(None), "")

1
src/allmydata/util/_python3.py
View File

@ -115,6 +115,7 @@ PORTED_MODULES = [
"allmydata.util.spans",
"allmydata.util.statistics",
"allmydata.util.time_format",
"allmydata.web.common",
"allmydata.web.logs",
"allmydata.webish",
]

193
src/allmydata/web/common.py
View File

@ -1,5 +1,22 @@
from past.builtins import unicode
from six import ensure_text, ensure_str
"""
Ported to Python 3.
"""
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
from past.builtins import unicode as str # prevent leaking newbytes/newstr into code that can't handle it
from six import ensure_str
try:
from typing import Optional, Union, Tuple, Any
except ImportError:
pass
import time
import json
@ -51,6 +68,7 @@ from twisted.web.resource import (
IResource,
)
from allmydata.dirnode import ONLY_FILES, _OnlyFiles
from allmydata import blacklist
from allmydata.interfaces import (
EmptyPathnameComponentError,
@ -74,11 +92,13 @@ from allmydata.util.encodingutil import (
quote_output,
to_bytes,
)
from allmydata.util import abbreviate
# Originally part of this module, so still part of its API:
from .common_py3 import ( # noqa: F401
get_arg, abbreviate_time, MultiFormatResource, WebError,
)
class WebError(Exception):
def __init__(self, text, code=http.BAD_REQUEST):
self.text = text
self.code = code
def get_filenode_metadata(filenode):
@ -98,17 +118,17 @@ def get_filenode_metadata(filenode):
metadata['size'] = size
return metadata
def boolean_of_arg(arg):
# TODO: ""
arg = ensure_text(arg)
if arg.lower() not in ("true", "t", "1", "false", "f", "0", "on", "off"):
def boolean_of_arg(arg): # type: (bytes) -> bool
assert isinstance(arg, bytes)
if arg.lower() not in (b"true", b"t", b"1", b"false", b"f", b"0", b"on", b"off"):
raise WebError("invalid boolean argument: %r" % (arg,), http.BAD_REQUEST)
return arg.lower() in ("true", "t", "1", "on")
return arg.lower() in (b"true", b"t", b"1", b"on")
def parse_replace_arg(replace):
replace = ensure_text(replace)
if replace.lower() == "only-files":
return replace
def parse_replace_arg(replace): # type: (bytes) -> Union[bool,_OnlyFiles]
assert isinstance(replace, bytes)
if replace.lower() == b"only-files":
return ONLY_FILES
try:
return boolean_of_arg(replace)
except WebError:
@ -145,19 +165,19 @@ def get_mutable_type(file_format): # accepts result of get_format()
return None
def parse_offset_arg(offset):
def parse_offset_arg(offset): # type: (bytes) -> Union[int,None]
# XXX: This will raise a ValueError when invoked on something that
# is not an integer. Is that okay? Or do we want a better error
# message? Since this call is going to be used by programmers and
# their tools rather than users (through the wui), it is not
# inconsistent to return that, I guess.
if offset is not None:
offset = int(offset)
return int(offset)
return offset
def get_root(req):
def get_root(req): # type: (IRequest) -> str
"""
Get a relative path with parent directory segments that refers to the root
location known to the given request. This seems a lot like the constant
@ -186,8 +206,8 @@ def convert_children_json(nodemaker, children_json):
children = {}
if children_json:
data = json.loads(children_json)
for (namex, (ctype, propdict)) in data.items():
namex = unicode(namex)
for (namex, (ctype, propdict)) in list(data.items()):
namex = str(namex)
writecap = to_bytes(propdict.get("rw_uri"))
readcap = to_bytes(propdict.get("ro_uri"))
metadata = propdict.get("metadata", {})
@ -208,7 +228,8 @@ def compute_rate(bytes, seconds):
assert bytes > -1
assert seconds > 0
return 1.0 * bytes / seconds
return bytes / seconds
def abbreviate_rate(data):
"""
@ -229,6 +250,7 @@ def abbreviate_rate(data):
return u"%.1fkBps" % (r/1000)
return u"%.0fBps" % r
def abbreviate_size(data):
"""
Convert number of bytes into human readable strings (unicode).
@ -265,7 +287,7 @@ def text_plain(text, req):
return text
def spaces_to_nbsp(text):
return unicode(text).replace(u' ', u'\u00A0')
return str(text).replace(u' ', u'\u00A0')
def render_time_delta(time_1, time_2):
return spaces_to_nbsp(format_delta(time_1, time_2))
@ -283,7 +305,7 @@ def render_time_attr(t):
# actual exception). The latter is growing increasingly annoying.
def should_create_intermediate_directories(req):
t = unicode(get_arg(req, "t", "").strip(), "ascii")
t = str(get_arg(req, "t", "").strip(), "ascii")
return bool(req.method in (b"PUT", b"POST") and
t not in ("delete", "rename", "rename-form", "check"))
@ -565,7 +587,7 @@ def _finish(result, render, request):
resource=fullyQualifiedName(type(result)),
)
result.render(request)
elif isinstance(result, unicode):
elif isinstance(result, str):
Message.log(
message_type=u"allmydata:web:common-render:unicode",
)
@ -647,7 +669,7 @@ def _renderHTTP_exception(request, failure):
def _renderHTTP_exception_simple(request, text, code):
request.setResponseCode(code)
request.setHeader("content-type", "text/plain;charset=utf-8")
if isinstance(text, unicode):
if isinstance(text, str):
text = text.encode("utf-8")
request.setHeader("content-length", b"%d" % len(text))
return text
@ -689,3 +711,124 @@ def url_for_string(req, url_string):
port=port,
)
return url
def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any]
"""Extract an argument from either the query args (req.args) or the form
body fields (req.fields). If multiple=False, this returns a single value
(or the default, which defaults to None), and the query args take
precedence. If multiple=True, this returns a tuple of arguments (possibly
empty), starting with all those in the query args.
:param TahoeLAFSRequest req: The request to consider.
:return: Either bytes or tuple of bytes.
"""
if isinstance(argname, str):
argname = argname.encode("utf-8")
if isinstance(default, str):
default = default.encode("utf-8")
results = []
if argname in req.args:
results.extend(req.args[argname])
argname_unicode = str(argname, "utf-8")
if req.fields and argname_unicode in req.fields:
value = req.fields[argname_unicode].value
if isinstance(value, str):
value = value.encode("utf-8")
results.append(value)
if multiple:
return tuple(results)
if results:
return results[0]
return default
class MultiFormatResource(resource.Resource, object):
"""
``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
a number of different formats.
Rendered format is controlled by a query argument (given by
``self.formatArgument``). Different resources may support different
formats but ``json`` is a pretty common one. ``html`` is the default
format if nothing else is given as the ``formatDefault``.
"""
formatArgument = "t"
formatDefault = None # type: Optional[str]
def render(self, req):
"""
Dispatch to a renderer for a particular format, as selected by a query
argument.
A renderer for the format given by the query argument matching
``formatArgument`` will be selected and invoked. render_HTML will be
used as a default if no format is selected (either by query arguments
or by ``formatDefault``).
:return: The result of the selected renderer.
"""
t = get_arg(req, self.formatArgument, self.formatDefault)
# It's either bytes or None.
if isinstance(t, bytes):
t = str(t, "ascii")
renderer = self._get_renderer(t)
result = renderer(req)
# On Python 3, json.dumps() returns Unicode for example, but
# twisted.web expects bytes. Instead of updating every single render
# method, just handle Unicode one time here.
if isinstance(result, str):
result = result.encode("utf-8")
return result
def _get_renderer(self, fmt):
"""
Get the renderer for the indicated format.
:param str fmt: The format. If a method with a prefix of ``render_``
and a suffix of this format (upper-cased) is found, it will be
used.
:return: A callable which takes a twisted.web Request and renders a
response.
"""
renderer = None
if fmt is not None:
try:
renderer = getattr(self, "render_{}".format(fmt.upper()))
except AttributeError:
return resource.ErrorPage(
http.BAD_REQUEST,
"Bad Format",
"Unknown {} value: {!r}".format(self.formatArgument, fmt),
).render
if renderer is None:
renderer = self.render_HTML
return renderer
def abbreviate_time(data):
"""
Convert number of seconds into human readable string.
:param data: Either ``None`` or integer or float, seconds.
:return: Unicode string.
"""
# 1.23s, 790ms, 132us
if data is None:
return u""
s = float(data)
if s >= 10:
return abbreviate.abbreviate_time(data)
if s >= 1.0:
return u"%.2fs" % s
if s >= 0.01:
return u"%.0fms" % (1000*s)
if s >= 0.001:
return u"%.1fms" % (1000*s)
return u"%.0fus" % (1000000*s)

143
src/allmydata/web/common_py3.py
View File

@ -1,143 +0,0 @@
"""
Common utilities that are available from Python 3.
Can eventually be merged back into allmydata.web.common.
"""
from past.builtins import unicode
try:
from typing import Optional
except ImportError:
pass
from twisted.web import resource, http
from allmydata.util import abbreviate
class WebError(Exception):
def __init__(self, text, code=http.BAD_REQUEST):
self.text = text
self.code = code
def get_arg(req, argname, default=None, multiple=False):
"""Extract an argument from either the query args (req.args) or the form
body fields (req.fields). If multiple=False, this returns a single value
(or the default, which defaults to None), and the query args take
precedence. If multiple=True, this returns a tuple of arguments (possibly
empty), starting with all those in the query args.
:param TahoeLAFSRequest req: The request to consider.
:return: Either bytes or tuple of bytes.
"""
if isinstance(argname, unicode):
argname = argname.encode("utf-8")
if isinstance(default, unicode):
default = default.encode("utf-8")
results = []
if argname in req.args:
results.extend(req.args[argname])
argname_unicode = unicode(argname, "utf-8")
if req.fields and argname_unicode in req.fields:
value = req.fields[argname_unicode].value
if isinstance(value, unicode):
value = value.encode("utf-8")
results.append(value)
if multiple:
return tuple(results)
if results:
return results[0]
return default
class MultiFormatResource(resource.Resource, object):
"""
``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
a number of different formats.
Rendered format is controlled by a query argument (given by
``self.formatArgument``). Different resources may support different
formats but ``json`` is a pretty common one. ``html`` is the default
format if nothing else is given as the ``formatDefault``.
"""
formatArgument = "t"
formatDefault = None # type: Optional[str]
def render(self, req):
"""
Dispatch to a renderer for a particular format, as selected by a query
argument.
A renderer for the format given by the query argument matching
``formatArgument`` will be selected and invoked. render_HTML will be
used as a default if no format is selected (either by query arguments
or by ``formatDefault``).
:return: The result of the selected renderer.
"""
t = get_arg(req, self.formatArgument, self.formatDefault)
# It's either bytes or None.
if isinstance(t, bytes):
t = unicode(t, "ascii")
renderer = self._get_renderer(t)
result = renderer(req)
# On Python 3, json.dumps() returns Unicode for example, but
# twisted.web expects bytes. Instead of updating every single render
# method, just handle Unicode one time here.
if isinstance(result, unicode):
result = result.encode("utf-8")
return result
def _get_renderer(self, fmt):
"""
Get the renderer for the indicated format.
:param str fmt: The format. If a method with a prefix of ``render_``
and a suffix of this format (upper-cased) is found, it will be
used.
:return: A callable which takes a twisted.web Request and renders a
response.
"""
renderer = None
if fmt is not None:
try:
renderer = getattr(self, "render_{}".format(fmt.upper()))
except AttributeError:
return resource.ErrorPage(
http.BAD_REQUEST,
"Bad Format",
"Unknown {} value: {!r}".format(self.formatArgument, fmt),
).render
if renderer is None:
renderer = self.render_HTML
return renderer
def abbreviate_time(data):
"""
Convert number of seconds into human readable string.
:param data: Either ``None`` or integer or float, seconds.
:return: Unicode string.
"""
# 1.23s, 790ms, 132us
if data is None:
return u""
s = float(data)
if s >= 10:
return abbreviate.abbreviate_time(data)
if s >= 1.0:
return u"%.2fs" % s
if s >= 0.01:
return u"%.0fms" % (1000*s)
if s >= 0.001:
return u"%.1fms" % (1000*s)
return u"%.0fus" % (1000000*s)

2
src/allmydata/web/storage.py
View File

@ -9,7 +9,7 @@ from twisted.web.template import (
renderer,
renderElement
)
from allmydata.web.common_py3 import (
from allmydata.web.common import (
abbreviate_time,
MultiFormatResource
)