diff -rN -u old-tahoe/src/allmydata/client.py new-tahoe/src/allmydata/client.py --- old-tahoe/src/allmydata/client.py 2010-01-23 12:59:08.664000000 +0000 +++ new-tahoe/src/allmydata/client.py 2010-01-23 12:59:11.145000000 +0000 @@ -471,13 +471,16 @@ # dirnodes. The first takes a URI and produces a filenode or (new-style) # dirnode. The other three create brand-new filenodes/dirnodes. - def create_node_from_uri(self, writecap, readcap=None): - # this returns synchronously. - return self.nodemaker.create_from_cap(writecap, readcap) + def create_node_from_uri(self, write_uri, read_uri=None, deep_immutable=False, name=""): + # This returns synchronously. + # Note that it does *not* validate the write_uri and read_uri; instead we + # may get an opaque node if there were any problems. + return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) def create_dirnode(self, initial_children={}): d = self.nodemaker.create_new_mutable_directory(initial_children) return d + def create_immutable_dirnode(self, children, convergence=None): return self.nodemaker.create_immutable_directory(children, convergence) diff -rN -u old-tahoe/src/allmydata/control.py new-tahoe/src/allmydata/control.py --- old-tahoe/src/allmydata/control.py 2010-01-23 12:59:08.685000000 +0000 +++ new-tahoe/src/allmydata/control.py 2010-01-23 12:59:11.158000000 +0000 @@ -5,7 +5,7 @@ from twisted.internet import defer from twisted.internet.interfaces import IConsumer from foolscap.api import Referenceable -from allmydata.interfaces import RIControlClient +from allmydata.interfaces import RIControlClient, IFileNode from allmydata.util import fileutil, mathutil from allmydata.immutable import upload from twisted.python import log @@ -67,7 +67,9 @@ return d def remote_download_from_uri_to_file(self, uri, filename): - filenode = self.parent.create_node_from_uri(uri) + filenode = self.parent.create_node_from_uri(uri, name=filename) + if not IFileNode.providedBy(filenode): + raise AssertionError("The URI does not reference a file.") c = FileWritingConsumer(filename) d = filenode.read(c) d.addCallback(lambda res: filename) @@ -199,6 +201,8 @@ if i >= self.count: return n = self.parent.create_node_from_uri(self.uris[i]) + if not IFileNode.providedBy(n): + raise AssertionError("The URI does not reference a file.") if n.is_mutable(): d1 = n.download_best_version() else: diff -rN -u old-tahoe/src/allmydata/dirnode.py new-tahoe/src/allmydata/dirnode.py --- old-tahoe/src/allmydata/dirnode.py 2010-01-23 12:59:08.694000000 +0000 +++ new-tahoe/src/allmydata/dirnode.py 2010-01-23 12:59:11.166000000 +0000 @@ -5,13 +5,13 @@ from twisted.internet import defer from foolscap.api import fireEventually import simplejson -from allmydata.mutable.common import NotMutableError +from allmydata.mutable.common import NotWriteableError from allmydata.mutable.filenode import MutableFileNode -from allmydata.unknown import UnknownNode +from allmydata.unknown import UnknownNode, strip_prefix_for_ro from allmydata.interfaces import IFilesystemNode, IDirectoryNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, \ ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable, \ - CannotPackUnknownNodeError + MustBeDeepImmutableError, CapConstraintError from allmydata.check_results import DeepCheckResults, \ DeepCheckAndRepairResults from allmydata.monitor import Monitor @@ -23,6 +23,11 @@ from pycryptopp.cipher.aes import AES from allmydata.util.dictutil import AuxValueDict + +# TODO: {Deleter,MetadataSetter,Adder}.modify all start by unpacking the +# contents and end by repacking them. It might be better to apply them to +# the unpacked contents. + class Deleter: def __init__(self, node, name, must_exist=True): self.node = node @@ -40,6 +45,7 @@ new_contents = self.node._pack_contents(children) return new_contents + class MetadataSetter: def __init__(self, node, name, metadata): self.node = node @@ -75,6 +81,11 @@ for (name, (child, new_metadata)) in self.entries.iteritems(): precondition(isinstance(name, unicode), name) precondition(IFilesystemNode.providedBy(child), child) + + # Strictly speaking this is redundant because we would raise the + # error again in pack_children. + child.raise_error() + if name in children: if not self.overwrite: raise ExistingChildError("child '%s' already exists" % name) @@ -123,25 +134,21 @@ new_contents = self.node._pack_contents(children) return new_contents -def _encrypt_rwcap(filenode, rwcap): - assert isinstance(rwcap, str) +def _encrypt_rw_uri(filenode, rw_uri): + assert isinstance(rw_uri, str) writekey = filenode.get_writekey() if not writekey: return "" - salt = hashutil.mutable_rwcap_salt_hash(rwcap) + salt = hashutil.mutable_rwcap_salt_hash(rw_uri) key = hashutil.mutable_rwcap_key_hash(salt, writekey) cryptor = AES(key) - crypttext = cryptor.process(rwcap) + crypttext = cryptor.process(rw_uri) mac = hashutil.hmac(key, salt + crypttext) assert len(mac) == 32 return salt + crypttext + mac # The MAC is not checked by readers in Tahoe >= 1.3.0, but we still # produce it for the sake of older readers. -class MustBeDeepImmutable(Exception): - """You tried to add a non-deep-immutable node to a deep-immutable - directory.""" - def pack_children(filenode, children, deep_immutable=False): """Take a dict that maps: children[unicode_name] = (IFileSystemNode, metadata_dict) @@ -152,7 +159,7 @@ time. If deep_immutable is True, I will require that all my children are deeply - immutable, and will raise a MustBeDeepImmutable exception if not. + immutable, and will raise a MustBeDeepImmutableError if not. """ has_aux = isinstance(children, AuxValueDict) @@ -161,25 +168,25 @@ assert isinstance(name, unicode) entry = None (child, metadata) = children[name] - if deep_immutable and child.is_mutable(): - # TODO: consider adding IFileSystemNode.is_deep_immutable() - raise MustBeDeepImmutable("child '%s' is mutable" % (name,)) + child.raise_error() + if deep_immutable and not child.is_allowed_in_immutable_directory(): + raise MustBeDeepImmutableError("child '%s' is not allowed in an immutable directory" % (name,), name) if has_aux: entry = children.get_aux(name) if not entry: assert IFilesystemNode.providedBy(child), (name,child) assert isinstance(metadata, dict) - rwcap = child.get_uri() # might be RO if the child is not writeable - if rwcap is None: - rwcap = "" - assert isinstance(rwcap, str), rwcap - rocap = child.get_readonly_uri() - if rocap is None: - rocap = "" - assert isinstance(rocap, str), rocap + rw_uri = child.get_write_uri() + if rw_uri is None: + rw_uri = "" + assert isinstance(rw_uri, str), rw_uri + ro_uri = child.get_readonly_uri() + if ro_uri is None: + ro_uri = "" + assert isinstance(ro_uri, str), ro_uri entry = "".join([netstring(name.encode("utf-8")), - netstring(rocap), - netstring(_encrypt_rwcap(filenode, rwcap)), + netstring(strip_prefix_for_ro(ro_uri, deep_immutable)), + netstring(_encrypt_rw_uri(filenode, rw_uri)), netstring(simplejson.dumps(metadata))]) entries.append(netstring(entry)) return "".join(entries) @@ -230,38 +237,66 @@ plaintext = cryptor.process(crypttext) return plaintext - def _create_node(self, rwcap, rocap): - return self._nodemaker.create_from_cap(rwcap, rocap) + def _create_and_validate_node(self, rw_uri, ro_uri, name): + #print "mutable? %r\n" % self.is_mutable() + #print "_create_and_validate_node(rw_uri=%r, ro_uri=%r, name=%r)\n" % (rw_uri, ro_uri, name) + node = self._nodemaker.create_from_cap(rw_uri, ro_uri, + deep_immutable=not self.is_mutable(), + name=name) + node.raise_error() + return node def _unpack_contents(self, data): # the directory is serialized as a list of netstrings, one per child. - # Each child is serialized as a list of four netstrings: (name, - # rocap, rwcap, metadata), in which the name,rocap,metadata are in - # cleartext. The 'name' is UTF-8 encoded. The rwcap is formatted as: - # pack("16ss32s", iv, AES(H(writekey+iv), plaintextrwcap), mac) + # Each child is serialized as a list of four netstrings: (name, ro_uri, + # rwcapdata, metadata), in which the name, ro_uri, metadata are in + # cleartext. The 'name' is UTF-8 encoded. The rwcapdata is formatted as: + # pack("16ss32s", iv, AES(H(writekey+iv), plaintext_rw_uri), mac) assert isinstance(data, str), (repr(data), type(data)) # an empty directory is serialized as an empty string if data == "": return AuxValueDict() writeable = not self.is_readonly() + mutable = self.is_mutable() children = AuxValueDict() position = 0 while position < len(data): entries, position = split_netstring(data, 1, position) entry = entries[0] - (name, rocap, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) + (name, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) name = name.decode("utf-8") - rwcap = None + rw_uri = "" if writeable: - rwcap = self._decrypt_rwcapdata(rwcapdata) - if not rwcap: - rwcap = None # rwcap is None or a non-empty string - if not rocap: - rocap = None # rocap is None or a non-empty string - child = self._create_node(rwcap, rocap) - metadata = simplejson.loads(metadata_s) - assert isinstance(metadata, dict) - children.set_with_aux(name, (child, metadata), auxilliary=entry) + rw_uri = self._decrypt_rwcapdata(rwcapdata) + #print "mutable=%r, writeable=%r, rw_uri=%r, ro_uri=%r, name=%r" % (mutable, writeable, rw_uri, ro_uri, name) + + # Since the encryption uses CTR mode, it currently leaks the length of the + # plaintext rw_uri -- and therefore whether it is present, i.e. whether the + # dirnode is writable (ticket #925). By stripping spaces in Tahoe >= 1.6.0, + # we may make it easier for future versions to plug this leak. + rw_uri = rw_uri.strip() + if not rw_uri: + rw_uri = None # rw_uri is None or a non-empty string + + # Treat ro_uri in the same way for consistency. + ro_uri = ro_uri.strip() + if not ro_uri: + ro_uri = None # ro_uri is None or a non-empty string + + try: + child = self._create_and_validate_node(rw_uri, ro_uri, name) + #print "%r.is_allowed_in_immutable_directory() = %r" % (child, child.is_allowed_in_immutable_directory()) + if mutable or child.is_allowed_in_immutable_directory(): + metadata = simplejson.loads(metadata_s) + assert isinstance(metadata, dict) + children[name] = (child, metadata) + children.set_with_aux(name, (child, metadata), auxilliary=entry) + except CapConstraintError, e: + #print "unmet constraint: (%s, %s)" % (e.args[0], e.args[1].encode("utf-8")) + log.msg(format="unmet constraint on cap for child '%(name)s' unpacked from a directory:\n" + "%(message)s", message=e.args[0], name=e.args[1].encode("utf-8"), + facility="tahoe.webish", level=log.UNUSUAL) + return children def _pack_contents(self, children): @@ -270,21 +305,39 @@ def is_readonly(self): return self._node.is_readonly() + def is_mutable(self): return self._node.is_mutable() + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return not self._node.is_mutable() + + def raise_error(self): + pass + def get_uri(self): return self._uri.to_string() + def get_write_uri(self): + if self.is_readonly(): + return None + return self._uri.to_string() + def get_readonly_uri(self): return self._uri.get_readonly().to_string() def get_cap(self): return self._uri + def get_readcap(self): return self._uri.get_readonly() + def get_verify_cap(self): return self._uri.get_verify_cap() + def get_repair_cap(self): if self._node.is_readonly(): return None # readonly (mutable) dirnodes are not yet repairable @@ -350,7 +403,7 @@ def set_metadata_for(self, name, metadata): assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) assert isinstance(metadata, dict) s = MetadataSetter(self, name, metadata) d = self._node.modify(s.modify) @@ -398,14 +451,10 @@ precondition(isinstance(name, unicode), name) precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) - child_node = self._create_node(writecap, readcap) - if isinstance(child_node, UnknownNode): - # don't be willing to pack unknown nodes: we might accidentally - # put some write-authority into the rocap slot because we don't - # know how to diminish the URI they gave us. We don't even know - # if they gave us a readcap or a writecap. - msg = "cannot pack unknown node as child %s" % str(name) - raise CannotPackUnknownNodeError(msg) + + # We now allow packing unknown nodes, provided they are valid + # for this type of directory. + child_node = self._create_and_validate_node(writecap, readcap, name) d = self.set_node(name, child_node, metadata, overwrite) d.addCallback(lambda res: child_node) return d @@ -423,10 +472,10 @@ writecap, readcap, metadata = e precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) - child_node = self._create_node(writecap, readcap) - if isinstance(child_node, UnknownNode): - msg = "cannot pack unknown node as child %s" % str(name) - raise CannotPackUnknownNodeError(msg) + + # We now allow packing unknown nodes, provided they are valid + # for this type of directory. + child_node = self._create_and_validate_node(writecap, readcap, name) a.set_node(name, child_node, metadata) d = self._node.modify(a.modify) d.addCallback(lambda ign: self) @@ -439,12 +488,12 @@ same name. If this directory node is read-only, the Deferred will errback with a - NotMutableError.""" + NotWriteableError.""" precondition(IFilesystemNode.providedBy(child), child) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) assert isinstance(name, unicode) assert IFilesystemNode.providedBy(child), child a = Adder(self, overwrite=overwrite) @@ -456,7 +505,7 @@ def set_nodes(self, entries, overwrite=True): precondition(isinstance(entries, dict), entries) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) a = Adder(self, entries, overwrite=overwrite) d = self._node.modify(a.modify) d.addCallback(lambda res: self) @@ -470,10 +519,10 @@ the operation completes.""" assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) d = self._uploader.upload(uploadable) - d.addCallback(lambda results: results.uri) - d.addCallback(self._nodemaker.create_from_cap) + d.addCallback(lambda results: + self._create_and_validate_node(results.uri, None, name)) d.addCallback(lambda node: self.set_node(name, node, metadata, overwrite)) return d @@ -483,7 +532,7 @@ fires (with the node just removed) when the operation finishes.""" assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) deleter = Deleter(self, name) d = self._node.modify(deleter.modify) d.addCallback(lambda res: deleter.old_child) @@ -493,7 +542,7 @@ mutable=True): assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) if mutable: d = self._nodemaker.create_new_mutable_directory(initial_children) else: @@ -515,7 +564,7 @@ Deferred that fires when the operation finishes.""" assert isinstance(current_child_name, unicode) if self.is_readonly() or new_parent.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) if new_child_name is None: new_child_name = current_child_name assert isinstance(new_child_name, unicode) diff -rN -u old-tahoe/src/allmydata/immutable/filenode.py new-tahoe/src/allmydata/immutable/filenode.py --- old-tahoe/src/allmydata/immutable/filenode.py 2010-01-23 12:59:08.893000000 +0000 +++ new-tahoe/src/allmydata/immutable/filenode.py 2010-01-23 12:59:11.317000000 +0000 @@ -17,6 +17,9 @@ class _ImmutableFileNodeBase(object): implements(IImmutableFileNode, ICheckable) + def get_write_uri(self): + return None + def get_readonly_uri(self): return self.get_uri() @@ -26,6 +29,15 @@ def is_readonly(self): return True + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return True + + def raise_error(self): + pass + def __hash__(self): return self.u.__hash__() def __eq__(self, other): diff -rN -u old-tahoe/src/allmydata/interfaces.py new-tahoe/src/allmydata/interfaces.py --- old-tahoe/src/allmydata/interfaces.py 2010-01-23 12:59:08.923000000 +0000 +++ new-tahoe/src/allmydata/interfaces.py 2010-01-23 12:59:11.366000000 +0000 @@ -426,6 +426,7 @@ """Return True if the data can be modified by *somebody* (perhaps someone who has a more powerful URI than this one).""" + # TODO: rename to get_read_cap() def get_readonly(): """Return another IURI instance, which represents a read-only form of this one. If is_readonly() is True, this returns self.""" @@ -456,7 +457,6 @@ class IDirnodeURI(Interface): """I am a URI which represents a dirnode.""" - class IFileURI(Interface): """I am a URI which represents a filenode.""" def get_size(): @@ -467,21 +467,28 @@ class IMutableFileURI(Interface): """I am a URI which represents a mutable filenode.""" + class IDirectoryURI(Interface): pass + class IReadonlyDirectoryURI(Interface): pass -class CannotPackUnknownNodeError(Exception): - """UnknownNodes (using filecaps from the future that we don't understand) - cannot yet be copied safely, so I refuse to copy them.""" - -class UnhandledCapTypeError(Exception): - """I recognize the cap/URI, but I cannot create an IFilesystemNode for - it.""" +class CapConstraintError(Exception): + """A constraint on a cap was violated.""" -class NotDeepImmutableError(Exception): - """Deep-immutable directories can only contain deep-immutable children""" +class MustBeDeepImmutableError(CapConstraintError): + """Mutable children cannot be added to an immutable directory. + Also, caps obtained from an immutable directory can trigger this error + if they are later found to refer to a mutable object and then used.""" + +class MustBeReadonlyError(CapConstraintError): + """Known write caps cannot be specified in a ro_uri field. Also, + caps obtained from a ro_uri field can trigger this error if they + are later found to be write caps and then used.""" + +class MustNotBeUnknownRWError(CapConstraintError): + """Cannot add an unknown child cap specified in a rw_uri field.""" # The hierarchy looks like this: # IFilesystemNode @@ -518,9 +525,8 @@ """ def get_uri(): - """ - Return the URI string that can be used by others to get access to - this node. If this node is read-only, the URI will only offer + """Return the URI string corresponding to the strongest cap associated + with this node. If this node is read-only, the URI will only offer read-only access. If this node is read-write, the URI will offer read-write access. @@ -528,6 +534,11 @@ read-only access with others, use get_readonly_uri(). """ + def get_write_uri(n): + """Return the URI string that can be used by others to get write + access to this node, if it is writeable. If this is a read-only node, + return None.""" + def get_readonly_uri(): """Return the URI string that can be used by others to get read-only access to this node. The result is a read-only URI, regardless of @@ -557,6 +568,18 @@ file. """ + def is_unknown(): + """Return True if this is an unknown node.""" + + def is_allowed_in_immutable_directory(): + """Return True if this node is allowed as a child of a deep-immutable + directory. This is true if either the node is of a known-immutable type, + or it is unknown and read-only. + """ + + def raise_error(): + """Raise any error associated with this node.""" + def get_size(): """Return the length (in bytes) of the data this node represents. For directory nodes, I return the size of the backing store. I return @@ -902,7 +925,7 @@ ctime/mtime semantics of traditional filesystems. If this directory node is read-only, the Deferred will errback with a - NotMutableError.""" + NotWriteableError.""" def set_children(entries, overwrite=True): """Add multiple children (by writecap+readcap) to a directory node. @@ -928,7 +951,7 @@ ctime/mtime semantics of traditional filesystems. If this directory node is read-only, the Deferred will errback with a - NotMutableError.""" + NotWriteableError.""" def set_nodes(entries, overwrite=True): """Add multiple children to a directory node. Takes a dict mapping @@ -2074,7 +2097,7 @@ Tahoe process will typically have a single NodeMaker, but unit tests may create simplified/mocked forms for testing purposes. """ - def create_from_cap(writecap, readcap=None): + def create_from_cap(writecap, readcap=None, **kwargs): """I create an IFilesystemNode from the given writecap/readcap. I can only provide nodes for existing file/directory objects: use my other methods to create new objects. I return synchronously.""" diff -rN -u old-tahoe/src/allmydata/mutable/common.py new-tahoe/src/allmydata/mutable/common.py --- old-tahoe/src/allmydata/mutable/common.py 2010-01-23 12:59:08.999000000 +0000 +++ new-tahoe/src/allmydata/mutable/common.py 2010-01-23 12:59:11.412000000 +0000 @@ -8,7 +8,7 @@ # creation MODE_READ = "MODE_READ" -class NotMutableError(Exception): +class NotWriteableError(Exception): pass class NeedMoreDataError(Exception): diff -rN -u old-tahoe/src/allmydata/mutable/filenode.py new-tahoe/src/allmydata/mutable/filenode.py --- old-tahoe/src/allmydata/mutable/filenode.py 2010-01-23 12:59:09.004000000 +0000 +++ new-tahoe/src/allmydata/mutable/filenode.py 2010-01-23 12:59:11.416000000 +0000 @@ -214,6 +214,12 @@ def get_uri(self): return self._uri.to_string() + + def get_write_uri(self): + if self.is_readonly(): + return None + return self._uri.to_string() + def get_readonly_uri(self): return self._uri.get_readonly().to_string() @@ -227,9 +233,19 @@ def is_mutable(self): return self._uri.is_mutable() + def is_readonly(self): return self._uri.is_readonly() + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return not self._uri.is_mutable() + + def raise_error(self): + pass + def __hash__(self): return hash((self.__class__, self._uri)) def __cmp__(self, them): diff -rN -u old-tahoe/src/allmydata/nodemaker.py new-tahoe/src/allmydata/nodemaker.py --- old-tahoe/src/allmydata/nodemaker.py 2010-01-23 12:59:09.045000000 +0000 +++ new-tahoe/src/allmydata/nodemaker.py 2010-01-23 12:59:11.445000000 +0000 @@ -1,7 +1,7 @@ import weakref from zope.interface import implements from allmydata.util.assertutil import precondition -from allmydata.interfaces import INodeMaker, NotDeepImmutableError +from allmydata.interfaces import INodeMaker, MustBeDeepImmutableError from allmydata.immutable.filenode import ImmutableFileNode, LiteralFileNode from allmydata.immutable.upload import Data from allmydata.mutable.filenode import MutableFileNode @@ -44,28 +44,36 @@ def _create_dirnode(self, filenode): return DirectoryNode(filenode, self, self.uploader) - def create_from_cap(self, writecap, readcap=None): + def create_from_cap(self, writecap, readcap=None, deep_immutable=False, name=u""): # this returns synchronously. It starts with a "cap string". assert isinstance(writecap, (str, type(None))), type(writecap) assert isinstance(readcap, (str, type(None))), type(readcap) + #import traceback + #traceback.print_stack() + #print '%r.create_from_cap(%r, %r, %r)' % (self, writecap, readcap, kwargs) + bigcap = writecap or readcap if not bigcap: # maybe the writecap was hidden because we're in a readonly # directory, and the future cap format doesn't have a readcap, or # something. - return UnknownNode(writecap, readcap) - if bigcap in self._node_cache: - return self._node_cache[bigcap] - cap = uri.from_string(bigcap) - node = self._create_from_cap(cap) + return UnknownNode(None, None) # deep_immutable and name not needed + + # The name doesn't matter for caching since it's only used in the error + # attribute of an UnknownNode, and we don't cache those. + memokey = ("I" if deep_immutable else "M") + bigcap + if memokey in self._node_cache: + return self._node_cache[memokey] + cap = uri.from_string(bigcap, deep_immutable=deep_immutable, name=name) + node = self._create_from_single_cap(cap) if node: - self._node_cache[bigcap] = node # note: WeakValueDictionary + self._node_cache[memokey] = node # note: WeakValueDictionary else: - node = UnknownNode(writecap, readcap) # don't cache UnknownNode + # don't cache UnknownNode + node = UnknownNode(writecap, readcap, deep_immutable=deep_immutable, name=name) return node - def _create_from_cap(self, cap): - # This starts with a "cap instance" + def _create_from_single_cap(self, cap): if isinstance(cap, uri.LiteralFileURI): return self._create_lit(cap) if isinstance(cap, uri.CHKFileURI): @@ -76,7 +84,7 @@ uri.ReadonlyDirectoryURI, uri.ImmutableDirectoryURI, uri.LiteralDirectoryURI)): - filenode = self._create_from_cap(cap.get_filenode_cap()) + filenode = self._create_from_single_cap(cap.get_filenode_cap()) return self._create_dirnode(filenode) return None @@ -89,13 +97,11 @@ return d def create_new_mutable_directory(self, initial_children={}): - # initial_children must have metadata (i.e. {} instead of None), and - # should not contain UnknownNodes + # initial_children must have metadata (i.e. {} instead of None) for (name, (node, metadata)) in initial_children.iteritems(): - precondition(not isinstance(node, UnknownNode), - "create_new_mutable_directory does not accept UnknownNode", node) precondition(isinstance(metadata, dict), "create_new_mutable_directory requires metadata to be a dict, not None", metadata) + node.raise_error() d = self.create_mutable_file(lambda n: pack_children(n, initial_children)) d.addCallback(self._create_dirnode) @@ -105,19 +111,15 @@ if convergence is None: convergence = self.secret_holder.get_convergence_secret() for (name, (node, metadata)) in children.iteritems(): - precondition(not isinstance(node, UnknownNode), - "create_immutable_directory does not accept UnknownNode", node) precondition(isinstance(metadata, dict), "create_immutable_directory requires metadata to be a dict, not None", metadata) - if node.is_mutable(): - raise NotDeepImmutableError("%s is not immutable" % (node,)) + node.raise_error() + if not node.is_allowed_in_immutable_directory(): + raise MustBeDeepImmutableError("%s is not immutable" % (node,), name) n = DummyImmutableFileNode() # writekey=None packed = pack_children(n, children) uploadable = Data(packed, convergence) d = self.uploader.upload(uploadable, history=self.history) - def _uploaded(results): - filecap = self.create_from_cap(results.uri) - return filecap - d.addCallback(_uploaded) + d.addCallback(lambda results: self.create_from_cap(None, results.uri)) d.addCallback(self._create_dirnode) return d diff -rN -u old-tahoe/src/allmydata/scripts/common.py new-tahoe/src/allmydata/scripts/common.py --- old-tahoe/src/allmydata/scripts/common.py 2010-01-23 12:59:09.089000000 +0000 +++ new-tahoe/src/allmydata/scripts/common.py 2010-01-23 12:59:11.483000000 +0000 @@ -128,12 +128,14 @@ pass def get_alias(aliases, path, default): + from allmydata import uri # transform "work:path/filename" into (aliases["work"], "path/filename"). # If default=None, then an empty alias is indicated by returning - # DefaultAliasMarker. We special-case "URI:" to make it easy to access - # specific files/directories by their read-cap. + # DefaultAliasMarker. We special-case strings with a recognized cap URI + # prefix, to make it easy to access specific files/directories by their + # caps. path = path.strip() - if path.startswith("URI:"): + if uri.has_uri_prefix(path): # The only way to get a sub-path is to use URI:blah:./foo, and we # strip out the :./ sequence. sep = path.find(":./") diff -rN -u old-tahoe/src/allmydata/scripts/tahoe_cp.py new-tahoe/src/allmydata/scripts/tahoe_cp.py --- old-tahoe/src/allmydata/scripts/tahoe_cp.py 2010-01-23 12:59:09.170000000 +0000 +++ new-tahoe/src/allmydata/scripts/tahoe_cp.py 2010-01-23 12:59:11.536000000 +0000 @@ -258,8 +258,7 @@ readcap = ascii_or_none(data[1].get("ro_uri")) self.children[name] = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) - else: - assert data[0] == "dirnode" + elif data[0] == "dirnode": writecap = ascii_or_none(data[1].get("rw_uri")) readcap = ascii_or_none(data[1].get("ro_uri")) if writecap and writecap in self.cache: @@ -277,6 +276,11 @@ if recurse: child.populate(True) self.children[name] = child + else: + # TODO: there should be an option to skip unknown nodes. + raise TahoeError("Cannot copy unknown nodes (ticket #839). " + "You probably need to use a later version of " + "Tahoe-LAFS to copy this directory.") class TahoeMissingTarget: def __init__(self, url): @@ -353,8 +357,7 @@ urllib.quote(name.encode('utf-8'))]) self.children[name] = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) - else: - assert data[0] == "dirnode" + elif data[0] == "dirnode": writecap = ascii_or_none(data[1].get("rw_uri")) readcap = ascii_or_none(data[1].get("ro_uri")) if writecap and writecap in self.cache: @@ -372,6 +375,11 @@ if recurse: child.populate(True) self.children[name] = child + else: + # TODO: there should be an option to skip unknown nodes. + raise TahoeError("Cannot copy unknown nodes (ticket #839). " + "You probably need to use a later version of " + "Tahoe-LAFS to copy this directory.") def get_child_target(self, name): # return a new target for a named subdirectory of this dir @@ -407,9 +415,11 @@ set_data = {} for (name, filecap) in self.new_children.items(): # it just so happens that ?t=set_children will accept both file - # read-caps and write-caps as ['rw_uri'], and will handle eithe + # read-caps and write-caps as ['rw_uri'], and will handle either # correctly. So don't bother trying to figure out whether the one # we have is read-only or read-write. + # TODO: think about how this affects forward-compatibility for + # unknown caps set_data[name] = ["filenode", {"rw_uri": filecap}] body = simplejson.dumps(set_data) POST(url, body) @@ -770,6 +780,7 @@ # local-file-in-the-way # touch proposed # tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt +# handling of unknown nodes # things that maybe should be errors but aren't # local-dir-in-the-way diff -rN -u old-tahoe/src/allmydata/scripts/tahoe_put.py new-tahoe/src/allmydata/scripts/tahoe_put.py --- old-tahoe/src/allmydata/scripts/tahoe_put.py 2010-01-23 12:59:09.198000000 +0000 +++ new-tahoe/src/allmydata/scripts/tahoe_put.py 2010-01-23 12:59:11.557000000 +0000 @@ -40,6 +40,7 @@ # DIRCAP:./subdir/foo : DIRCAP/subdir/foo # MUTABLE-FILE-WRITECAP : filecap + # FIXME: this shouldn't rely on a particular prefix. if to_file.startswith("URI:SSK:"): url = nodeurl + "uri/%s" % urllib.quote(to_file) else: diff -rN -u old-tahoe/src/allmydata/test/common.py new-tahoe/src/allmydata/test/common.py --- old-tahoe/src/allmydata/test/common.py 2010-01-23 12:59:09.443000000 +0000 +++ new-tahoe/src/allmydata/test/common.py 2010-01-23 12:59:11.729000000 +0000 @@ -51,6 +51,8 @@ def get_uri(self): return self.my_uri.to_string() + def get_write_uri(self): + return None def get_readonly_uri(self): return self.my_uri.to_string() def get_cap(self): @@ -103,6 +105,12 @@ return False def is_readonly(self): return True + def is_unknown(self): + return False + def is_allowed_in_immutable_directory(self): + return True + def raise_error(self): + pass def get_size(self): try: @@ -190,6 +198,10 @@ return self.my_uri.get_readonly() def get_uri(self): return self.my_uri.to_string() + def get_write_uri(self): + if self.is_readonly(): + return None + return self.my_uri.to_string() def get_readonly(self): return self.my_uri.get_readonly() def get_readonly_uri(self): @@ -200,6 +212,12 @@ return self.my_uri.is_readonly() def is_mutable(self): return self.my_uri.is_mutable() + def is_unknown(self): + return False + def is_allowed_in_immutable_directory(self): + return not self.my_uri.is_mutable() + def raise_error(self): + pass def get_writekey(self): return "\x00"*16 def get_size(self): diff -rN -u old-tahoe/src/allmydata/test/test_client.py new-tahoe/src/allmydata/test/test_client.py --- old-tahoe/src/allmydata/test/test_client.py 2010-01-23 12:59:09.713000000 +0000 +++ new-tahoe/src/allmydata/test/test_client.py 2010-01-23 12:59:11.853000000 +0000 @@ -288,11 +288,14 @@ self.failUnless(n.is_readonly()) self.failUnless(n.is_mutable()) - future = "x-tahoe-crazy://future_cap_format." - n = c.create_node_from_uri(future) + unknown_rw = "lafs://from_the_future" + unknown_ro = "lafs://readonly_from_the_future" + n = c.create_node_from_uri(unknown_rw, unknown_ro) self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) self.failIf(IMutableFileNode.providedBy(n)) self.failIf(IDirectoryNode.providedBy(n)) - self.failUnlessEqual(n.get_uri(), future) + self.failUnless(n.is_unknown()) + self.failUnlessEqual(n.get_uri(), unknown_rw) + self.failUnlessEqual(n.get_readonly_uri(), "ro." + unknown_ro) diff -rN -u old-tahoe/src/allmydata/test/test_dirnode.py new-tahoe/src/allmydata/test/test_dirnode.py --- old-tahoe/src/allmydata/test/test_dirnode.py 2010-01-23 12:59:09.774000000 +0000 +++ new-tahoe/src/allmydata/test/test_dirnode.py 2010-01-23 12:59:11.898000000 +0000 @@ -7,8 +7,8 @@ from allmydata.client import Client from allmydata.immutable import upload from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ - ExistingChildError, NoSuchChildError, NotDeepImmutableError, \ - IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError + ExistingChildError, NoSuchChildError, MustBeDeepImmutableError, \ + IDeepCheckResults, IDeepCheckAndRepairResults, MustNotBeUnknownRWError from allmydata.mutable.filenode import MutableFileNode from allmydata.mutable.common import UncoordinatedWriteError from allmydata.util import hashutil, base32 @@ -32,6 +32,11 @@ d = c.create_dirnode() def _done(res): self.failUnless(isinstance(res, dirnode.DirectoryNode)) + self.failUnless(res.is_mutable()) + self.failIf(res.is_readonly()) + self.failIf(res.is_unknown()) + self.failIf(res.is_allowed_in_immutable_directory()) + res.raise_error() rep = str(res) self.failUnless("RW-MUT" in rep) d.addCallback(_done) @@ -44,36 +49,74 @@ nm = c.nodemaker setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" one_uri = "URI:LIT:n5xgk" # LIT for "one" + mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), + u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}), + u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}), + u"fro": (nm.create_from_cap(None, future_read_uri), {}), } d = c.create_dirnode(kids) + def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failUnless(dn.is_mutable()) + self.failIf(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failIf(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RW-MUT" in rep) return dn.list() d.addCallback(_created) + def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) + self.failUnlessEqual(sorted(children.keys()), + [u"fro", u"fut", u"mut", u"one", u"two"]) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] + mut_node, mut_metadata = children[u"mut"] + fut_node, fut_metadata = children[u"fut"] + fro_node, fro_metadata = children[u"fro"] + self.failUnlessEqual(one_node.get_size(), 3) - self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(one_node.get_uri(), one_uri) + self.failUnlessEqual(one_node.get_readonly_uri(), one_uri) self.failUnless(isinstance(one_metadata, dict), one_metadata) + + self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(two_node.get_uri(), setup_py_uri) + self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri) self.failUnlessEqual(two_metadata["metakey"], "metavalue") + + self.failUnlessEqual(mut_node.get_uri(), mut_write_uri) + self.failUnlessEqual(mut_node.get_readonly_uri(), mut_read_uri) + self.failUnless(isinstance(mut_metadata, dict), mut_metadata) + + self.failUnless(fut_node.is_unknown()) + self.failUnlessEqual(fut_node.get_uri(), future_write_uri) + self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri) + self.failUnless(isinstance(fut_metadata, dict), fut_metadata) + + self.failUnless(fro_node.is_unknown()) + self.failUnlessEqual(fro_node.get_uri(), "ro." + future_read_uri) + self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri) + self.failUnless(isinstance(fro_metadata, dict), fro_metadata) d.addCallback(_check_kids) + d.addCallback(lambda ign: nm.create_new_mutable_directory(kids)) d.addCallback(lambda dn: dn.list()) d.addCallback(_check_kids) - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) - bad_kids1 = {u"one": (future_node, {})} + + bad_future_node = UnknownNode(future_write_uri, None) + bad_kids1 = {u"one": (bad_future_node, {})} d.addCallback(lambda ign: - self.shouldFail(AssertionError, "bad_kids1", - "does not accept UnknownNode", + self.shouldFail(MustNotBeUnknownRWError, "bad_kids1", + "cannot attach unknown", nm.create_new_mutable_directory, bad_kids1)) bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} @@ -91,17 +134,24 @@ nm = c.nodemaker setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" one_uri = "URI:LIT:n5xgk" # LIT for "one" - mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" - mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), + u"fut": (nm.create_from_cap(None, future_read_uri), {}), } d = c.create_immutable_dirnode(kids) + def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failUnless(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() @@ -109,50 +159,73 @@ self.cap = cap return dn.list() d.addCallback(_created) + def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) + self.failUnlessEqual(sorted(children.keys()), [u"fut", u"one", u"two"]) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] + fut_node, fut_metadata = children[u"fut"] + self.failUnlessEqual(one_node.get_size(), 3) - self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(one_node.get_uri(), one_uri) + self.failUnlessEqual(one_node.get_readonly_uri(), one_uri) self.failUnless(isinstance(one_metadata, dict), one_metadata) + + self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(two_node.get_uri(), setup_py_uri) + self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri) self.failUnlessEqual(two_metadata["metakey"], "metavalue") + + self.failUnless(fut_node.is_unknown()) + self.failUnlessEqual(fut_node.get_uri(), "imm." + future_read_uri) + self.failUnlessEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri) + self.failUnless(isinstance(fut_metadata, dict), fut_metadata) d.addCallback(_check_kids) + d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) d.addCallback(lambda dn: dn.list()) d.addCallback(_check_kids) - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) - bad_kids1 = {u"one": (future_node, {})} + + bad_future_node1 = UnknownNode(future_write_uri, None) + bad_kids1 = {u"one": (bad_future_node1, {})} d.addCallback(lambda ign: - self.shouldFail(AssertionError, "bad_kids1", - "does not accept UnknownNode", + self.shouldFail(MustNotBeUnknownRWError, "bad_kids1", + "cannot attach unknown", c.create_immutable_dirnode, bad_kids1)) - bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} + bad_future_node2 = UnknownNode(future_write_uri, future_read_uri) + bad_kids2 = {u"one": (bad_future_node2, {})} d.addCallback(lambda ign: - self.shouldFail(AssertionError, "bad_kids2", - "requires metadata to be a dict", + self.shouldFail(MustBeDeepImmutableError, "bad_kids2", + "is not immutable", c.create_immutable_dirnode, bad_kids2)) - bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})} + bad_kids3 = {u"one": (nm.create_from_cap(one_uri), None)} d.addCallback(lambda ign: - self.shouldFail(NotDeepImmutableError, "bad_kids3", - "is not immutable", + self.shouldFail(AssertionError, "bad_kids3", + "requires metadata to be a dict", c.create_immutable_dirnode, bad_kids3)) - bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})} + bad_kids4 = {u"one": (nm.create_from_cap(mut_write_uri), {})} d.addCallback(lambda ign: - self.shouldFail(NotDeepImmutableError, "bad_kids4", + self.shouldFail(MustBeDeepImmutableError, "bad_kids4", "is not immutable", c.create_immutable_dirnode, bad_kids4)) + bad_kids5 = {u"one": (nm.create_from_cap(mut_read_uri), {})} + d.addCallback(lambda ign: + self.shouldFail(MustBeDeepImmutableError, "bad_kids5", + "is not immutable", + c.create_immutable_dirnode, + bad_kids5)) d.addCallback(lambda ign: c.create_immutable_dirnode({})) def _created_empty(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failUnless(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() @@ -168,6 +241,9 @@ self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failUnless(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() @@ -193,9 +269,9 @@ d.addCallback(_check_kids) d.addCallback(lambda ign: n.get(u"subdir")) d.addCallback(lambda sd: self.failIf(sd.is_mutable())) - bad_kids = {u"one": (nm.create_from_cap(mut_writecap), {})} + bad_kids = {u"one": (nm.create_from_cap(mut_write_uri), {})} d.addCallback(lambda ign: - self.shouldFail(NotDeepImmutableError, "YZ", + self.shouldFail(MustBeDeepImmutableError, "YZ", "is not immutable", n.create_subdirectory, u"sub2", bad_kids, mutable=False)) @@ -203,7 +279,6 @@ d.addCallback(_made_parent) return d - def test_check(self): self.basedir = "dirnode/Dirnode/test_check" self.set_up_grid() @@ -337,24 +412,27 @@ ro_dn = c.create_node_from_uri(ro_uri) self.failUnless(ro_dn.is_readonly()) self.failUnless(ro_dn.is_mutable()) + self.failIf(ro_dn.is_unknown()) + self.failIf(ro_dn.is_allowed_in_immutable_directory()) + ro_dn.raise_error() - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.set_uri, u"newchild", filecap, filecap) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.set_node, u"newchild", filenode) - self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None, ro_dn.set_nodes, { u"newchild": (filenode, None) }) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.add_file, u"newchild", uploadable) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.delete, u"child") - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.create_subdirectory, u"newchild") - self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None, ro_dn.set_metadata_for, u"child", {}) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.move_child_to, u"child", rw_dn) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, rw_dn.move_child_to, u"child", ro_dn) return ro_dn.list() d.addCallback(_ready) @@ -901,8 +979,8 @@ nodemaker = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10}, None) - writecap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" - filenode = nodemaker.create_from_cap(writecap) + write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + filenode = nodemaker.create_from_cap(write_uri) node = dirnode.DirectoryNode(filenode, nodemaker, None) children = node._unpack_contents(known_tree) self._check_children(children) @@ -975,23 +1053,23 @@ self.failUnlessIn("lit", packed) kids = self._make_kids(nm, ["imm", "lit", "write"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) # read-only is not enough: all children must be immutable kids = self._make_kids(nm, ["imm", "lit", "read"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) kids = self._make_kids(nm, ["imm", "lit", "dirwrite"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) kids = self._make_kids(nm, ["imm", "lit", "dirread"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) @@ -1017,16 +1095,31 @@ def get_cap(self): return self.uri + def get_uri(self): return self.uri.to_string() + + def get_write_uri(self): + return self.uri.to_string() + def download_best_version(self): return defer.succeed(self.data) + def get_writekey(self): return "writekey" + def is_readonly(self): return False + def is_mutable(self): return True + + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return False + def modify(self, modifier): self.data = modifier(self.data, None, True) return defer.succeed(None) @@ -1050,47 +1143,59 @@ def test_from_future(self): # create a dirnode that contains unknown URI types, and make sure we - # tolerate them properly. Since dirnodes aren't allowed to add - # unknown node types, we have to be tricky. + # tolerate them properly. d = self.nodemaker.create_new_mutable_directory() - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_write_uri, future_read_uri) def _then(n): self._node = n return n.set_node(u"future", future_node) d.addCallback(_then) - # we should be prohibited from adding an unknown URI to a directory, - # since we don't know how to diminish the cap to a readcap (for the - # dirnode's rocap slot), and we don't want to accidentally grant - # write access to a holder of the dirnode's readcap. + # We should be prohibited from adding an unknown URI to a directory + # just in the rw_uri slot, since we don't know how to diminish the cap + # to a readcap (for the ro_uri slot). d.addCallback(lambda ign: - self.shouldFail(CannotPackUnknownNodeError, + self.shouldFail(MustNotBeUnknownRWError, "copy unknown", - "cannot pack unknown node as child add", + "cannot attach unknown rw cap as child", self._node.set_uri, u"add", - future_writecap, future_readcap)) + future_write_uri, None)) + + # However, we should be able to add both rw_uri and ro_uri as a pair of + # unknown URIs. + d.addCallback(lambda ign: self._node.set_uri(u"add-pair", + future_write_uri, future_read_uri)) + d.addCallback(lambda ign: self._node.list()) def _check(children): - self.failUnlessEqual(len(children), 1) + self.failUnlessEqual(len(children), 2) (fn, metadata) = children[u"future"] self.failUnless(isinstance(fn, UnknownNode), fn) - self.failUnlessEqual(fn.get_uri(), future_writecap) - self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) - # but we *should* be allowed to copy this node, because the + self.failUnlessEqual(fn.get_uri(), future_write_uri) + self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri) + + (fn2, metadata2) = children[u"add-pair"] + self.failUnless(isinstance(fn2, UnknownNode), fn2) + self.failUnlessEqual(fn2.get_uri(), future_write_uri) + self.failUnlessEqual(fn2.get_readonly_uri(), "ro." + future_read_uri) + + # we should also be allowed to copy this node, because the # UnknownNode contains all the information that was in the # original directory (readcap and writecap), so we're preserving # everything. return self._node.set_node(u"copy", fn) d.addCallback(_check) + d.addCallback(lambda ign: self._node.list()) def _check2(children): - self.failUnlessEqual(len(children), 2) + self.failUnlessEqual(len(children), 3) (fn, metadata) = children[u"copy"] self.failUnless(isinstance(fn, UnknownNode), fn) - self.failUnlessEqual(fn.get_uri(), future_writecap) - self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) + self.failUnlessEqual(fn.get_uri(), future_write_uri) + self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri) + d.addCallback(_check2) return d class DeepStats(unittest.TestCase): diff -rN -u old-tahoe/src/allmydata/test/test_filenode.py new-tahoe/src/allmydata/test/test_filenode.py --- old-tahoe/src/allmydata/test/test_filenode.py 2010-01-23 12:59:09.796000000 +0000 +++ new-tahoe/src/allmydata/test/test_filenode.py 2010-01-23 12:59:11.912000000 +0000 @@ -41,14 +41,21 @@ self.failUnlessEqual(fn1.get_readcap(), u) self.failUnlessEqual(fn1.is_readonly(), True) self.failUnlessEqual(fn1.is_mutable(), False) + self.failUnlessEqual(fn1.is_unknown(), False) + self.failUnlessEqual(fn1.is_allowed_in_immutable_directory(), True) + self.failUnlessEqual(fn1.get_write_uri(), None) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_size(), 1000) self.failUnlessEqual(fn1.get_storage_index(), u.storage_index) + fn1.raise_error() + fn2.raise_error() d = {} d[fn1] = 1 # exercise __hash__ v = fn1.get_verify_cap() self.failUnless(isinstance(v, uri.CHKFileVerifierURI)) self.failUnlessEqual(fn1.get_repair_cap(), v) + self.failUnlessEqual(v.is_readonly(), True) + self.failUnlessEqual(v.is_mutable(), False) def test_literal_filenode(self): @@ -64,9 +71,14 @@ self.failUnlessEqual(fn1.get_readcap(), u) self.failUnlessEqual(fn1.is_readonly(), True) self.failUnlessEqual(fn1.is_mutable(), False) + self.failUnlessEqual(fn1.is_unknown(), False) + self.failUnlessEqual(fn1.is_allowed_in_immutable_directory(), True) + self.failUnlessEqual(fn1.get_write_uri(), None) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_size(), len(DATA)) self.failUnlessEqual(fn1.get_storage_index(), None) + fn1.raise_error() + fn2.raise_error() d = {} d[fn1] = 1 # exercise __hash__ @@ -99,24 +111,29 @@ self.failUnlessEqual(n.get_writekey(), wk) self.failUnlessEqual(n.get_readkey(), rk) self.failUnlessEqual(n.get_storage_index(), si) - # these itmes are populated on first read (or create), so until that + # these items are populated on first read (or create), so until that # happens they'll be None self.failUnlessEqual(n.get_privkey(), None) self.failUnlessEqual(n.get_encprivkey(), None) self.failUnlessEqual(n.get_pubkey(), None) self.failUnlessEqual(n.get_uri(), u.to_string()) + self.failUnlessEqual(n.get_write_uri(), u.to_string()) self.failUnlessEqual(n.get_readonly_uri(), u.get_readonly().to_string()) self.failUnlessEqual(n.get_cap(), u) self.failUnlessEqual(n.get_readcap(), u.get_readonly()) self.failUnlessEqual(n.is_mutable(), True) self.failUnlessEqual(n.is_readonly(), False) + self.failUnlessEqual(n.is_unknown(), False) + self.failUnlessEqual(n.is_allowed_in_immutable_directory(), False) + n.raise_error() n2 = MutableFileNode(None, None, client.get_encoding_parameters(), None).init_from_cap(u) self.failUnlessEqual(n, n2) self.failIfEqual(n, "not even the right type") self.failIfEqual(n, u) # not the right class + n.raise_error() d = {n: "can these be used as dictionary keys?"} d[n2] = "replace the old one" self.failUnlessEqual(len(d), 1) @@ -127,12 +144,16 @@ self.failUnlessEqual(nro.get_readonly(), nro) self.failUnlessEqual(nro.get_cap(), u.get_readonly()) self.failUnlessEqual(nro.get_readcap(), u.get_readonly()) + self.failUnlessEqual(nro.is_mutable(), True) + self.failUnlessEqual(nro.is_readonly(), True) + self.failUnlessEqual(nro.is_unknown(), False) + self.failUnlessEqual(nro.is_allowed_in_immutable_directory(), False) nro_u = nro.get_uri() self.failUnlessEqual(nro_u, nro.get_readonly_uri()) self.failUnlessEqual(nro_u, u.get_readonly().to_string()) - self.failUnlessEqual(nro.is_mutable(), True) - self.failUnlessEqual(nro.is_readonly(), True) + self.failUnlessEqual(nro.get_write_uri(), None) self.failUnlessEqual(nro.get_repair_cap(), None) # RSAmut needs writecap + nro.raise_error() v = n.get_verify_cap() self.failUnless(isinstance(v, uri.SSKVerifierURI)) diff -rN -u old-tahoe/src/allmydata/test/test_system.py new-tahoe/src/allmydata/test/test_system.py --- old-tahoe/src/allmydata/test/test_system.py 2010-01-23 12:59:10.091000000 +0000 +++ new-tahoe/src/allmydata/test/test_system.py 2010-01-23 12:59:12.085000000 +0000 @@ -17,7 +17,7 @@ from allmydata.interfaces import IDirectoryNode, IFileNode, \ NoSuchChildError, NoSharesError from allmydata.monitor import Monitor -from allmydata.mutable.common import NotMutableError +from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from foolscap.api import DeadReferenceError from twisted.python.failure import Failure @@ -890,11 +890,11 @@ d1.addCallback(lambda res: dirnode.list()) d1.addCallback(self.log, "dirnode.list") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope")) d1.addCallback(self.log, "doing add_file(ro)") ut = upload.Data("I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence="99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut)) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut)) d1.addCallback(self.log, "doing get(ro)") d1.addCallback(lambda res: dirnode.get(u"mydata992")) @@ -902,17 +902,17 @@ self.failUnless(IFileNode.providedBy(filenode))) d1.addCallback(self.log, "doing delete(ro)") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "delete(nope)", None, dirnode.delete, u"mydata992")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "delete(nope)", None, dirnode.delete, u"mydata992")) - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "set_uri(nope)", None, dirnode.set_uri, u"hopeless", self.uri, self.uri)) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "set_uri(nope)", None, dirnode.set_uri, u"hopeless", self.uri, self.uri)) d1.addCallback(lambda res: self.shouldFail2(NoSuchChildError, "get(missing)", "missing", dirnode.get, u"missing")) personal = self._personal_node - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "mv from readonly", None, dirnode.move_child_to, u"mydata992", personal, u"nope")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mv from readonly", None, dirnode.move_child_to, u"mydata992", personal, u"nope")) d1.addCallback(self.log, "doing move_child_to(ro)2") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "mv to readonly", None, personal.move_child_to, u"sekrit data", dirnode, u"nope")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mv to readonly", None, personal.move_child_to, u"sekrit data", dirnode, u"nope")) d1.addCallback(self.log, "finished with _got_s2ro") return d1 diff -rN -u old-tahoe/src/allmydata/test/test_uri.py new-tahoe/src/allmydata/test/test_uri.py --- old-tahoe/src/allmydata/test/test_uri.py 2010-01-23 12:59:10.134000000 +0000 +++ new-tahoe/src/allmydata/test/test_uri.py 2010-01-23 12:59:12.094000000 +0000 @@ -3,7 +3,7 @@ from allmydata import uri from allmydata.util import hashutil, base32 from allmydata.interfaces import IURI, IFileURI, IDirnodeURI, IMutableFileURI, \ - IVerifierURI + IVerifierURI, CapConstraintError class Literal(unittest.TestCase): def _help_test(self, data): @@ -22,8 +22,16 @@ self.failIf(IDirnodeURI.providedBy(u2)) self.failUnlessEqual(u2.data, data) self.failUnlessEqual(u2.get_size(), len(data)) - self.failUnless(u.is_readonly()) - self.failIf(u.is_mutable()) + self.failUnless(u2.is_readonly()) + self.failIf(u2.is_mutable()) + + u2i = uri.from_string(u.to_string(), deep_immutable=True) + self.failUnless(IFileURI.providedBy(u2i)) + self.failIf(IDirnodeURI.providedBy(u2i)) + self.failUnlessEqual(u2i.data, data) + self.failUnlessEqual(u2i.get_size(), len(data)) + self.failUnless(u2i.is_readonly()) + self.failIf(u2i.is_mutable()) u3 = u.get_readonly() self.failUnlessIdentical(u, u3) @@ -51,18 +59,36 @@ fileURI = 'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834' chk1 = uri.CHKFileURI.init_from_string(fileURI) chk2 = uri.CHKFileURI.init_from_string(fileURI) + unk = uri.UnknownURI("lafs://from_the_future") self.failIfEqual(lit1, chk1) self.failUnlessEqual(chk1, chk2) self.failIfEqual(chk1, "not actually a URI") # these should be hashable too - s = set([lit1, chk1, chk2]) - self.failUnlessEqual(len(s), 2) # since chk1==chk2 + s = set([lit1, chk1, chk2, unk]) + self.failUnlessEqual(len(s), 3) # since chk1==chk2 def test_is_uri(self): lit1 = uri.LiteralFileURI("some data").to_string() self.failUnless(uri.is_uri(lit1)) self.failIf(uri.is_uri(None)) + def test_is_literal_file_uri(self): + lit1 = uri.LiteralFileURI("some data").to_string() + self.failUnless(uri.is_literal_file_uri(lit1)) + self.failIf(uri.is_literal_file_uri(None)) + self.failIf(uri.is_literal_file_uri("foo")) + self.failIf(uri.is_literal_file_uri("ro.foo")) + self.failIf(uri.is_literal_file_uri("URI:LITfoo")) + self.failUnless(uri.is_literal_file_uri("ro.URI:LIT:foo")) + self.failUnless(uri.is_literal_file_uri("imm.URI:LIT:foo")) + + def test_has_uri_prefix(self): + self.failUnless(uri.has_uri_prefix("URI:foo")) + self.failUnless(uri.has_uri_prefix("ro.URI:foo")) + self.failUnless(uri.has_uri_prefix("imm.URI:foo")) + self.failIf(uri.has_uri_prefix(None)) + self.failIf(uri.has_uri_prefix("foo")) + class CHKFile(unittest.TestCase): def test_pack(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -88,8 +114,7 @@ self.failUnless(IFileURI.providedBy(u)) self.failIf(IDirnodeURI.providedBy(u)) self.failUnlessEqual(u.get_size(), 1234) - self.failUnless(u.is_readonly()) - self.failIf(u.is_mutable()) + u_ro = u.get_readonly() self.failUnlessIdentical(u, u_ro) he = u.to_human_encoding() @@ -109,11 +134,19 @@ self.failUnless(IFileURI.providedBy(u2)) self.failIf(IDirnodeURI.providedBy(u2)) self.failUnlessEqual(u2.get_size(), 1234) - self.failUnless(u2.is_readonly()) - self.failIf(u2.is_mutable()) + + u2i = uri.from_string(u.to_string(), deep_immutable=True) + self.failUnlessEqual(u.to_string(), u2i.to_string()) + u2ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u.to_string()) + self.failUnlessEqual(u.to_string(), u2ro.to_string()) + u2imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u.to_string()) + self.failUnlessEqual(u.to_string(), u2imm.to_string()) v = u.get_verify_cap() self.failUnless(isinstance(v.to_string(), str)) + self.failUnless(v.is_readonly()) + self.failIf(v.is_mutable()) + v2 = uri.from_string(v.to_string()) self.failUnlessEqual(v, v2) he = v.to_human_encoding() @@ -126,6 +159,8 @@ total_shares=10, size=1234) self.failUnless(isinstance(v3.to_string(), str)) + self.failUnless(v3.is_readonly()) + self.failIf(v3.is_mutable()) def test_pack_badly(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -179,13 +214,20 @@ self.failUnlessEqual(readable["UEB_hash"], base32.b2a(hashutil.uri_extension_hash(ext))) -class Invalid(unittest.TestCase): +class Unknown(unittest.TestCase): def test_from_future(self): # any URI type that we don't recognize should be treated as unknown future_uri = "I am a URI from the future. Whatever you do, don't " u = uri.from_string(future_uri) self.failUnless(isinstance(u, uri.UnknownURI)) self.failUnlessEqual(u.to_string(), future_uri) + self.failUnless(u.get_readonly() is None) + self.failUnless(u.get_error() is None) + + u2 = uri.UnknownURI(future_uri, error=CapConstraintError("...")) + self.failUnlessEqual(u.to_string(), future_uri) + self.failUnless(u2.get_readonly() is None) + self.failUnless(isinstance(u2.get_error(), CapConstraintError)) class Constraint(unittest.TestCase): def test_constraint(self): @@ -226,6 +268,13 @@ self.failUnless(IMutableFileURI.providedBy(u2)) self.failIf(IDirnodeURI.providedBy(u2)) + u2i = uri.from_string(u.to_string(), deep_immutable=True) + self.failUnless(isinstance(u2i, uri.UnknownURI), u2i) + u2ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u.to_string()) + self.failUnless(isinstance(u2ro, uri.UnknownURI), u2ro) + u2imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u.to_string()) + self.failUnless(isinstance(u2imm, uri.UnknownURI), u2imm) + u3 = u2.get_readonly() readkey = hashutil.ssk_readkey_hash(writekey) self.failUnlessEqual(u3.fingerprint, fingerprint) @@ -236,6 +285,13 @@ self.failUnless(IMutableFileURI.providedBy(u3)) self.failIf(IDirnodeURI.providedBy(u3)) + u3i = uri.from_string(u3.to_string(), deep_immutable=True) + self.failUnless(isinstance(u3i, uri.UnknownURI), u3i) + u3ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u3.to_string()) + self.failUnlessEqual(u3.to_string(), u3ro.to_string()) + u3imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u3.to_string()) + self.failUnless(isinstance(u3imm, uri.UnknownURI), u3imm) + he = u3.to_human_encoding() u3_h = uri.ReadonlySSKFileURI.init_from_human_encoding(he) self.failUnlessEqual(u3, u3_h) @@ -249,6 +305,13 @@ self.failUnless(IMutableFileURI.providedBy(u4)) self.failIf(IDirnodeURI.providedBy(u4)) + u4i = uri.from_string(u4.to_string(), deep_immutable=True) + self.failUnless(isinstance(u4i, uri.UnknownURI), u4i) + u4ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u4.to_string()) + self.failUnlessEqual(u4.to_string(), u4ro.to_string()) + u4imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u4.to_string()) + self.failUnless(isinstance(u4imm, uri.UnknownURI), u4imm) + u4a = uri.from_string(u4.to_string()) self.failUnlessEqual(u4a, u4) self.failUnless("ReadonlySSKFileURI" in str(u4a)) @@ -291,12 +354,19 @@ self.failIf(IFileURI.providedBy(u2)) self.failUnless(IDirnodeURI.providedBy(u2)) + u2i = uri.from_string(u1.to_string(), deep_immutable=True) + self.failUnless(isinstance(u2i, uri.UnknownURI)) + u3 = u2.get_readonly() self.failUnless(u3.is_readonly()) self.failUnless(u3.is_mutable()) self.failUnless(IURI.providedBy(u3)) self.failIf(IFileURI.providedBy(u3)) self.failUnless(IDirnodeURI.providedBy(u3)) + + u3i = uri.from_string(u2.to_string(), deep_immutable=True) + self.failUnless(isinstance(u3i, uri.UnknownURI)) + u3n = u3._filenode_uri self.failUnless(u3n.is_readonly()) self.failUnless(u3n.is_mutable()) @@ -363,10 +433,16 @@ self.failIf(IFileURI.providedBy(u2)) self.failUnless(IDirnodeURI.providedBy(u2)) + u2i = uri.from_string(u1.to_string(), deep_immutable=True) + self.failUnlessEqual(u1.to_string(), u2i.to_string()) + u3 = u2.get_readonly() self.failUnlessEqual(u3.to_string(), u2.to_string()) self.failUnless(str(u3)) + u3i = uri.from_string(u2.to_string(), deep_immutable=True) + self.failUnlessEqual(u2.to_string(), u3i.to_string()) + u2_verifier = u2.get_verify_cap() self.failUnless(isinstance(u2_verifier, uri.ImmutableDirectoryURIVerifier), diff -rN -u old-tahoe/src/allmydata/test/test_web.py new-tahoe/src/allmydata/test/test_web.py --- old-tahoe/src/allmydata/test/test_web.py 2010-01-23 12:59:10.149000000 +0000 +++ new-tahoe/src/allmydata/test/test_web.py 2010-01-23 12:59:12.131000000 +0000 @@ -7,7 +7,7 @@ from twisted.web import client, error, http from twisted.python import failure, log from nevow import rend -from allmydata import interfaces, uri, webish +from allmydata import interfaces, uri, webish, dirnode from allmydata.storage.shares import get_share_file from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import upload, download @@ -18,6 +18,7 @@ from allmydata.scripts.debug import CorruptShareOptions, corrupt_share from allmydata.util import fileutil, base32 from allmydata.util.consumer import download_to_data +from allmydata.util.netstring import split_netstring from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \ create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri from allmydata.interfaces import IMutableFileNode @@ -366,25 +367,101 @@ self.fail("%s was supposed to Error(404), not get '%s'" % (which, res)) + def _dump_res(self, res): + import traceback + s = "%r\n" % (res,) + if hasattr(res, 'tb_frame'): + s += "Traceback:\n%s\n" % (traceback.format_tb(res),) + if hasattr(res, 'value'): + s += "%r\n" % (res.value,) + if hasattr(res.value, 'tb_frame'): + s += "Traceback:\n%s\n" % (res, res.value, traceback.format_tb(res)) + if hasattr(res.value, 'response'): + s += "Response body:\n%s\n" % (res.value.response,) + return s + + def shouldSucceedGET(self, urlpath, followRedirect=False, + expected_statuscode=http.OK, return_response=False, **kwargs): + d = self.GET(urlpath, followRedirect=followRedirect, return_response=True, **kwargs) + def done((res, statuscode, headers)): + if isinstance(res, failure.Failure): + self.fail(("'GET %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it failed with statuscode %s instead.\n" + "%s\nThe response headers were:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, + self._dump_res(res), headers)) + if str(statuscode) != str(expected_statuscode): + self.fail(("'GET %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it succeeded with statuscode %s instead.\n" + "The response headers were:\n%s\n\n" + "The response body was:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, headers, res)) + if return_response: + return (res, statuscode, headers) + else: + return res + d.addBoth(done) + return d + + def shouldSucceedHEAD(self, urlpath, expected_statuscode=http.OK, + return_response=False, **kwargs): + d = self.HEAD(urlpath, return_response=True, **kwargs) + def done((res, statuscode, headers)): + if isinstance(res, failure.Failure): + self.fail(("'HEAD %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it failed with statuscode %s instead.\n" + "%s\nThe response headers were:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, + self._dump_res(res), headers)) + if str(statuscode) != str(expected_statuscode): + self.fail(("'HEAD %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it succeeded with statuscode %s instead.\n" + "The response headers were:\n%s\n\n" + "The response body was:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, headers, res)) + if return_response: + return (res, statuscode, headers) + else: + return res + d.addBoth(done) + return d + + def shouldSucceed(self, which, expected_statuscode, callable, *args, **kwargs): + d = defer.maybeDeferred(callable, *args, **kwargs) + def done(res): + if isinstance(res, failure.Failure): + self.fail(("%s:\nAn HTTP op with args %r and kwargs %r was supposed to " + "succeed with statuscode %s, but it failed:\n%s") % ( + which, args, kwargs, expected_statuscode, + self._dump_res(res))) + #if str(statuscode) != str(expected_statuscode): + # self.fail(("%s:\nAn HTTP op with args %r and kwargs %r was supposed to " + # "succeed with statuscode %s, but it succeeded with statuscode %s instead.\n" + # "The response body was:\n%s") % ( + # which, args, kwargs, expected_statuscode, statuscode, res)) + return res + d.addBoth(done) + return d + class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): def test_create(self): pass def test_welcome(self): - d = self.GET("/") + d = self.shouldSucceedGET("/") def _check(res): self.failUnless('Welcome To Tahoe-LAFS' in res, res) self.s.basedir = 'web/test_welcome' fileutil.make_dirs("web/test_welcome") fileutil.make_dirs("web/test_welcome/private") - return self.GET("/") + return self.shouldSucceedGET("/") d.addCallback(_check) return d def test_provisioning(self): - d = self.GET("/provisioning/") + d = self.shouldSucceedGET("/provisioning/") def _check(res): self.failUnless('Tahoe Provisioning Tool' in res) fields = {'filled': True, @@ -400,9 +477,10 @@ "delete_rate": 10, "lease_timer": 7, } - return self.POST("/provisioning/", **fields) - + return self.shouldSucceed("POST_provisioning-1", http.OK, self.POST, + "/provisioning/", **fields) d.addCallback(_check) + def _check2(res): self.failUnless('Tahoe Provisioning Tool' in res) self.failUnless("Share space consumed: 167.01TB" in res) @@ -422,13 +500,17 @@ "delete_rate": 100, "lease_timer": 7, } - return self.POST("/provisioning/", **fields) + return self.shouldSucceed("POST_provisioning-2", http.OK, self.POST, + "/provisioning/", **fields) d.addCallback(_check2) + def _check3(res): self.failUnless("Share space consumed: huge!" in res) fields = {'filled': True} - return self.POST("/provisioning/", **fields) + return self.shouldSucceed("POST_provisioning-3", http.OK, self.POST, + "/provisioning/", **fields) d.addCallback(_check3) + def _check4(res): self.failUnless("Share space consumed:" in res) d.addCallback(_check4) @@ -442,7 +524,7 @@ except: raise unittest.SkipTest("reliability tool requires NumPy") - d = self.GET("/reliability/") + d = self.shouldSucceedGET("/reliability/") def _check(res): self.failUnless('Tahoe Reliability Tool' in res) fields = {'drive_lifetime': "8Y", @@ -471,7 +553,7 @@ mu_num = h.list_all_mapupdate_statuses()[0].get_counter() pub_num = h.list_all_publish_statuses()[0].get_counter() ret_num = h.list_all_retrieve_statuses()[0].get_counter() - d = self.GET("/status", followRedirect=True) + d = self.shouldSucceedGET("/status", followRedirect=True) def _check(res): self.failUnless('Upload and Download Status' in res, res) self.failUnless('"down-%d"' % dl_num in res, res) @@ -480,7 +562,7 @@ self.failUnless('"publish-%d"' % pub_num in res, res) self.failUnless('"retrieve-%d"' % ret_num in res, res) d.addCallback(_check) - d.addCallback(lambda res: self.GET("/status/?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET("/status/?t=json")) def _check_json(res): data = simplejson.loads(res) self.failUnless(isinstance(data, dict)) @@ -489,23 +571,23 @@ # here. d.addCallback(_check_json) - d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/down-%d" % dl_num)) def _check_dl(res): self.failUnless("File Download Status" in res, res) d.addCallback(_check_dl) - d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/up-%d" % ul_num)) def _check_ul(res): self.failUnless("File Upload Status" in res, res) d.addCallback(_check_ul) - d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/mapupdate-%d" % mu_num)) def _check_mapupdate(res): self.failUnless("Mutable File Servermap Update Status" in res, res) d.addCallback(_check_mapupdate) - d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/publish-%d" % pub_num)) def _check_publish(res): self.failUnless("Mutable File Publish Status" in res, res) d.addCallback(_check_publish) - d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/retrieve-%d" % ret_num)) def _check_retrieve(res): self.failUnless("Mutable File Retrieve Status" in res, res) d.addCallback(_check_retrieve) @@ -536,16 +618,15 @@ self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps") def test_GET_FILEURL(self): - d = self.GET(self.public_url + "/foo/bar.txt") + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt") d.addCallback(self.failUnlessIsBarDotTxt) return d def test_GET_FILEURL_range(self): headers = {"range": "bytes=1-10"} - d = self.GET(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): - self.failUnlessEqual(int(status), 206) + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 1-10/%d" % len(self.BAR_CONTENTS)) @@ -556,10 +637,9 @@ def test_GET_FILEURL_partial_range(self): headers = {"range": "bytes=5-"} length = len(self.BAR_CONTENTS) - d = self.GET(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): - self.failUnlessEqual(int(status), 206) + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 5-%d/%d" % (length-1, length)) @@ -569,11 +649,10 @@ def test_HEAD_FILEURL_range(self): headers = {"range": "bytes=1-10"} - d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): + d = self.shouldSucceedHEAD(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnlessEqual(res, "") - self.failUnlessEqual(int(status), 206) self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 1-10/%d" % len(self.BAR_CONTENTS)) @@ -583,10 +662,9 @@ def test_HEAD_FILEURL_partial_range(self): headers = {"range": "bytes=5-"} length = len(self.BAR_CONTENTS) - d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): - self.failUnlessEqual(int(status), 206) + d = self.shouldSucceedHEAD(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 5-%d/%d" % (length-1, length)) @@ -595,7 +673,7 @@ def test_GET_FILEURL_range_bad(self): headers = {"range": "BOGUS=fizbop-quarnak"} - d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad", + d = self.shouldFail2(error.Error, "GET_FILEURL_range_bad", "400 Bad Request", "Syntactically invalid http range header", self.GET, self.public_url + "/foo/bar.txt", @@ -603,8 +681,9 @@ return d def test_HEAD_FILEURL(self): - d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True) - def _got((res, status, headers)): + d = self.shouldSucceedHEAD(self.public_url + "/foo/bar.txt", + expected_statuscode=http.OK, return_response=True) + def _got((res, statuscode, headers)): self.failUnlessEqual(res, "") self.failUnlessEqual(headers["content-length"][0], str(len(self.BAR_CONTENTS))) @@ -615,27 +694,27 @@ def test_GET_FILEURL_named(self): base = "/file/%s" % urllib.quote(self._bar_txt_uri) base2 = "/named/%s" % urllib.quote(self._bar_txt_uri) - d = self.GET(base + "/@@name=/blah.txt") + d = self.shouldSucceedGET(base + "/@@name=/blah.txt") d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base + "/blah.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base + "/blah.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base + "/ignore/lots/blah.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base2 + "/@@name=/blah.txt")) d.addCallback(self.failUnlessIsBarDotTxt) save_url = base + "?save=true&filename=blah.txt" - d.addCallback(lambda res: self.GET(save_url)) + d.addCallback(lambda res: self.shouldSucceedGET(save_url)) d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t u_fn_e = urllib.quote(u_filename.encode("utf-8")) u_url = base + "?save=true&filename=" + u_fn_e - d.addCallback(lambda res: self.GET(u_url)) + d.addCallback(lambda res: self.shouldSucceedGET(u_url)) d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers return d def test_PUT_FILEURL_named_bad(self): base = "/file/%s" % urllib.quote(self._bar_txt_uri) - d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad", + d = self.shouldFail2(error.Error, "PUT_FILEURL_named_bad", "400 Bad Request", "/file can only be used with GET or HEAD", self.PUT, base + "/@@name=/blah.txt", "") @@ -643,14 +722,14 @@ def test_GET_DIRURL_named_bad(self): base = "/file/%s" % urllib.quote(self._foo_uri) - d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad", + d = self.shouldFail2(error.Error, "PUT_DIRURL_named_bad", "400 Bad Request", "is not a file-cap", self.GET, base + "/@@name=/blah.txt") return d def test_GET_slash_file_bad(self): - d = self.shouldFail2(error.Error, "test_GET_slash_file_bad", + d = self.shouldFail2(error.Error, "GET_slash_file_bad", "404 Not Found", "/file must be followed by a file-cap and a name", self.GET, "/file") @@ -671,7 +750,7 @@ verifier_cap = n.get_verify_cap().to_string() base = "/uri/%s" % urllib.quote(verifier_cap) # client.create_node_from_uri() can't handle verify-caps - d = self.shouldFail2(error.Error, "test_GET_unhandled_URI", + d = self.shouldFail2(error.Error, "GET_unhandled_URI", "400 Bad Request", "GET unknown URI type: can only do t=info", self.GET, base) @@ -679,14 +758,14 @@ def test_GET_FILE_URI(self): base = "/uri/%s" % urllib.quote(self._bar_txt_uri) - d = self.GET(base) + d = self.shouldSucceedGET(base) d.addCallback(self.failUnlessIsBarDotTxt) return d def test_GET_FILE_URI_badchild(self): base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri) errmsg = "Files have no children, certainly not named 'boguschild'" - d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild", + d = self.shouldFail2(error.Error, "GET_FILE_URI_badchild", "400 Bad Request", errmsg, self.GET, base) return d @@ -694,35 +773,42 @@ def test_PUT_FILE_URI_badchild(self): base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri) errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory" - d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild", + d = self.shouldFail2(error.Error, "GET_FILE_URI_badchild", "400 Bad Request", errmsg, self.PUT, base, "") return d + # TODO: version of this with a Unicode filename def test_GET_FILEURL_save(self): - d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true") - # TODO: look at the headers, expect a Content-Disposition: attachment - # header. - d.addCallback(self.failUnlessIsBarDotTxt) + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true", + return_response=True) + def _got((res, statuscode, headers)): + content_disposition = headers["content-disposition"][0] + self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition) + self.failUnlessIsBarDotTxt(res) + d.addCallback(_got) return d def test_GET_FILEURL_missing(self): d = self.GET(self.public_url + "/foo/missing") - d.addBoth(self.should404, "test_GET_FILEURL_missing") + d.addBoth(self.should404, "GET_FILEURL_missing") return d def test_PUT_overwrite_only_files(self): # create a directory, put a file in that directory. contents, n, filecap = self.makefile(8) - d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "") + d = self.shouldSucceed("PUT_overwrite_only_files_1", http.OK, self.PUT, + self.public_url + "/foo/dir?t=mkdir", "") d.addCallback(lambda res: - self.PUT(self.public_url + "/foo/dir/file1.txt", - self.NEWFILE_CONTENTS)) + self.shouldSucceed("PUT_overwrite_only_files_2", http.OK, self.PUT, + self.public_url + "/foo/dir/file1.txt", + self.NEWFILE_CONTENTS)) # try to overwrite the file with replace=only-files # (this should work) d.addCallback(lambda res: - self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files", - filecap)) + self.shouldSucceed("PUT_overwrite_only_files_3", http.OK, self.PUT, + self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files", + filecap)) d.addCallback(lambda res: self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict", "There was already a child by that name, and you asked me " @@ -732,21 +818,19 @@ return d def test_PUT_NEWFILEURL(self): - d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 201) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d = self.shouldSucceed("PUT_NEWFILEURL", http.CREATED, self.PUT, + self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", self.NEWFILE_CONTENTS)) return d def test_PUT_NEWFILEURL_not_mutable(self): - d = self.PUT(self.public_url + "/foo/new.txt?mutable=false", - self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 201) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d = self.shouldSucceed("PUT_NEWFILEURL_not_mutable", http.CREATED, self.PUT, + self.public_url + "/foo/new.txt?mutable=false", + self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", self.NEWFILE_CONTENTS)) @@ -755,7 +839,7 @@ def test_PUT_NEWFILEURL_range_bad(self): headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)} target = self.public_url + "/foo/new.txt" - d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad", + d = self.shouldFail2(error.Error, "PUT_NEWFILEURL_range_bad", "501 Not Implemented", "Content-Range in PUT not yet supported", # (and certainly not for immutable files) @@ -766,17 +850,16 @@ return d def test_PUT_NEWFILEURL_mutable(self): - d = self.PUT(self.public_url + "/foo/new.txt?mutable=true", - self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 201) + d = self.shouldSucceed("PUT_NEWFILEURL_mutable", http.CREATED, self.PUT, + self.public_url + "/foo/new.txt?mutable=true", + self.NEWFILE_CONTENTS) def _check_uri(res): u = uri.from_string_mutable_filenode(res) self.failUnless(u.is_mutable()) self.failIf(u.is_readonly()) return res d.addCallback(_check_uri) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(self._foo_node, u"new.txt", @@ -784,7 +867,7 @@ return d def test_PUT_NEWFILEURL_mutable_toobig(self): - d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig", + d = self.shouldFail2(error.Error, "PUT_NEWFILEURL_mutable_toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.PUT, @@ -793,10 +876,9 @@ return d def test_PUT_NEWFILEURL_replace(self): - d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 200) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt") + d = self.shouldSucceed("PUT_NEWFILEURL_replace", http.OK, self.PUT, + self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"bar.txt", self.NEWFILE_CONTENTS)) @@ -819,9 +901,11 @@ return d def test_PUT_NEWFILEURL_mkdirs(self): - d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS) + d = self.shouldSucceed("PUT_NEWFILEURL_mkdirs", http.OK, self.PUT, + self.public_url + "/foo/newdir/new.txt", + self.NEWFILE_CONTENTS) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt") + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt") d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt")) d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir")) d.addCallback(lambda res: @@ -839,26 +923,27 @@ def test_PUT_NEWFILEURL_emptyname(self): # an empty pathname component (i.e. a double-slash) is disallowed - d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname", + d = self.shouldFail2(error.Error, "PUT_NEWFILEURL_emptyname", "400 Bad Request", "The webapi does not allow empty pathname components", self.PUT, self.public_url + "/foo//new.txt", "") return d def test_DELETE_FILEURL(self): - d = self.DELETE(self.public_url + "/foo/bar.txt") + d = self.shouldSucceed("DELETE_FILEURL", http.OK, self.DELETE, + self.public_url + "/foo/bar.txt") d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) return d def test_DELETE_FILEURL_missing(self): d = self.DELETE(self.public_url + "/foo/missing") - d.addBoth(self.should404, "test_DELETE_FILEURL_missing") + d.addBoth(self.should404, "DELETE_FILEURL_missing") return d def test_DELETE_FILEURL_missing2(self): d = self.DELETE(self.public_url + "/missing/missing") - d.addBoth(self.should404, "test_DELETE_FILEURL_missing2") + d.addBoth(self.should404, "DELETE_FILEURL_missing2") return d def failUnlessHasBarDotTxtMetadata(self, res): @@ -875,7 +960,7 @@ # I can't do "GET /path?json", I have to do "GET /path/t=json" # instead. This may make it tricky to emulate the S3 interface # completely. - d = self.GET(self.public_url + "/foo/bar.txt?t=json") + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=json") def _check1(data): self.failUnlessIsBarJSON(data) self.failUnlessHasBarDotTxtMetadata(data) @@ -885,16 +970,16 @@ def test_GET_FILEURL_json_missing(self): d = self.GET(self.public_url + "/foo/missing?json") - d.addBoth(self.should404, "test_GET_FILEURL_json_missing") + d.addBoth(self.should404, "GET_FILEURL_json_missing") return d def test_GET_FILEURL_uri(self): - d = self.GET(self.public_url + "/foo/bar.txt?t=uri") + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=uri") def _check(res): self.failUnlessEqual(res, self._bar_txt_uri) d.addCallback(_check) d.addCallback(lambda res: - self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri")) + self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=readonly-uri")) def _check2(res): # for now, for files, uris and readonly-uris are the same self.failUnlessEqual(res, self._bar_txt_uri) @@ -910,14 +995,14 @@ def test_GET_FILEURL_uri_missing(self): d = self.GET(self.public_url + "/foo/missing?t=uri") - d.addBoth(self.should404, "test_GET_FILEURL_uri_missing") + d.addBoth(self.should404, "GET_FILEURL_uri_missing") return d def test_GET_DIRURL(self): # the addSlash means we get a redirect here # from /uri/$URI/foo/ , we need ../../../ to get back to the root ROOT = "../../.." - d = self.GET(self.public_url + "/foo", followRedirect=True) + d = self.shouldSucceedGET(self.public_url + "/foo", followRedirect=True) def _check(res): self.failUnless(('Return to Welcome page' % ROOT) in res, res) @@ -954,9 +1039,9 @@ self.failUnless(re.search(get_sub, res), res) d.addCallback(_check) - # look at a directory which is readonly + # look at a readonly directory d.addCallback(lambda res: - self.GET(self.public_url + "/reedownlee", followRedirect=True)) + self.shouldSucceedGET(self.public_url + "/reedownlee", followRedirect=True)) def _check2(res): self.failUnless("(read-only)" in res, res) self.failIf("Upload a file" in res, res) @@ -964,14 +1049,14 @@ # and at a directory that contains a readonly directory d.addCallback(lambda res: - self.GET(self.public_url, followRedirect=True)) + self.shouldSucceedGET(self.public_url, followRedirect=True)) def _check3(res): self.failUnless(re.search('DIR-RO' r'\s+reedownlee', res), res) d.addCallback(_check3) # and an empty directory - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty/")) def _check4(res): self.failUnless("directory is empty" in res, res) MKDIR_BUTTON_RE=re.compile('.*Create a new directory in this directory.*', re.I) @@ -981,7 +1066,7 @@ return d def test_GET_DIRURL_badtype(self): - d = self.shouldHTTPError("test_GET_DIRURL_badtype", + d = self.shouldHTTPError("GET_DIRURL_badtype", 400, "Bad Request", "bad t=bogus", self.GET, @@ -989,14 +1074,14 @@ return d def test_GET_DIRURL_json(self): - d = self.GET(self.public_url + "/foo?t=json") + d = self.shouldSucceedGET(self.public_url + "/foo?t=json") d.addCallback(self.failUnlessIsFooJSON) return d def test_POST_DIRURL_manifest_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_manifest_no_ophandle", + "POST_DIRURL_manifest_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-manifest") @@ -1005,8 +1090,9 @@ def test_POST_DIRURL_manifest(self): d = defer.succeed(None) def getman(ignored, output): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125", - followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_manifest", http.OK, self.POST, + self.public_url + "/foo/?t=start-manifest&ophandle=125", + followRedirect=True) d.addCallback(self.wait_for_operation, "125") d.addCallback(self.get_operation_results, "125", output) return d @@ -1019,7 +1105,7 @@ d.addCallback(_got_html) # both t=status and unadorned GET should be identical - d.addCallback(lambda res: self.GET("/operations/125")) + d.addCallback(lambda res: self.shouldSucceedGET("/operations/125")) d.addCallback(_got_html) d.addCallback(getman, "html") @@ -1047,15 +1133,16 @@ def test_POST_DIRURL_deepsize_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_deepsize_no_ophandle", + "POST_DIRURL_deepsize_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-deep-size") return d def test_POST_DIRURL_deepsize(self): - d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126", - followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_deepsize", http.OK, self.POST, + self.public_url + "/foo/?t=start-deep-size&ophandle=126", + followRedirect=True) d.addCallback(self.wait_for_operation, "126") d.addCallback(self.get_operation_results, "126", "json") def _got_json(data): @@ -1075,15 +1162,16 @@ def test_POST_DIRURL_deepstats_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_deepstats_no_ophandle", + "POST_DIRURL_deepstats_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-deep-stats") return d def test_POST_DIRURL_deepstats(self): - d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127", - followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_deepstats", http.OK, self.POST, + self.public_url + "/foo/?t=start-deep-stats&ophandle=127", + followRedirect=True) d.addCallback(self.wait_for_operation, "127") d.addCallback(self.get_operation_results, "127", "json") def _got_json(stats): @@ -1109,7 +1197,8 @@ return d def test_POST_DIRURL_stream_manifest(self): - d = self.POST(self.public_url + "/foo/?t=stream-manifest") + d = self.shouldSucceed("POST_DIRURL_stream_manifest", http.OK, self.POST, + self.public_url + "/foo/?t=stream-manifest") def _check(res): self.failUnless(res.endswith("\n")) units = [simplejson.loads(t) for t in res[:-1].split("\n")] @@ -1129,21 +1218,22 @@ return d def test_GET_DIRURL_uri(self): - d = self.GET(self.public_url + "/foo?t=uri") + d = self.shouldSucceedGET(self.public_url + "/foo?t=uri") def _check(res): self.failUnlessEqual(res, self._foo_uri) d.addCallback(_check) return d def test_GET_DIRURL_readonly_uri(self): - d = self.GET(self.public_url + "/foo?t=readonly-uri") + d = self.shouldSucceedGET(self.public_url + "/foo?t=readonly-uri") def _check(res): self.failUnlessEqual(res, self._foo_readonly_uri) d.addCallback(_check) return d def test_PUT_NEWDIRURL(self): - d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.shouldSucceed("PUT_NEWDIRURL", http.OK, self.PUT, + self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -1151,7 +1241,8 @@ return d def test_POST_NEWDIRURL(self): - d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.shouldSucceed("POST_NEWDIRURL", http.OK, self.POST2, + self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -1160,30 +1251,41 @@ def test_POST_NEWDIRURL_emptyname(self): # an empty pathname component (i.e. a double-slash) is disallowed - d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname", + d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname", "400 Bad Request", "The webapi does not allow empty pathname components, i.e. a double slash", self.POST, self.public_url + "//?t=mkdir") return d def test_POST_NEWDIRURL_initial_children(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_initial_children() + d = self.shouldSucceed("POST_NEWDIRURL_initial_children", http.OK, self.POST2, + self.public_url + "/foo/newdir?t=mkdir-with-children", + simplejson.dumps(newkids)) def _check(uri): n = self.s.create_node_from_uri(uri.strip()) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"child-mutable", + caps['filecap2'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"child-mutable-ro", + caps['filecap3'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable", - filecap2)) + self.failUnlessROChildURIIs(n, u"unknownchild-ro", + caps['unknown_rocap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable-ro", - filecap3)) + self.failUnlessRWChildURIIs(n, u"unknownchild-rw", + caps['unknown_rwcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild", dircap)) + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"dirchild", + caps['dircap'])) return d2 d.addCallback(_check) d.addCallback(lambda res: @@ -1191,21 +1293,26 @@ d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) return d def test_POST_NEWDIRURL_immutable(self): - (newkids, filecap1, immdircap) = self._create_immutable_children() - d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_immutable_children() + d = self.shouldSucceed("POST_NEWDIRURL_immutable", http.OK, self.POST2, + self.public_url + "/foo/newdir?t=mkdir-immutable", + simplejson.dumps(newkids)) def _check(uri): n = self.s.create_node_from_uri(uri.strip()) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild-imm", - immdircap)) + self.failUnlessROChildURIIs(n, u"dirchild-imm", + caps['immdircap'])) return d2 d.addCallback(_check) d.addCallback(lambda res: @@ -1213,25 +1320,27 @@ d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap']) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap']) d.addErrback(self.explain_web_error) return d def test_POST_NEWDIRURL_immutable_bad(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad", + (newkids, caps) = self._create_initial_children() + d = self.shouldFail2(error.Error, "POST_NEWDIRURL_immutable_bad", "400 Bad Request", - "a mkdir-immutable operation was given a child that was not itself immutable", + "needed to be immutable but was not", self.POST2, self.public_url + "/foo/newdir?t=mkdir-immutable", simplejson.dumps(newkids)) return d def test_PUT_NEWDIRURL_exists(self): - d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "") + d = self.shouldSucceed("PUT_NEWDIRURL_exists", http.OK, self.PUT, + self.public_url + "/foo/sub?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"sub")) d.addCallback(lambda res: self._foo_node.get(u"sub")) @@ -1249,18 +1358,21 @@ d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"]) return d - def test_PUT_NEWDIRURL_mkdir_p(self): + def test_POST_NEWDIRURL_mkdir_p(self): d = defer.succeed(None) - d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp')) + d.addCallback(lambda res: self.shouldSucceed("POST_NEWDIRURL_mkdir_p-1", http.OK, self.POST, + self.public_url + "/foo", t='mkdir', name='mkp')) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp")) d.addCallback(lambda res: self._foo_node.get(u"mkp")) def mkdir_p(mkpnode): url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri()) - d = self.POST(url) + d = self.shouldSucceed("POST_NEWDIRURL_mkdir_p-2", http.OK, self.POST, + url) def made_subsub(ssuri): d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2") d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri)) - d = self.POST(url) + d = self.shouldSucceed("POST_NEWDIRURL_mkdir_p-3", http.OK, self.POST, + url) d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri)) return d d.addCallback(made_subsub) @@ -1269,7 +1381,8 @@ return d def test_PUT_NEWDIRURL_mkdirs(self): - d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "") + d = self.shouldSucceed("PUT_NEWDIRURL_mkdirs", http.OK, self.PUT, + self.public_url + "/foo/subdir/newdir?t=mkdir", "") d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: @@ -1280,21 +1393,22 @@ return d def test_DELETE_DIRURL(self): - d = self.DELETE(self.public_url + "/foo") + d = self.shouldSucceed("DELETE_DIRURL", http.OK, self.DELETE, + self.public_url + "/foo") d.addCallback(lambda res: self.failIfNodeHasChild(self.public_root, u"foo")) return d def test_DELETE_DIRURL_missing(self): d = self.DELETE(self.public_url + "/foo/missing") - d.addBoth(self.should404, "test_DELETE_DIRURL_missing") + d.addBoth(self.should404, "DELETE_DIRURL_missing") d.addCallback(lambda res: self.failUnlessNodeHasChild(self.public_root, u"foo")) return d def test_DELETE_DIRURL_missing2(self): d = self.DELETE(self.public_url + "/missing") - d.addBoth(self.should404, "test_DELETE_DIRURL_missing2") + d.addBoth(self.should404, "DELETE_DIRURL_missing2") return d def dump_root(self): @@ -1346,18 +1460,44 @@ d.addCallback(_check) return d - def failUnlessChildURIIs(self, node, name, expected_uri): + def failUnlessRWChildURIIs(self, node, name, expected_uri): + assert isinstance(name, unicode) + d = node.get_child_at_path(name) + def _check(child): + self.failUnless(child.is_unknown() or not child.is_readonly()) + self.failUnlessEqual(child.get_uri(), expected_uri.strip()) + expected_ro_uri = self._make_readonly(expected_uri) + if expected_ro_uri: + self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip()) + d.addCallback(_check) + return d + + def failUnlessROChildURIIs(self, node, name, expected_uri): assert isinstance(name, unicode) d = node.get_child_at_path(name) def _check(child): + self.failUnless(child.is_unknown() or child.is_readonly()) self.failUnlessEqual(child.get_uri(), expected_uri.strip()) d.addCallback(_check) return d - def failUnlessURIMatchesChild(self, got_uri, node, name): + def failUnlessURIMatchesRWChild(self, got_uri, node, name): + assert isinstance(name, unicode) + d = node.get_child_at_path(name) + def _check(child): + self.failUnless(child.is_unknown() or not child.is_readonly()) + self.failUnlessEqual(child.get_uri(), got_uri.strip()) + expected_ro_uri = self._make_readonly(got_uri) + if expected_ro_uri: + self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip()) + d.addCallback(_check) + return d + + def failUnlessURIMatchesROChild(self, got_uri, node, name): assert isinstance(name, unicode) d = node.get_child_at_path(name) def _check(child): + self.failUnless(child.is_unknown() or child.is_readonly()) self.failUnlessEqual(got_uri.strip(), child.get_uri()) d.addCallback(_check) return d @@ -1366,10 +1506,11 @@ self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents) def test_POST_upload(self): - d = self.POST(self.public_url + "/foo", t="upload", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload", http.OK, self.POST, + self.public_url + "/foo", t="upload", + file=("new.txt", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, u"new.txt", self.NEWFILE_CONTENTS)) @@ -1377,15 +1518,16 @@ def test_POST_upload_unicode(self): filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t - d = self.POST(self.public_url + "/foo", t="upload", - file=(filename, self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_unicode", http.OK, self.POST, + self.public_url + "/foo", t="upload", + file=(filename, self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, filename) + d.addCallback(self.failUnlessURIMatchesROChild, fn, filename) d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, filename, self.NEWFILE_CONTENTS)) target_url = self.public_url + "/foo/" + filename.encode("utf-8") - d.addCallback(lambda res: self.GET(target_url)) + d.addCallback(lambda res: self.shouldSucceedGET(target_url)) d.addCallback(lambda contents: self.failUnlessEqual(contents, self.NEWFILE_CONTENTS, contents)) @@ -1393,24 +1535,26 @@ def test_POST_upload_unicode_named(self): filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t - d = self.POST(self.public_url + "/foo", t="upload", - name=filename, - file=("overridden", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_unicode_named", http.OK, self.POST, + self.public_url + "/foo", t="upload", + name=filename, + file=("overridden", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, filename) + d.addCallback(self.failUnlessURIMatchesROChild, fn, filename) d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, filename, self.NEWFILE_CONTENTS)) target_url = self.public_url + "/foo/" + filename.encode("utf-8") - d.addCallback(lambda res: self.GET(target_url)) + d.addCallback(lambda res: self.shouldSucceedGET(target_url)) d.addCallback(lambda contents: self.failUnlessEqual(contents, self.NEWFILE_CONTENTS, contents)) return d def test_POST_upload_no_link(self): - d = self.POST("/uri", t="upload", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_no_link", http.OK, self.POST, + "/uri", t="upload", + file=("new.txt", self.NEWFILE_CONTENTS)) def _check_upload_results(page): # this should be a page which describes the results of the upload # that just finished. @@ -1449,7 +1593,7 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnless(target.startswith(self.webish_url), target) return client.getPage(target, method="GET") - d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results", + d = self.shouldRedirect2("POST_upload_no_link_whendone_results", check, self.POST, "/uri", t="upload", when_done="/uri/%(uri)s", @@ -1459,8 +1603,9 @@ return d def test_POST_upload_no_link_mutable(self): - d = self.POST("/uri", t="upload", mutable="true", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_no_link_mutable", http.OK, self.POST, + "/uri", t="upload", mutable="true", + file=("new.txt", self.NEWFILE_CONTENTS)) def _check(filecap): filecap = filecap.strip() self.failUnless(filecap.startswith("URI:SSK:"), filecap) @@ -1472,11 +1617,11 @@ d.addCallback(_check) def _check2(data): self.failUnlessEqual(data, self.NEWFILE_CONTENTS) - return self.GET("/uri/%s" % urllib.quote(self.filecap)) + return self.shouldSucceedGET("/uri/%s" % urllib.quote(self.filecap)) d.addCallback(_check2) def _check3(data): self.failUnlessEqual(data, self.NEWFILE_CONTENTS) - return self.GET("/file/%s" % urllib.quote(self.filecap)) + return self.shouldSucceedGET("/file/%s" % urllib.quote(self.filecap)) d.addCallback(_check3) def _check4(data): self.failUnlessEqual(data, self.NEWFILE_CONTENTS) @@ -1485,7 +1630,7 @@ def test_POST_upload_no_link_mutable_toobig(self): d = self.shouldFail2(error.Error, - "test_POST_upload_no_link_mutable_toobig", + "POST_upload_no_link_mutable_toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.POST, @@ -1496,10 +1641,11 @@ def test_POST_upload_mutable(self): # this creates a mutable file - d = self.POST(self.public_url + "/foo", t="upload", mutable="true", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_mutable", http.OK, self.POST, + self.public_url + "/foo", t="upload", mutable="true", + file=("new.txt", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(fn, u"new.txt", self.NEWFILE_CONTENTS)) @@ -1515,10 +1661,11 @@ # now upload it again and make sure that the URI doesn't change NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n" d.addCallback(lambda res: - self.POST(self.public_url + "/foo", t="upload", - mutable="true", - file=("new.txt", NEWER_CONTENTS))) - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + self.shouldSucceed("POST_upload_mutable-again", http.OK, self.POST, + self.public_url + "/foo", t="upload", + mutable="true", + file=("new.txt", NEWER_CONTENTS))) + d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(fn, u"new.txt", NEWER_CONTENTS)) @@ -1533,8 +1680,9 @@ # upload a second time, using PUT instead of POST NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n" d.addCallback(lambda res: - self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS)) - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + self.shouldSucceed("POST_upload_mutable-again-with-PUT", http.OK, self.PUT, + self.public_url + "/foo/new.txt", NEW2_CONTENTS)) + d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(fn, u"new.txt", NEW2_CONTENTS)) @@ -1543,8 +1691,8 @@ # slightly differently d.addCallback(lambda res: - self.GET(self.public_url + "/foo/", - followRedirect=True)) + self.shouldSucceedGET(self.public_url + "/foo/", + followRedirect=True)) def _check_page(res): # TODO: assert more about the contents self.failUnless("SSK" in res) @@ -1561,8 +1709,8 @@ # look at the JSON form of the enclosing directory d.addCallback(lambda res: - self.GET(self.public_url + "/foo/?t=json", - followRedirect=True)) + self.shouldSucceedGET(self.public_url + "/foo/?t=json", + followRedirect=True)) def _check_page_json(res): parsed = simplejson.loads(res) self.failUnlessEqual(parsed[0], "dirnode") @@ -1580,7 +1728,7 @@ # and the JSON form of the file d.addCallback(lambda res: - self.GET(self.public_url + "/foo/new.txt?t=json")) + self.shouldSucceedGET(self.public_url + "/foo/new.txt?t=json")) def _check_file_json(res): parsed = simplejson.loads(res) self.failUnlessEqual(parsed[0], "filenode") @@ -1592,10 +1740,10 @@ # and look at t=uri and t=readonly-uri d.addCallback(lambda res: - self.GET(self.public_url + "/foo/new.txt?t=uri")) + self.shouldSucceedGET(self.public_url + "/foo/new.txt?t=uri")) d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri)) d.addCallback(lambda res: - self.GET(self.public_url + "/foo/new.txt?t=readonly-uri")) + self.shouldSucceedGET(self.public_url + "/foo/new.txt?t=readonly-uri")) def _check_ro_uri(res): ro_uri = unicode(self._mutable_node.get_readonly().to_string()) self.failUnlessEqual(res, ro_uri) @@ -1603,15 +1751,15 @@ # make sure we can get to it from /uri/URI d.addCallback(lambda res: - self.GET("/uri/%s" % urllib.quote(self._mutable_uri))) + self.shouldSucceedGET("/uri/%s" % urllib.quote(self._mutable_uri))) d.addCallback(lambda res: self.failUnlessEqual(res, NEW2_CONTENTS)) # and that HEAD computes the size correctly d.addCallback(lambda res: - self.HEAD(self.public_url + "/foo/new.txt", - return_response=True)) - def _got_headers((res, status, headers)): + self.shouldSucceedHEAD(self.public_url + "/foo/new.txt", + return_response=True)) + def _got_headers((res, statuscode, headers)): self.failUnlessEqual(res, "") self.failUnlessEqual(headers["content-length"][0], str(len(NEW2_CONTENTS))) @@ -1621,7 +1769,7 @@ # make sure that size errors are displayed correctly for overwrite d.addCallback(lambda res: self.shouldFail2(error.Error, - "test_POST_upload_mutable-toobig", + "POST_upload_mutable-toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.POST, @@ -1636,7 +1784,7 @@ def test_POST_upload_mutable_toobig(self): d = self.shouldFail2(error.Error, - "test_POST_upload_mutable_toobig", + "POST_upload_mutable_toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.POST, @@ -1660,19 +1808,21 @@ return f def test_POST_upload_replace(self): - d = self.POST(self.public_url + "/foo", t="upload", - file=("bar.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_replace", http.OK, self.POST, + self.public_url + "/foo", t="upload", + file=("bar.txt", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt") + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, u"bar.txt", self.NEWFILE_CONTENTS)) return d def test_POST_upload_no_replace_ok(self): - d = self.POST(self.public_url + "/foo?replace=false", t="upload", - file=("new.txt", self.NEWFILE_CONTENTS)) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt")) + d = self.shouldSucceed("POST_upload_no_replace_ok", http.OK, self.POST, + self.public_url + "/foo?replace=false", t="upload", + file=("new.txt", self.NEWFILE_CONTENTS)) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/new.txt")) d.addCallback(lambda res: self.failUnlessEqual(res, self.NEWFILE_CONTENTS)) return d @@ -1685,7 +1835,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d @@ -1696,7 +1846,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d @@ -1712,9 +1862,10 @@ def test_POST_upload_named(self): fn = self._foo_node - d = self.POST(self.public_url + "/foo", t="upload", - name="new.txt", file=self.NEWFILE_CONTENTS) - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + d = self.shouldSucceed("POST_upload_named", http.OK, self.POST, + self.public_url + "/foo", t="upload", + name="new.txt", file=self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, u"new.txt", self.NEWFILE_CONTENTS)) @@ -1724,7 +1875,7 @@ d = self.POST(self.public_url + "/foo", t="upload", name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS) d.addBoth(self.shouldFail, error.Error, - "test_POST_upload_named_badfilename", + "POST_upload_named_badfilename", "400 Bad Request", "name= may not contain a slash", ) @@ -1738,7 +1889,8 @@ def test_POST_FILEURL_check(self): bar_url = self.public_url + "/foo/bar.txt" - d = self.POST(bar_url, t="check") + d = self.shouldSucceed("POST_FILEURL_check-1", http.OK, self.POST, + bar_url, t="check") def _check(res): self.failUnless("Healthy :" in res) d.addCallback(_check) @@ -1747,13 +1899,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_FILEURL_check", + self.shouldRedirect2("POST_FILEURL_check-2", _check2, self.POST, bar_url, t="check", when_done=redir_url)) d.addCallback(lambda res: - self.POST(bar_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_FILEURL_check-3", http.OK, self.POST, + bar_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res) self.failUnless("Return to file" in res) @@ -1761,7 +1914,8 @@ d.addCallback(_check3) d.addCallback(lambda res: - self.POST(bar_url, t="check", output="JSON")) + self.shouldSucceed("POST_FILEURL_check-4", http.OK, self.POST, + bar_url, t="check", output="JSON")) def _check_json(res): data = simplejson.loads(res) self.failUnless("storage-index" in data) @@ -1772,7 +1926,8 @@ def test_POST_FILEURL_check_and_repair(self): bar_url = self.public_url + "/foo/bar.txt" - d = self.POST(bar_url, t="check", repair="true") + d = self.shouldSucceed("POST_FILEURL_check_and_repair-1", http.OK, self.POST, + bar_url, t="check", repair="true") def _check(res): self.failUnless("Healthy :" in res) d.addCallback(_check) @@ -1781,13 +1936,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_FILEURL_check_and_repair", + self.shouldRedirect2("POST_FILEURL_check_and_repair-2", _check2, self.POST, bar_url, t="check", repair="true", when_done=redir_url)) d.addCallback(lambda res: - self.POST(bar_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_FILEURL_check_and_repair-3", http.OK, self.POST, + bar_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res) self.failUnless("Return to file" in res) @@ -1797,7 +1953,8 @@ def test_POST_DIRURL_check(self): foo_url = self.public_url + "/foo/" - d = self.POST(foo_url, t="check") + d = self.shouldSucceed("POST_DIRURL_check-1", http.OK, self.POST, + foo_url, t="check") def _check(res): self.failUnless("Healthy :" in res, res) d.addCallback(_check) @@ -1806,13 +1963,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_DIRURL_check", + self.shouldRedirect2("POST_DIRURL_check-2", _check2, self.POST, foo_url, t="check", when_done=redir_url)) d.addCallback(lambda res: - self.POST(foo_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_DIRURL_check-3", http.OK, self.POST, + foo_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res, res) self.failUnless("Return to file/directory" in res) @@ -1820,7 +1978,8 @@ d.addCallback(_check3) d.addCallback(lambda res: - self.POST(foo_url, t="check", output="JSON")) + self.shouldSucceed("POST_DIRURL_check-4", http.OK, self.POST, + foo_url, t="check", output="JSON")) def _check_json(res): data = simplejson.loads(res) self.failUnless("storage-index" in data) @@ -1831,7 +1990,8 @@ def test_POST_DIRURL_check_and_repair(self): foo_url = self.public_url + "/foo/" - d = self.POST(foo_url, t="check", repair="true") + d = self.shouldSucceed("POST_DIRURL_check_and_repair-1", http.OK, self.POST, + foo_url, t="check", repair="true") def _check(res): self.failUnless("Healthy :" in res, res) d.addCallback(_check) @@ -1840,13 +2000,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_DIRURL_check_and_repair", + self.shouldRedirect2("POST_DIRURL_check_and_repair-2", _check2, self.POST, foo_url, t="check", repair="true", when_done=redir_url)) d.addCallback(lambda res: - self.POST(foo_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_DIRURL_check_and_repair-3", http.OK, self.POST, + foo_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res) self.failUnless("Return to file/directory" in res) @@ -1857,7 +2018,7 @@ def wait_for_operation(self, ignored, ophandle): url = "/operations/" + ophandle url += "?t=status&output=JSON" - d = self.GET(url) + d = self.shouldSucceedGET(url) def _got(res): data = simplejson.loads(res) if not data["finished"]: @@ -1873,7 +2034,7 @@ url += "?t=status" if output: url += "&output=" + output - d = self.GET(url) + d = self.shouldSucceedGET(url) def _got(res): if output and output.lower() == "json": return simplejson.loads(res) @@ -1883,7 +2044,7 @@ def test_POST_DIRURL_deepcheck_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_deepcheck_no_ophandle", + "POST_DIRURL_deepcheck_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-deep-check") @@ -1893,7 +2054,7 @@ def _check_redirect(statuscode, target): self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnless(target.endswith("/operations/123")) - d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect, + d = self.shouldRedirect2("POST_DIRURL_deepcheck", _check_redirect, self.POST, self.public_url, t="start-deep-check", ophandle="123") d.addCallback(self.wait_for_operation, "123") @@ -1909,7 +2070,7 @@ d.addCallback(_check_html) d.addCallback(lambda res: - self.GET("/operations/123/")) + self.shouldSucceedGET("/operations/123/")) d.addCallback(_check_html) # should be the same as without the slash d.addCallback(lambda res: @@ -1920,7 +2081,7 @@ foo_si = self._foo_node.get_storage_index() foo_si_s = base32.b2a(foo_si) d.addCallback(lambda res: - self.GET("/operations/123/%s?output=JSON" % foo_si_s)) + self.shouldSucceedGET("/operations/123/%s?output=JSON" % foo_si_s)) def _check_foo_json(res): data = simplejson.loads(res) self.failUnlessEqual(data["storage-index"], foo_si_s) @@ -1929,8 +2090,9 @@ return d def test_POST_DIRURL_deepcheck_and_repair(self): - d = self.POST(self.public_url, t="start-deep-check", repair="true", - ophandle="124", output="json", followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_deepcheck_and_repair", http.OK, self.POST, + self.public_url, t="start-deep-check", repair="true", + ophandle="124", output="json", followRedirect=True) d.addCallback(self.wait_for_operation, "124") def _check_json(data): self.failUnlessEqual(data["finished"], True) @@ -1971,45 +2133,47 @@ return d def test_POST_mkdir(self): # return value? - d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir") + d = self.shouldSucceed("POST_mkdir", http.OK, self.POST, + self.public_url + "/foo", t="mkdir", name="newdir") d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, []) return d def test_POST_mkdir_initial_children(self): - newkids, filecap1, ign, ign, ign = self._create_initial_children() - d = self.POST2(self.public_url + - "/foo?t=mkdir-with-children&name=newdir", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_initial_children() + d = self.shouldSucceed("POST_mkdir_initial_children", http.OK, self.POST2, + self.public_url + "/foo?t=mkdir-with-children&name=newdir", + simplejson.dumps(newkids)) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) return d def test_POST_mkdir_immutable(self): - (newkids, filecap1, immdircap) = self._create_immutable_children() - d = self.POST2(self.public_url + - "/foo?t=mkdir-immutable&name=newdir", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_immutable_children() + d = self.shouldSucceed("POST_mkdir_immutable", http.OK, self.POST2, + self.public_url + "/foo?t=mkdir-immutable&name=newdir", + simplejson.dumps(newkids)) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap']) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap']) return d def test_POST_mkdir_immutable_bad(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad", + (newkids, caps) = self._create_initial_children() + d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad", "400 Bad Request", - "a mkdir-immutable operation was given a child that was not itself immutable", + "needed to be immutable but was not", self.POST2, self.public_url + "/foo?t=mkdir-immutable&name=newdir", @@ -2017,7 +2181,8 @@ return d def test_POST_mkdir_2(self): - d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.shouldSucceed("POST_mkdir_2", http.OK, self.POST, + self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -2025,7 +2190,8 @@ return d def test_POST_mkdirs_2(self): - d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "") + d = self.shouldSucceed("POST_mkdirs_2", http.OK, self.POST, + self.public_url + "/foo/bardir/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bardir")) d.addCallback(lambda res: self._foo_node.get(u"bardir")) @@ -2034,7 +2200,8 @@ return d def test_POST_mkdir_no_parentdir_noredirect(self): - d = self.POST("/uri?t=mkdir") + d = self.shouldSucceed("POST_mkdir_no_parentdir_noredirect", http.OK, self.POST, + "/uri?t=mkdir") def _after_mkdir(res): uri.DirectoryURI.init_from_string(res) d.addCallback(_after_mkdir) @@ -2049,21 +2216,43 @@ d.addCallback(_check_target) return d + def _make_readonly(self, u): + ro_uri = uri.from_string(u).get_readonly() + if ro_uri is None: + return None + return ro_uri.to_string() + def _create_initial_children(self): contents, n, filecap1 = self.makefile(12) md1 = {"metakey1": "metavalue1"} filecap2 = make_mutable_file_uri() node3 = self.s.create_node_from_uri(make_mutable_file_uri()) filecap3 = node3.get_readonly_uri() + unknown_rwcap = "lafs://from_the_future" + unknown_rocap = "ro.lafs://readonly_from_the_future" + unknown_immcap = "imm.lafs://immutable_from_the_future" node4 = self.s.create_node_from_uri(make_mutable_file_uri()) dircap = DirectoryNode(node4, None, None).get_uri() - newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, - "metadata": md1, }], - u"child-mutable": ["filenode", {"rw_uri": filecap2}], + newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1, + "ro_uri": self._make_readonly(filecap1), + "metadata": md1, }], + u"child-mutable": ["filenode", {"rw_uri": filecap2, + "ro_uri": self._make_readonly(filecap2)}], u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}], - u"dirchild": ["dirnode", {"rw_uri": dircap}], + u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap, + "ro_uri": unknown_rocap}], + u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}], + u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}], + u"dirchild": ["dirnode", {"rw_uri": dircap, + "ro_uri": self._make_readonly(dircap)}], } - return newkids, filecap1, filecap2, filecap3, dircap + return newkids, {'filecap1': filecap1, + 'filecap2': filecap2, + 'filecap3': filecap3, + 'unknown_rwcap': unknown_rwcap, + 'unknown_rocap': unknown_rocap, + 'unknown_immcap': unknown_immcap, + 'dircap': dircap} def _create_immutable_children(self): contents, n, filecap1 = self.makefile(12) @@ -2071,31 +2260,46 @@ tnode = create_chk_filenode("immutable directory contents\n"*10) dnode = DirectoryNode(tnode, None, None) assert not dnode.is_mutable() + unknown_immcap = "imm.lafs://immutable_from_the_future" immdircap = dnode.get_uri() - newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, - "metadata": md1, }], - u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}], + newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, + "metadata": md1, }], + u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}], + u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}], } - return newkids, filecap1, immdircap + return newkids, {'filecap1': filecap1, + 'unknown_immcap': unknown_immcap, + 'immdircap': immdircap} def test_POST_mkdir_no_parentdir_initial_children(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids)) + (newkids, caps) = self._create_initial_children() + d = self.shouldSucceed("POST_mkdir_no_parentdir_initial_children", http.OK, self.POST2, + "/uri?t=mkdir-with-children", simplejson.dumps(newkids)) def _after_mkdir(res): self.failUnless(res.startswith("URI:DIR"), res) n = self.s.create_node_from_uri(res) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"child-mutable", + caps['filecap2'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"child-mutable-ro", + caps['filecap3'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable", - filecap2)) + self.failUnlessRWChildURIIs(n, u"unknownchild-rw", + caps['unknown_rwcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable-ro", - filecap3)) + self.failUnlessROChildURIIs(n, u"unknownchild-ro", + caps['unknown_rocap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild", dircap)) + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"dirchild", + caps['dircap'])) return d2 d.addCallback(_after_mkdir) return d @@ -2103,8 +2307,7 @@ def test_POST_mkdir_no_parentdir_unexpected_children(self): # the regular /uri?t=mkdir operation is specified to ignore its body. # Only t=mkdir-with-children pays attention to it. - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() + (newkids, caps) = self._create_initial_children() d = self.shouldHTTPError("POST t=mkdir unexpected children", 400, "Bad Request", "t=mkdir does not accept children=, " @@ -2121,28 +2324,32 @@ return d def test_POST_mkdir_no_parentdir_immutable(self): - (newkids, filecap1, immdircap) = self._create_immutable_children() - d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids)) + (newkids, caps) = self._create_immutable_children() + d = self.shouldSucceed("POST_mkdir_no_parentdir_immutable", http.OK, self.POST2, + "/uri?t=mkdir-immutable", simplejson.dumps(newkids)) def _after_mkdir(res): self.failUnless(res.startswith("URI:DIR"), res) n = self.s.create_node_from_uri(res) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild-imm", - immdircap)) + self.failUnlessROChildURIIs(n, u"dirchild-imm", + caps['immdircap'])) return d2 d.addCallback(_after_mkdir) return d def test_POST_mkdir_no_parentdir_immutable_bad(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() + (newkids, caps) = self._create_initial_children() d = self.shouldFail2(error.Error, - "test_POST_mkdir_no_parentdir_immutable_bad", + "POST_mkdir_no_parentdir_immutable_bad", "400 Bad Request", - "a mkdir-immutable operation was given a child that was not itself immutable", + "needed to be immutable but was not", self.POST2, "/uri?t=mkdir-immutable", simplejson.dumps(newkids)) @@ -2150,9 +2357,14 @@ def test_welcome_page_mkdir_button(self): # Fetch the welcome page. - d = self.GET("/") + d = self.shouldSucceedGET("/") def _after_get_welcome_page(res): - MKDIR_BUTTON_RE=re.compile('
', re.I) + MKDIR_BUTTON_RE = re.compile( + '' + '' + '', + re.I) mo = MKDIR_BUTTON_RE.search(res) formaction = mo.group(1) formt = mo.group(2) @@ -2168,7 +2380,8 @@ return d def test_POST_mkdir_replace(self): # return value? - d = self.POST(self.public_url + "/foo", t="mkdir", name="sub") + d = self.shouldSucceed("POST_mkdir_replace", http.OK, self.POST, + self.public_url + "/foo", t="mkdir", name="sub") d.addCallback(lambda res: self._foo_node.get(u"sub")) d.addCallback(self.failUnlessNodeKeysAre, []) return d @@ -2250,9 +2463,9 @@ d = client.getPage(url, method="POST", postdata=reqbody) def _then(res): - self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1") - self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2") - self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3") + self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1") + self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2") + self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3") d.addCallback(_then) d.addErrback(self.dump_error) @@ -2260,8 +2473,9 @@ def test_POST_put_uri(self): contents, n, newuri = self.makefile(8) - d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d = self.shouldSucceed("POST_put_uri", http.OK, self.POST, + self.public_url + "/foo", t="uri", name="new.txt", uri=newuri) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", contents)) @@ -2269,8 +2483,9 @@ def test_POST_put_uri_replace(self): contents, n, newuri = self.makefile(8) - d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt") + d = self.shouldSucceed("POST_put_uri_replace", http.OK, self.POST, + self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"bar.txt", contents)) @@ -2285,7 +2500,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d @@ -2298,12 +2513,13 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d def test_POST_delete(self): - d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt") + d = self.shouldSucceed("POST_delete", http.OK, self.POST, + self.public_url + "/foo", t="delete", name="bar.txt") d.addCallback(lambda res: self._foo_node.list()) def _check(children): self.failIf(u"bar.txt" in children) @@ -2311,40 +2527,43 @@ return d def test_POST_rename_file(self): - d = self.POST(self.public_url + "/foo", t="rename", - from_name="bar.txt", to_name='wibble.txt') + d = self.shouldSucceed("POST_rename_file", http.OK, self.POST, + self.public_url + "/foo", t="rename", + from_name="bar.txt", to_name='wibble.txt') d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt")) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/wibble.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/wibble.txt?t=json")) d.addCallback(self.failUnlessIsBarJSON) return d def test_POST_rename_file_redundant(self): - d = self.POST(self.public_url + "/foo", t="rename", - from_name="bar.txt", to_name='bar.txt') + d = self.shouldSucceed("POST_rename_file_redundant", http.OK, self.POST, + self.public_url + "/foo", t="rename", + from_name="bar.txt", to_name='bar.txt') d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt")) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=json")) d.addCallback(self.failUnlessIsBarJSON) return d def test_POST_rename_file_replace(self): # rename a file and replace a directory with it - d = self.POST(self.public_url + "/foo", t="rename", - from_name="bar.txt", to_name='empty') + d = self.shouldSucceed("POST_rename_file_replace", http.OK, self.POST, + self.public_url + "/foo", t="rename", + from_name="bar.txt", to_name='empty') d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"empty")) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty?t=json")) d.addCallback(self.failUnlessIsBarJSON) return d @@ -2357,7 +2576,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty?t=json")) d.addCallback(self.failUnlessIsEmptyJSON) return d @@ -2370,7 +2589,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty?t=json")) d.addCallback(self.failUnlessIsEmptyJSON) return d @@ -2383,7 +2602,7 @@ d = self.POST(self.public_url + "/foo", t="rename", from_name="bar.txt", to_name='kirk/spock.txt') d.addBoth(self.shouldFail, error.Error, - "test_POST_rename_file_slash_fail", + "POST_rename_file_slash_fail", "400 Bad Request", "to_name= may not contain a slash", ) @@ -2392,13 +2611,14 @@ return d def test_POST_rename_dir(self): - d = self.POST(self.public_url, t="rename", - from_name="foo", to_name='plunk') + d = self.shouldSucceed("POST_rename_dir", http.OK, self.POST, + self.public_url, t="rename", + from_name="foo", to_name='plunk') d.addCallback(lambda res: self.failIfNodeHasChild(self.public_root, u"foo")) d.addCallback(lambda res: self.failUnlessNodeHasChild(self.public_root, u"plunk")) - d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/plunk?t=json")) d.addCallback(self.failUnlessIsFooJSON) return d @@ -2433,24 +2653,24 @@ d.addCallback(lambda res: self.GET(base+"&t=json")) d.addBoth(self.shouldRedirect, targetbase+"?t=json") d.addCallback(self.log, "about to get file by uri") - d.addCallback(lambda res: self.GET(base, followRedirect=True)) + d.addCallback(lambda res: self.shouldSucceedGET(base, followRedirect=True)) d.addCallback(self.failUnlessIsBarDotTxt) d.addCallback(self.log, "got file by uri, about to get dir by uri") - d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri, - followRedirect=True)) + d.addCallback(lambda res: self.shouldSucceedGET("/uri?uri=%s&t=json" % self._foo_uri, + followRedirect=True)) d.addCallback(self.failUnlessIsFooJSON) d.addCallback(self.log, "got dir by uri") return d def test_GET_URI_form_bad(self): - d = self.shouldFail2(error.Error, "test_GET_URI_form_bad", + d = self.shouldFail2(error.Error, "GET_URI_form_bad", "400 Bad Request", "GET /uri requires uri=", self.GET, "/uri") return d def test_GET_rename_form(self): - d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt", + d = self.shouldSucceedGET(self.public_url + "/foo?t=rename-form&name=bar.txt", followRedirect=True) def _check(res): self.failUnless('name="when_done" value="."' in res, res) @@ -2465,23 +2685,23 @@ def test_GET_URI_URL(self): base = "/uri/%s" % self._bar_txt_uri - d = self.GET(base) + d = self.shouldSucceedGET(base) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base+"?filename=bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base+"?filename=bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true")) + d.addCallback(lambda res: self.shouldSucceedGET(base+"?filename=bar.txt&save=true")) d.addCallback(self.failUnlessIsBarDotTxt) return d def test_GET_URI_URL_dir(self): base = "/uri/%s?t=json" % self._foo_uri - d = self.GET(base) + d = self.shouldSucceedGET(base) d.addCallback(self.failUnlessIsFooJSON) return d def test_GET_URI_URL_missing(self): base = "/uri/%s" % self._bad_file_uri - d = self.shouldHTTPError("test_GET_URI_URL_missing", + d = self.shouldHTTPError("GET_URI_URL_missing", http.GONE, None, "NotEnoughSharesError", self.GET, base) # TODO: how can we exercise both sides of WebDownloadTarget.fail @@ -2499,9 +2719,9 @@ d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri)) d.addCallback(lambda res: - self.failUnlessChildURIIs(self.public_root, - u"foo", - new_uri)) + self.failUnlessRWChildURIIs(self.public_root, + u"foo", + new_uri)) return d d.addCallback(_made_dir) return d @@ -2512,32 +2732,33 @@ new_uri = dn.get_uri() # replace /foo with a new (empty) directory, but ask that # replace=false, so it should fail - d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace", + d = self.shouldFail2(error.Error, "PUT_DIRURL_uri_noreplace", "409 Conflict", "There was already a child by that name, and you asked me to not replace it", self.PUT, self.public_url + "/foo?t=uri&replace=false", new_uri) d.addCallback(lambda res: - self.failUnlessChildURIIs(self.public_root, - u"foo", - self._foo_uri)) + self.failUnlessRWChildURIIs(self.public_root, + u"foo", + self._foo_uri)) return d d.addCallback(_made_dir) return d def test_PUT_DIRURL_bad_t(self): - d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t", + d = self.shouldFail2(error.Error, "PUT_DIRURL_bad_t", "400 Bad Request", "PUT to a directory", self.PUT, self.public_url + "/foo?t=BOGUS", "") d.addCallback(lambda res: - self.failUnlessChildURIIs(self.public_root, - u"foo", - self._foo_uri)) + self.failUnlessRWChildURIIs(self.public_root, + u"foo", + self._foo_uri)) return d def test_PUT_NEWFILEURL_uri(self): contents, n, new_uri = self.makefile(8) - d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri) + d = self.shouldSucceed("PUT_NEWFILEURL_uri", http.OK, self.PUT, + self.public_url + "/foo/new.txt?t=uri", new_uri) d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri)) d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", @@ -2564,13 +2785,14 @@ def test_PUT_NEWFILE_URI(self): file_contents = "New file contents here\n" - d = self.PUT("/uri", file_contents) + d = self.shouldSucceed("PUT_NEWFILE_URI", http.OK, self.PUT, + "/uri", file_contents) def _check(uri): assert isinstance(uri, str), uri self.failUnless(uri in FakeCHKFileNode.all_contents) self.failUnlessEqual(FakeCHKFileNode.all_contents[uri], file_contents) - return self.GET("/uri/%s" % uri) + return self.shouldSucceedGET("/uri/%s" % uri) d.addCallback(_check) def _check2(res): self.failUnlessEqual(res, file_contents) @@ -2579,13 +2801,14 @@ def test_PUT_NEWFILE_URI_not_mutable(self): file_contents = "New file contents here\n" - d = self.PUT("/uri?mutable=false", file_contents) + d = self.shouldSucceed("PUT_NEWFILE_URI_not_mutable", http.OK, self.PUT, + "/uri?mutable=false", file_contents) def _check(uri): assert isinstance(uri, str), uri self.failUnless(uri in FakeCHKFileNode.all_contents) self.failUnlessEqual(FakeCHKFileNode.all_contents[uri], file_contents) - return self.GET("/uri/%s" % uri) + return self.shouldSucceedGET("/uri/%s" % uri) d.addCallback(_check) def _check2(res): self.failUnlessEqual(res, file_contents) @@ -2602,7 +2825,8 @@ def test_PUT_NEWFILE_URI_mutable(self): file_contents = "New file contents here\n" - d = self.PUT("/uri?mutable=true", file_contents) + d = self.shouldSucceed("PUT_NEWFILE_URI_mutable", http.OK, self.PUT, + "/uri?mutable=true", file_contents) def _check1(filecap): filecap = filecap.strip() self.failUnless(filecap.startswith("URI:SSK:"), filecap) @@ -2614,7 +2838,7 @@ d.addCallback(_check1) def _check2(data): self.failUnlessEqual(data, file_contents) - return self.GET("/uri/%s" % urllib.quote(self.filecap)) + return self.shouldSucceedGET("/uri/%s" % urllib.quote(self.filecap)) d.addCallback(_check2) def _check3(res): self.failUnlessEqual(res, file_contents) @@ -2622,19 +2846,21 @@ return d def test_PUT_mkdir(self): - d = self.PUT("/uri?t=mkdir", "") + d = self.shouldSucceed("PUT_mkdir", http.OK, self.PUT, + "/uri?t=mkdir", "") def _check(uri): n = self.s.create_node_from_uri(uri.strip()) d2 = self.failUnlessNodeKeysAre(n, []) d2.addCallback(lambda res: - self.GET("/uri/%s?t=json" % uri)) + self.shouldSucceedGET("/uri/%s?t=json" % uri)) return d2 d.addCallback(_check) d.addCallback(self.failUnlessIsEmptyJSON) return d def test_POST_check(self): - d = self.POST(self.public_url + "/foo", t="check", name="bar.txt") + d = self.shouldSucceed("POST_check", http.OK, self.POST, + self.public_url + "/foo", t="check", name="bar.txt") def _done(res): # this returns a string form of the results, which are probably # None since we're using fake filenodes. @@ -2647,7 +2873,7 @@ def test_bad_method(self): url = self.webish_url + self.public_url + "/foo/bar.txt" - d = self.shouldHTTPError("test_bad_method", + d = self.shouldHTTPError("bad_method", 501, "Not Implemented", "I don't know how to treat a BOGUS request.", client.getPage, url, method="BOGUS") @@ -2655,28 +2881,30 @@ def test_short_url(self): url = self.webish_url + "/uri" - d = self.shouldHTTPError("test_short_url", 501, "Not Implemented", + d = self.shouldHTTPError("short_url", 501, "Not Implemented", "I don't know how to treat a DELETE request.", client.getPage, url, method="DELETE") return d def test_ophandle_bad(self): url = self.webish_url + "/operations/bogus?t=status" - d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found", + d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found", "unknown/expired handle 'bogus'", client.getPage, url) return d def test_ophandle_cancel(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128", - followRedirect=True) + d = self.shouldSucceed("ophandle_cancel-1", http.OK, self.POST, + self.public_url + "/foo/?t=start-manifest&ophandle=128", + followRedirect=True) d.addCallback(lambda ignored: - self.GET("/operations/128?t=status&output=JSON")) + self.shouldSucceedGET("/operations/128?t=status&output=JSON")) def _check1(res): data = simplejson.loads(res) self.failUnless("finished" in data, res) monitor = self.ws.root.child_operations.handles["128"][0] - d = self.POST("/operations/128?t=cancel&output=JSON") + d = self.shouldSucceed("ophandle_cancel-2", http.OK, self.POST, + "/operations/128?t=cancel&output=JSON") def _check2(res): data = simplejson.loads(res) self.failUnless("finished" in data, res) @@ -2686,7 +2914,7 @@ return d d.addCallback(_check1) d.addCallback(lambda ignored: - self.shouldHTTPError("test_ophandle_cancel", + self.shouldHTTPError("ophandle_cancel", 404, "404 Not Found", "unknown/expired handle '128'", self.GET, @@ -2697,7 +2925,7 @@ d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60", followRedirect=True) d.addCallback(lambda ignored: - self.GET("/operations/129?t=status&output=JSON&retain-for=0")) + self.shouldSucceedGET("/operations/129?t=status&output=JSON&retain-for=0")) def _check1(res): data = simplejson.loads(res) self.failUnless("finished" in data, res) @@ -2705,7 +2933,7 @@ # the retain-for=0 will cause the handle to be expired very soon d.addCallback(self.stall, 2.0) d.addCallback(lambda ignored: - self.shouldHTTPError("test_ophandle_retainfor", + self.shouldHTTPError("ophandle_retainfor", 404, "404 Not Found", "unknown/expired handle '129'", self.GET, @@ -2713,14 +2941,15 @@ return d def test_ophandle_release_after_complete(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130", - followRedirect=True) + d = self.shouldSucceed("ophandle_release_after_complete", http.OK, self.POST, + self.public_url + "/foo/?t=start-manifest&ophandle=130", + followRedirect=True) d.addCallback(self.wait_for_operation, "130") d.addCallback(lambda ignored: - self.GET("/operations/130?t=status&output=JSON&release-after-complete=true")) + self.shouldSucceedGET("/operations/130?t=status&output=JSON&release-after-complete=true")) # the release-after-complete=true will cause the handle to be expired d.addCallback(lambda ignored: - self.shouldHTTPError("test_ophandle_release_after_complete", + self.shouldHTTPError("ophandle_release_after_complete", 404, "404 Not Found", "unknown/expired handle '130'", self.GET, @@ -2728,7 +2957,8 @@ return d def test_incident(self): - d = self.POST("/report_incident", details="eek") + d = self.shouldSucceed("incident", http.OK, self.POST, + "/report_incident", details="eek") def _done(res): self.failUnless("Thank you for your report!" in res, res) d.addCallback(_done) @@ -2741,7 +2971,7 @@ f.write("hello") f.close() - d = self.GET("/static/subdir/hello.txt") + d = self.shouldSucceedGET("/static/subdir/hello.txt") def _check(res): self.failUnlessEqual(res, "hello") d.addCallback(_check) @@ -2754,7 +2984,7 @@ self.failUnlessEqual(common.parse_replace_arg("false"), False) self.failUnlessEqual(common.parse_replace_arg("only-files"), "only-files") - self.shouldFail(AssertionError, "test_parse_replace_arg", "", + self.shouldFail(AssertionError, "parse_replace_arg", "", common.parse_replace_arg, "only_fles") def test_abbreviate_time(self): @@ -3059,71 +3289,225 @@ d.addErrback(self.explain_web_error) return d - def test_unknown(self): + def test_unknown(self, immutable=False): self.basedir = "web/Grid/unknown" + if immutable: + self.basedir = "web/Grid/unknown-immutable" + self.set_up_grid() c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." # the future cap format may contain slashes, which must be tolerated - expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap, + expected_info_url = "uri/%s?t=info" % urllib.quote(future_write_uri, safe="") - future_node = UnknownNode(future_writecap, future_readcap) - d = c0.create_dirnode() + if immutable: + name = u"future-imm" + future_node = UnknownNode(None, future_read_uri, deep_immutable=True) + d = c0.create_immutable_dirnode({name: (future_node, {})}) + else: + name = u"future" + future_node = UnknownNode(future_write_uri, future_read_uri) + d = c0.create_dirnode() + def _stash_root_and_create_file(n): self.rootnode = n self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/" self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/" - return self.rootnode.set_node(u"future", future_node) + if not immutable: + return self.rootnode.set_node(name, future_node) d.addCallback(_stash_root_and_create_file) + # make sure directory listing tolerates unknown nodes d.addCallback(lambda ign: self.GET(self.rooturl)) def _check_html(res): - self.failUnlessIn("future", res) - # find the More Info link for "future", should be relative + self.failUnlessIn("%s" % (str(name),), res) + # find the More Info link for name, should be relative mo = re.search(r'More Info', res) info_url = mo.group(1) - self.failUnlessEqual(info_url, "future?t=info") + self.failUnlessEqual(info_url, "%s?t=info" % (str(name),)) d.addCallback(_check_html) d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) - def _check_json(res, expect_writecap): + def _check_json(res, expect_rw_uri): data = simplejson.loads(res) self.failUnlessEqual(data[0], "dirnode") - f = data[1]["children"]["future"] + f = data[1]["children"][name] self.failUnlessEqual(f[0], "unknown") - if expect_writecap: - self.failUnlessEqual(f[1]["rw_uri"], future_writecap) + if expect_rw_uri: + self.failUnlessEqual(f[1]["rw_uri"], future_write_uri) else: self.failIfIn("rw_uri", f[1]) - self.failUnlessEqual(f[1]["ro_uri"], future_readcap) + self.failUnlessEqual(f[1]["ro_uri"], + ("imm." if immutable else "ro.") + future_read_uri) self.failUnless("metadata" in f[1]) - d.addCallback(_check_json, expect_writecap=True) - d.addCallback(lambda ign: self.GET(expected_info_url)) - def _check_info(res, expect_readcap): + d.addCallback(_check_json, expect_rw_uri=not immutable) + + def _check_info(res, expect_rw_uri, expect_ro_uri): self.failUnlessIn("Object Type: unknown", res) - self.failUnlessIn(future_writecap, res) - if expect_readcap: - self.failUnlessIn(future_readcap, res) + if expect_rw_uri: + self.failUnlessIn(future_write_uri, res) + if expect_ro_uri: + self.failUnlessIn(future_read_uri, res) + else: + self.failIfIn(future_read_uri, res) self.failIfIn("Raw data as", res) self.failIfIn("Directory writecap", res) self.failIfIn("Checker Operations", res) self.failIfIn("Mutable File Operations", res) self.failIfIn("Directory Operations", res) - d.addCallback(_check_info, expect_readcap=False) - d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info")) - d.addCallback(_check_info, expect_readcap=True) + + # Known bug: these should have expect_rw_uri=not immutable, but the + # info pages are currently broken. Related to ticket #922. + + d.addCallback(lambda ign: self.GET(expected_info_url)) + d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False) + d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name)))) + d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True) # and make sure that a read-only version of the directory can be - # rendered too. This version will not have future_writecap + # rendered too. This version will not have future_write_uri, whether + # or not future_node was immutable. d.addCallback(lambda ign: self.GET(self.rourl)) d.addCallback(_check_html) d.addCallback(lambda ign: self.GET(self.rourl+"?t=json")) - d.addCallback(_check_json, expect_writecap=False) + d.addCallback(_check_json, expect_rw_uri=False) + return d + + def test_immutable_unknown(self): + return self.test_unknown(immutable=True) + + def test_mutant_dirnodes_are_omitted(self): + self.basedir = "web/Grid/mutant_dirnodes_are_omitted" + + self.set_up_grid() + c = self.g.clients[0] + nm = c.nodemaker + self.uris = {} + self.fileurls = {} + + lonely_uri = "URI:LIT:n5xgk" # LIT for "one" + mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + + # This method tests mainly dirnode, but we'd have to duplicate code in order to + # test the dirnode and web layers separately. + + # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap, + # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field. + # When the directory is read, the mutants should be silently disposed of, leaving + # their lonely sibling. + # We don't test the case of a retrieving a cap from the encrypted rw_uri field, + # because immutable directories don't have a writecap and therefore that field + # isn't (and can't be) decrypted. + # TODO: The field still exists in the netstring. Technically we should check what + # happens if something is put there (it should be ignored), but that can wait. + + lonely_child = nm.create_from_cap(lonely_uri) + mutant_ro_child = nm.create_from_cap(mut_read_uri) + mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri) + + def _by_hook_or_by_crook(): + return True + for n in [mutant_ro_child, mutant_write_in_ro_child]: + n.is_allowed_in_immutable_directory = _by_hook_or_by_crook + + mutant_write_in_ro_child.get_write_uri = lambda: None + mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri + + kids = {u"lonely": (lonely_child, {}), + u"ro": (mutant_ro_child, {}), + u"write-in-ro": (mutant_write_in_ro_child, {}), + } + d = c.create_immutable_dirnode(kids) + + def _created(dn): + self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failIf(dn.is_mutable()) + self.failUnless(dn.is_readonly()) + # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail. + self.failIf(hasattr(dn._node, 'get_writekey')) + rep = str(dn) + self.failUnless("RO-IMM" in rep) + cap = dn.get_cap() + self.failUnlessIn("CHK", cap.to_string()) + self.cap = cap + self.rootnode = dn + self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/" + return download_to_data(dn._node) + d.addCallback(_created) + + def _check_data(data): + # Decode the netstring representation of the directory to check that all children + # are present. This is a bit of an abstraction violation, but there's not really + # any other way to do it given that the real DirectoryNode._unpack_contents would + # strip the mutant children out (which is what we're trying to test, later). + position = 0 + numkids = 0 + while position < len(data): + entries, position = split_netstring(data, 1, position) + entry = entries[0] + (name, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) + name = name.decode("utf-8") + self.failUnless(rwcapdata == "") + ro_uri = ro_uri.strip() + if name in kids: + self.failIfEqual(ro_uri, "") + (expected_child, ign) = kids[name] + self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri()) + numkids += 1 + + self.failUnlessEqual(numkids, 3) + return self.rootnode.list() + d.addCallback(_check_data) + + # Now when we use the real directory listing code, the mutants should be absent. + def _check_kids(children): + self.failUnlessEqual(sorted(children.keys()), [u"lonely"]) + lonely_node, lonely_metadata = children[u"lonely"] + + self.failUnlessEqual(lonely_node.get_write_uri(), None) + self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri) + d.addCallback(_check_kids) + + d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) + d.addCallback(lambda n: n.list()) + d.addCallback(_check_kids) # again with dirnode recreated from cap + + # Make sure the lonely child can be listed in HTML... + d.addCallback(lambda ign: self.GET(self.rooturl)) + def _check_html(res): + self.failIfIn("URI:SSK", res) + get_lonely = "".join([r'FILE', + r'\s+', + r'lonely' % (urllib.quote(lonely_uri),), + r'', + r'\s+%d' % len("one"), + ]) + self.failUnless(re.search(get_lonely, res), res) + + # find the More Info link for name, should be relative + mo = re.search(r'More Info', res) + info_url = mo.group(1) + self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url) + d.addCallback(_check_html) + + # ... and in JSON. + d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) + def _check_json(res): + data = simplejson.loads(res) + self.failUnlessEqual(data[0], "dirnode") + listed_children = data[1]["children"] + self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"]) + ll_type, ll_data = listed_children[u"lonely"] + self.failUnlessEqual(ll_type, "filenode") + self.failIf("rw_uri" in ll_data) + self.failUnlessEqual(ll_data["ro_uri"], lonely_uri) + d.addCallback(_check_json) return d def test_deep_check(self): @@ -3156,10 +3540,10 @@ # this tests that deep-check and stream-manifest will ignore # UnknownNode instances. Hopefully this will also cover deep-stats. - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) - d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node)) + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_write_uri, future_read_uri) + d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node)) def _clobber_shares(ignored): self.delete_shares_numbered(self.uris["sick"], [0,1]) diff -rN -u old-tahoe/src/allmydata/unknown.py new-tahoe/src/allmydata/unknown.py --- old-tahoe/src/allmydata/unknown.py 2010-01-23 12:59:10.164000000 +0000 +++ new-tahoe/src/allmydata/unknown.py 2010-01-23 12:59:12.153000000 +0000 @@ -1,29 +1,146 @@ + from zope.interface import implements from twisted.internet import defer -from allmydata.interfaces import IFilesystemNode +from allmydata.interfaces import IFilesystemNode, MustNotBeUnknownRWError +from allmydata import uri +from allmydata.uri import ALLEGED_READONLY_PREFIX, ALLEGED_IMMUTABLE_PREFIX + + +# See ticket #833 for design rationale of UnknownNodes. + +"""Strip prefixes when storing an URI in a ro_uri field.""" +def strip_prefix_for_ro(ro_uri, deep_immutable): + # It is possible for an alleged-immutable URI to be put into a + # mutable directory. In that case the ALLEGED_IMMUTABLE_PREFIX + # should not be stripped. In other cases, the prefix can safely + # be stripped because it is implied by the context. + + if ro_uri.startswith(ALLEGED_IMMUTABLE_PREFIX): + if not deep_immutable: + return ro_uri + return ro_uri[len(ALLEGED_IMMUTABLE_PREFIX):] + elif ro_uri.startswith(ALLEGED_READONLY_PREFIX): + return ro_uri[len(ALLEGED_READONLY_PREFIX):] + else: + return ro_uri class UnknownNode: implements(IFilesystemNode) - def __init__(self, writecap, readcap): - assert writecap is None or isinstance(writecap, str) - self.writecap = writecap - assert readcap is None or isinstance(readcap, str) - self.readcap = readcap + + def __init__(self, rw_uri, ro_uri, deep_immutable=False, + name=u""): + #traceback.print_stack() + #print '%r.__init__(%r, %r, deep_immutable=%r, name=%r)' % (self, rw_uri, ro_uri, deep_immutable, name) + assert rw_uri is None or isinstance(rw_uri, str) + assert ro_uri is None or isinstance(ro_uri, str) + + # We don't raise errors when creating an UnknownNode; we instead create an + # opaque node that records the error. This avoids breaking operations that + # never store the opaque node. + # Note that this means that if a stored dirnode has only a rw_uri, it + # might be dropped. Any future "write-only" cap formats should have a dummy + # unusable read cap to stop that from happening. + + self.error = None + self.rw_uri = self.ro_uri = None + if rw_uri is not None: + if deep_immutable: + self.error = MustNotBeUnknownRWError("cannot attach unknown rw cap as immutable child", + name, True) + return + elif ro_uri is None: + # If we have a single unknown cap (specified as a single cap + # argument, or from a rw_uri slot when ro_uri has been omitted), + # then we cannot tell whether it is a rw_uri, and we cannot + # diminish it to a ro_uri. Prefixing it with ALLEGED_READONLY_PREFIX + # would not be sufficient because we have no reason to believe + # that it is a ro_uri, so that might grant excess authority. + self.error = MustNotBeUnknownRWError("cannot attach unknown rw cap as child", + name, False) + return + + # If ro_uri definitely fails the constraint, it should be treated as opaque. + if ro_uri is not None: + read_cap = uri.from_string(ro_uri, deep_immutable=deep_immutable, name=name) + if isinstance(read_cap, uri.UnknownURI): + self.error = read_cap.get_error() + if self.error: + return + + if deep_immutable: + # strengthen ro_uri to have ALLEGED_IMMUTABLE_PREFIX + if ro_uri is not None: + if ro_uri.startswith(ALLEGED_IMMUTABLE_PREFIX): + self.ro_uri = ro_uri + elif ro_uri.startswith(ALLEGED_READONLY_PREFIX): + self.ro_uri = ALLEGED_IMMUTABLE_PREFIX + ro_uri[len(ALLEGED_READONLY_PREFIX):] + else: + self.ro_uri = ALLEGED_IMMUTABLE_PREFIX + ro_uri + else: + self.rw_uri = rw_uri + # strengthen ro_uri to have ALLEGED_READONLY_PREFIX + if ro_uri is not None: + if (ro_uri.startswith(ALLEGED_READONLY_PREFIX) or + ro_uri.startswith(ALLEGED_IMMUTABLE_PREFIX)): + self.ro_uri = ro_uri + else: + self.ro_uri = ALLEGED_READONLY_PREFIX + ro_uri + + #print 'self.(error, rw_uri, ro_uri) = (%r, %r, %r)' % (self.error, self.rw_uri, self.ro_uri) + + def get_cap(self): + return uri.UnknownURI(self.rw_uri or self.ro_uri) + + def get_readcap(self): + return uri.UnknownURI(self.ro_uri) + + def is_readonly(self): + raise AssertionError("an UnknownNode might be either read-only or " + "read/write, so we shouldn't be calling is_readonly") + + def is_mutable(self): + raise AssertionError("an UnknownNode might be either mutable or immutable, " + "so we shouldn't be calling is_mutable") + + def is_unknown(self): + return True + + def is_allowed_in_immutable_directory(self): + # An UnknownNode consisting only of a ro_uri is allowed in an + # immutable directory, even though we do not know that it is + # immutable (or even read-only), provided that no error was detected. + return not self.error and not self.rw_uri + + def raise_error(self): + if self.error is not None: + raise self.error + def get_uri(self): - return self.writecap + return self.rw_uri or self.ro_uri + + def get_write_uri(self): + return self.rw_uri + def get_readonly_uri(self): - return self.readcap + return self.ro_uri + def get_storage_index(self): return None + def get_verify_cap(self): return None + def get_repair_cap(self): return None + def get_size(self): return None + def get_current_size(self): return defer.succeed(None) + def check(self, monitor, verify, add_lease): return defer.succeed(None) + def check_and_repair(self, monitor, verify, add_lease): return defer.succeed(None) diff -rN -u old-tahoe/src/allmydata/uri.py new-tahoe/src/allmydata/uri.py --- old-tahoe/src/allmydata/uri.py 2010-01-23 12:59:10.175000000 +0000 +++ new-tahoe/src/allmydata/uri.py 2010-01-23 12:59:12.157000000 +0000 @@ -5,14 +5,16 @@ from allmydata.storage.server import si_a2b, si_b2a from allmydata.util import base32, hashutil from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IImmutableFileURI, \ - IVerifierURI, IMutableFileURI, IDirectoryURI, IReadonlyDirectoryURI + IVerifierURI, IMutableFileURI, IDirectoryURI, IReadonlyDirectoryURI, \ + MustBeDeepImmutableError, MustBeReadonlyError class BadURIError(Exception): pass -# the URI shall be an ascii representation of the file. It shall contain -# enough information to retrieve and validate the contents. It shall be -# expressed in a limited character set (namely [TODO]). +# The URI shall be an ASCII representation of a reference to the file/directory. +# It shall contain enough information to retrieve and validate the contents. +# It shall be expressed in a limited character set (currently base32 plus ':' and +# capital letters, but future URIs might use a larger charset). BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) @@ -39,6 +41,10 @@ return self.to_string() != them.to_string() else: return True + + def is_unknown(self): + return False + def to_human_encoding(self): return 'http://127.0.0.1:3456/uri/'+self.to_string() @@ -97,8 +103,10 @@ def is_readonly(self): return True + def is_mutable(self): return False + def get_readonly(self): return self @@ -157,6 +165,18 @@ self.total_shares, self.size)) + def is_readonly(self): + return True + + def is_mutable(self): + return False + + def get_readonly(self): + return self + + def get_verify_cap(self): + return self + class LiteralFileURI(_BaseURI): implements(IURI, IImmutableFileURI) @@ -297,10 +317,13 @@ def is_readonly(self): return True + def is_mutable(self): return True + def get_readonly(self): return self + def get_verify_cap(self): return SSKVerifierURI(self.storage_index, self.fingerprint) @@ -334,6 +357,15 @@ return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), base32.b2a(self.fingerprint)) + def is_readonly(self): + return True + def is_mutable(self): + return False + def get_readonly(self): + return self + def get_verify_cap(self): + return self + class _DirectoryBaseURI(_BaseURI): implements(IURI, IDirnodeURI) def __init__(self, filenode_uri=None): @@ -376,12 +408,12 @@ def abbrev_si(self): return base32.b2a(self._filenode_uri.storage_index)[:5] - def get_filenode_cap(self): - return self._filenode_uri - def is_mutable(self): return True + def get_filenode_cap(self): + return self._filenode_uri + def get_verify_cap(self): return DirectoryURIVerifier(self._filenode_uri.get_verify_cap()) @@ -432,12 +464,12 @@ assert isinstance(filenode_uri, self.INNER_URI_CLASS), filenode_uri _DirectoryBaseURI.__init__(self, filenode_uri) - def is_mutable(self): - return False - def is_readonly(self): return True + def is_mutable(self): + return False + def get_readonly(self): return self @@ -460,6 +492,7 @@ # LIT caps have no verifier, since they aren't distributed return None + def wrap_dirnode_cap(filecap): if isinstance(filecap, WriteableSSKFileURI): return DirectoryURI(filecap) @@ -469,7 +502,8 @@ return ImmutableDirectoryURI(filecap) if isinstance(filecap, LiteralFileURI): return LiteralDirectoryURI(filecap) - assert False, "cannot wrap a dirnode around %s" % filecap.__class__ + assert False, "cannot interpret as a directory cap: %s" % filecap.__class__ + class DirectoryURIVerifier(_DirectoryBaseURI): implements(IVerifierURI) @@ -487,6 +521,10 @@ def get_filenode_cap(self): return self._filenode_uri + def is_mutable(self): + return False + + class ImmutableDirectoryURIVerifier(DirectoryURIVerifier): implements(IVerifierURI) BASE_STRING='URI:DIR2-CHK-Verifier:' @@ -494,68 +532,133 @@ BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-CHK-VERIFIER'+SEP) INNER_URI_CLASS=CHKFileVerifierURI + class UnknownURI: - def __init__(self, uri): + def __init__(self, uri, error=None): self._uri = uri + self._error = error + def to_string(self): return self._uri -def from_string(s): - if not isinstance(s, str): - raise TypeError("unknown URI type: %s.." % str(s)[:100]) - elif s.startswith('URI:CHK:'): + def get_readonly(self): + return None + + def get_error(self): + return self._error + + +ALLEGED_READONLY_PREFIX = 'ro.' +ALLEGED_IMMUTABLE_PREFIX = 'imm.' + +def from_string(u, deep_immutable=False, name=u""): + if not isinstance(u, str): + raise TypeError("unknown URI type: %s.." % str(u)[:100]) + + # We allow and check ALLEGED_READONLY_PREFIX or ALLEGED_IMMUTABLE_PREFIX + # on all URIs, even though we would only strictly need to do so for caps of + # new formats (post Tahoe-LAFS 1.6). URIs that are not consistent with their + # prefix are treated as unknown. This should be revisited when we add the + # new cap formats. See . + s = u + can_be_mutable = can_be_writeable = not deep_immutable + if s.startswith(ALLEGED_IMMUTABLE_PREFIX): + can_be_mutable = can_be_writeable = False + s = s[len(ALLEGED_IMMUTABLE_PREFIX):] + elif s.startswith(ALLEGED_READONLY_PREFIX): + can_be_writeable = False + s = s[len(ALLEGED_READONLY_PREFIX):] + + error = None + if s.startswith('URI:CHK:'): return CHKFileURI.init_from_string(s) elif s.startswith('URI:CHK-Verifier:'): return CHKFileVerifierURI.init_from_string(s) elif s.startswith('URI:LIT:'): return LiteralFileURI.init_from_string(s) elif s.startswith('URI:SSK:'): - return WriteableSSKFileURI.init_from_string(s) + if can_be_writeable: + return WriteableSSKFileURI.init_from_string(s) + error = MustBeReadonlyError("URI:SSK file writecap used in a read-only context", + name) elif s.startswith('URI:SSK-RO:'): - return ReadonlySSKFileURI.init_from_string(s) + if can_be_mutable: + return ReadonlySSKFileURI.init_from_string(s) + error = MustBeDeepImmutableError("URI:SSK-RO readcap to a mutable file used in an immutable context", + name) elif s.startswith('URI:SSK-Verifier:'): return SSKVerifierURI.init_from_string(s) elif s.startswith('URI:DIR2:'): - return DirectoryURI.init_from_string(s) + if can_be_writeable: + return DirectoryURI.init_from_string(s) + error = MustBeReadonlyError("URI:DIR2 directory writecap used in a read-only context", + name) elif s.startswith('URI:DIR2-RO:'): - return ReadonlyDirectoryURI.init_from_string(s) + if can_be_mutable: + return ReadonlyDirectoryURI.init_from_string(s) + error = MustBeDeepImmutableError("URI:DIR2-RO readcap to a mutable directory used in an immutable context", + name) elif s.startswith('URI:DIR2-Verifier:'): return DirectoryURIVerifier.init_from_string(s) elif s.startswith('URI:DIR2-CHK:'): return ImmutableDirectoryURI.init_from_string(s) elif s.startswith('URI:DIR2-LIT:'): return LiteralDirectoryURI.init_from_string(s) - return UnknownURI(s) + elif s.startswith('x-tahoe-future-test-writeable:') and not can_be_writeable: + # For testing how future writeable caps would behave in read-only contexts. + error = MustBeReadonlyError("x-tahoe-future-test-writeable: testing cap used in a read-only context", + name) + elif s.startswith('x-tahoe-future-test-mutable:') and not can_be_mutable: + # For testing how future mutable readcaps would behave in immutable contexts. + error = MustBeDeepImmutableError("x-tahoe-future-test-mutable: testing cap used in an immutable context", + name) + + #if error: print error + return UnknownURI(u, error=error) def is_uri(s): try: - from_string(s) + from_string(s, deep_immutable=False) return True except (TypeError, AssertionError): return False -def from_string_dirnode(s): - u = from_string(s) +def is_literal_file_uri(s): + if not isinstance(s, str): + return False + return (s.startswith('URI:LIT:') or + s.startswith(ALLEGED_READONLY_PREFIX + 'URI:LIT:') or + s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:LIT:')) + +def has_uri_prefix(s): + if not isinstance(s, str): + return False + return (s.startswith("URI:") or + s.startswith(ALLEGED_READONLY_PREFIX + 'URI:') or + s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:')) + +def from_string_dirnode(s, **kwargs): + u = from_string(s, **kwargs) assert IDirnodeURI.providedBy(u) return u registerAdapter(from_string_dirnode, str, IDirnodeURI) -def from_string_filenode(s): - u = from_string(s) +def from_string_filenode(s, **kwargs): + u = from_string(s, **kwargs) assert IFileURI.providedBy(u) return u registerAdapter(from_string_filenode, str, IFileURI) -def from_string_mutable_filenode(s): - u = from_string(s) +def from_string_mutable_filenode(s, **kwargs): + u = from_string(s, **kwargs) assert IMutableFileURI.providedBy(u) return u registerAdapter(from_string_mutable_filenode, str, IMutableFileURI) -def from_string_verifier(s): - u = from_string(s) +def from_string_verifier(s, **kwargs): + u = from_string(s, **kwargs) assert IVerifierURI.providedBy(u) return u registerAdapter(from_string_verifier, str, IVerifierURI) diff -rN -u old-tahoe/src/allmydata/web/common.py new-tahoe/src/allmydata/web/common.py --- old-tahoe/src/allmydata/web/common.py 2010-01-23 12:59:10.472000000 +0000 +++ new-tahoe/src/allmydata/web/common.py 2010-01-23 12:59:12.357000000 +0000 @@ -8,7 +8,8 @@ from nevow.util import resource_filename from allmydata.interfaces import ExistingChildError, NoSuchChildError, \ FileTooLargeError, NotEnoughSharesError, NoSharesError, \ - NotDeepImmutableError, EmptyPathnameComponentError + EmptyPathnameComponentError, MustBeDeepImmutableError, \ + MustBeReadonlyError, MustNotBeUnknownRWError from allmydata.mutable.common import UnrecoverableFileError from allmydata.util import abbreviate # TODO: consolidate @@ -181,9 +182,42 @@ "failure, or disk corruption. You should perform a filecheck on " "this object to learn more.") return (t, http.GONE) - if f.check(NotDeepImmutableError): - t = ("NotDeepImmutableError: a mkdir-immutable operation was given " - "a child that was not itself immutable: %s" % (f.value,)) + if f.check(MustNotBeUnknownRWError): + name = f.value.args[1] + immutable = f.value.args[2] + if immutable: + t = ("MustNotBeUnknownRWError: an operation to add a child named " + "'%s' to a directory was given an unknown cap in a write slot.\n" + "If the cap is actually an immutable readcap, then using a " + "webapi server that supports a later version of Tahoe may help.\n\n" + "If you are using the webapi directly, then specifying an immutable " + "readcap in the read slot (ro_uri) of the JSON PROPDICT, and " + "omitting the write slot (rw_uri), would also work in this " + "case.") % name.encode("utf-8") + else: + t = ("MustNotBeUnknownRWError: an operation to add a child named " + "'%s' to a directory was given an unknown cap in a write slot.\n" + "Using a webapi server that supports a later version of Tahoe " + "may help.\n\n" + "If you are using the webapi directly, specifying a readcap in " + "the read slot (ro_uri) of the JSON PROPDICT, as well as a " + "writecap in the write slot if desired, would also work in this " + "case.") % name.encode("utf-8") + return (t, http.BAD_REQUEST) + if f.check(MustBeDeepImmutableError): + name = f.value.args[1] + t = ("MustBeDeepImmutableError: a cap passed to this operation for " + "the child named '%s', needed to be immutable but was not. Either " + "the cap is being added to an immutable directory, or it was " + "originally retrieved from an immutable directory as an unknown " + "cap." % name.encode("utf-8")) + return (t, http.BAD_REQUEST) + if f.check(MustBeReadonlyError): + name = f.value.args[1] + t = ("MustBeReadonlyError: a cap passed to this operation for " + "the child named '%s', needed to be read-only but was not. " + "The cap is being passed in a read slot (ro_uri), or was retrieved " + "from a read slot as an unknown cap." % name.encode("utf-8")) return (t, http.BAD_REQUEST) if f.check(WebError): return (f.value.text, f.value.code) diff -rN -u old-tahoe/src/allmydata/web/directory.py new-tahoe/src/allmydata/web/directory.py --- old-tahoe/src/allmydata/web/directory.py 2010-01-23 12:59:10.503000000 +0000 +++ new-tahoe/src/allmydata/web/directory.py 2010-01-23 12:59:12.384000000 +0000 @@ -351,7 +351,12 @@ charset = get_arg(req, "_charset", "utf-8") name = name.decode(charset) replace = boolean_of_arg(get_arg(req, "replace", "true")) - d = self.node.set_uri(name, childcap, childcap, overwrite=replace) + + # We mustn't pass childcap for the readcap argument because we don't + # know whether it is a read cap. Passing a read cap as the writecap + # argument will work (it ends up calling NodeMaker.create_from_cap, + # which derives a readcap if necessary and possible). + d = self.node.set_uri(name, childcap, None, overwrite=replace) d.addCallback(lambda res: childcap) return d @@ -362,9 +367,9 @@ # won't show up in the resulting encoded form.. the 'name' # field is completely missing. So to allow deletion of an # empty file, we have to pretend that None means ''. The only - # downide of this is a slightly confusing error message if + # downside of this is a slightly confusing error message if # someone does a POST without a name= field. For our own HTML - # thisn't a big deal, because we create the 'delete' POST + # this isn't a big deal, because we create the 'delete' POST # buttons ourselves. name = '' charset = get_arg(req, "_charset", "utf-8") @@ -584,7 +589,11 @@ def render_title(self, ctx, data): si_s = abbreviated_dirnode(self.node) header = ["Tahoe-LAFS - Directory SI=%s" % si_s] - if self.node.is_readonly(): + if self.node.is_unknown(): + header.append(" (unknown)") + elif not self.node.is_mutable(): + header.append(" (immutable)") + elif self.node.is_readonly(): header.append(" (read-only)") else: header.append(" (modifiable)") @@ -593,7 +602,11 @@ def render_header(self, ctx, data): si_s = abbreviated_dirnode(self.node) header = ["Tahoe-LAFS Directory SI=", T.span(class_="data-chars")[si_s]] - if self.node.is_readonly(): + if self.node.is_unknown(): + header.append(" (unknown)") + elif not self.node.is_mutable(): + header.append(" (immutable)") + elif self.node.is_readonly(): header.append(" (read-only)") return ctx.tag[header] @@ -602,7 +615,7 @@ return T.div[T.a(href=link)["Return to Welcome page"]] def render_show_readonly(self, ctx, data): - if self.node.is_readonly(): + if self.node.is_unknown() or self.node.is_readonly(): return "" rocap = self.node.get_readonly_uri() root = get_root(ctx) @@ -629,7 +642,7 @@ root = get_root(ctx) here = "%s/uri/%s/" % (root, urllib.quote(self.node.get_uri())) - if self.node.is_readonly(): + if self.node.is_unknown() or self.node.is_readonly(): delete = "-" rename = "-" else: @@ -677,8 +690,8 @@ ctx.fillSlots("times", times) assert IFilesystemNode.providedBy(target), target - writecap = target.get_uri() or "" - quoted_uri = urllib.quote(writecap, safe="") # escape slashes too + target_uri = target.get_uri() or "" + quoted_uri = urllib.quote(target_uri, safe="") # escape slashes too if IMutableFileNode.providedBy(target): # to prevent javascript in displayed .html files from stealing a @@ -707,7 +720,7 @@ elif IDirectoryNode.providedBy(target): # directory - uri_link = "%s/uri/%s/" % (root, urllib.quote(writecap)) + uri_link = "%s/uri/%s/" % (root, urllib.quote(target_uri)) ctx.fillSlots("filename", T.a(href=uri_link)[html.escape(name)]) if not target.is_mutable(): @@ -794,35 +807,30 @@ kids = {} for name, (childnode, metadata) in children.iteritems(): assert IFilesystemNode.providedBy(childnode), childnode - rw_uri = childnode.get_uri() + rw_uri = childnode.get_write_uri() ro_uri = childnode.get_readonly_uri() if IFileNode.providedBy(childnode): - if childnode.is_readonly(): - rw_uri = None kiddata = ("filenode", {'size': childnode.get_size(), 'mutable': childnode.is_mutable(), }) elif IDirectoryNode.providedBy(childnode): - if childnode.is_readonly(): - rw_uri = None kiddata = ("dirnode", {'mutable': childnode.is_mutable()}) else: kiddata = ("unknown", {}) + kiddata[1]["metadata"] = metadata - if ro_uri: - kiddata[1]["ro_uri"] = ro_uri if rw_uri: kiddata[1]["rw_uri"] = rw_uri + if ro_uri: + kiddata[1]["ro_uri"] = ro_uri verifycap = childnode.get_verify_cap() if verifycap: kiddata[1]['verify_uri'] = verifycap.to_string() + kids[name] = kiddata - if dirnode.is_readonly(): - drw_uri = None - dro_uri = dirnode.get_uri() - else: - drw_uri = dirnode.get_uri() - dro_uri = dirnode.get_readonly_uri() + + drw_uri = dirnode.get_write_uri() + dro_uri = dirnode.get_readonly_uri() contents = { 'children': kids } if dro_uri: contents['ro_uri'] = dro_uri @@ -833,13 +841,14 @@ contents['verify_uri'] = verifycap.to_string() contents['mutable'] = dirnode.is_mutable() data = ("dirnode", contents) - return simplejson.dumps(data, indent=1) + "\n" + json = simplejson.dumps(data, indent=1) + "\n" + #print json + return json d.addCallback(_got) d.addCallback(text_plain, ctx) return d - def DirectoryURI(ctx, dirnode): return text_plain(dirnode.get_uri(), ctx) diff -rN -u old-tahoe/src/allmydata/web/filenode.py new-tahoe/src/allmydata/web/filenode.py --- old-tahoe/src/allmydata/web/filenode.py 2010-01-23 12:59:10.572000000 +0000 +++ new-tahoe/src/allmydata/web/filenode.py 2010-01-23 12:59:12.403000000 +0000 @@ -6,10 +6,9 @@ from nevow import url, rend from nevow.inevow import IRequest -from allmydata.interfaces import ExistingChildError, CannotPackUnknownNodeError +from allmydata.interfaces import ExistingChildError from allmydata.monitor import Monitor from allmydata.immutable.upload import FileHandle -from allmydata.unknown import UnknownNode from allmydata.util import log, base32 from allmydata.web.common import text_plain, WebError, RenderMixin, \ @@ -20,7 +19,6 @@ from allmydata.web.info import MoreInfo class ReplaceMeMixin: - def replace_me_with_a_child(self, req, client, replace): # a new file is being uploaded in our place. mutable = boolean_of_arg(get_arg(req, "mutable", "false")) @@ -55,14 +53,7 @@ def replace_me_with_a_childcap(self, req, client, replace): req.content.seek(0) childcap = req.content.read() - childnode = client.create_node_from_uri(childcap, childcap+"readonly") - if isinstance(childnode, UnknownNode): - # don't be willing to pack unknown nodes: we might accidentally - # put some write-authority into the rocap slot because we don't - # know how to diminish the URI they gave us. We don't even know - # if they gave us a readcap or a writecap. - msg = "cannot attach unknown node as child %s" % str(self.name) - raise CannotPackUnknownNodeError(msg) + childnode = client.create_node_from_uri(childcap, None, name=self.name) d = self.parentnode.set_node(self.name, childnode, overwrite=replace) d.addCallback(lambda res: childnode.get_uri()) return d @@ -426,12 +417,8 @@ def FileJSONMetadata(ctx, filenode, edge_metadata): - if filenode.is_readonly(): - rw_uri = None - ro_uri = filenode.get_uri() - else: - rw_uri = filenode.get_uri() - ro_uri = filenode.get_readonly_uri() + rw_uri = filenode.get_write_uri() + ro_uri = filenode.get_readonly_uri() data = ("filenode", {}) data[1]['size'] = filenode.get_size() if ro_uri: diff -rN -u old-tahoe/src/allmydata/web/info.py new-tahoe/src/allmydata/web/info.py --- old-tahoe/src/allmydata/web/info.py 2010-01-23 12:59:10.609000000 +0000 +++ new-tahoe/src/allmydata/web/info.py 2010-01-23 12:59:12.419000000 +0000 @@ -21,6 +21,8 @@ def get_type(self): node = self.original if IDirectoryNode.providedBy(node): + if not node.is_mutable(): + return "immutable directory" return "directory" if IFileNode.providedBy(node): si = node.get_storage_index() @@ -28,7 +30,7 @@ if node.is_mutable(): return "mutable file" return "immutable file" - return "LIT file" + return "immutable LIT file" return "unknown" def render_title(self, ctx, data): @@ -68,10 +70,10 @@ def render_directory_writecap(self, ctx, data): node = self.original - if node.is_readonly(): - return "" if not IDirectoryNode.providedBy(node): return "" + if node.is_readonly(): + return "" return ctx.tag[node.get_uri()] def render_directory_readcap(self, ctx, data): @@ -86,27 +88,24 @@ return "" return ctx.tag[node.get_verify_cap().to_string()] - def render_file_writecap(self, ctx, data): node = self.original if IDirectoryNode.providedBy(node): node = node._node - if ((IDirectoryNode.providedBy(node) or IFileNode.providedBy(node)) - and node.is_readonly()): - return "" - writecap = node.get_uri() - if not writecap: + write_uri = node.get_write_uri() + #print "write_uri = %r, node = %r" % (write_uri, node) + if not write_uri: return "" - return ctx.tag[writecap] + return ctx.tag[write_uri] def render_file_readcap(self, ctx, data): node = self.original if IDirectoryNode.providedBy(node): node = node._node - readcap = node.get_readonly_uri() - if not readcap: + read_uri = node.get_readonly_uri() + if not read_uri: return "" - return ctx.tag[readcap] + return ctx.tag[read_uri] def render_file_verifycap(self, ctx, data): node = self.original diff -rN -u old-tahoe/src/allmydata/web/root.py new-tahoe/src/allmydata/web/root.py --- old-tahoe/src/allmydata/web/root.py 2010-01-23 12:59:10.718000000 +0000 +++ new-tahoe/src/allmydata/web/root.py 2010-01-23 12:59:12.488000000 +0000 @@ -12,7 +12,7 @@ from allmydata import get_package_versions_string from allmydata import provisioning from allmydata.util import idlib, log -from allmydata.interfaces import IFileNode, UnhandledCapTypeError +from allmydata.interfaces import IFileNode from allmydata.web import filenode, directory, unlinked, status, operations from allmydata.web import reliability, storage from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \ @@ -85,7 +85,7 @@ try: node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) - except (TypeError, UnhandledCapTypeError, AssertionError): + except (TypeError, AssertionError): raise WebError("'%s' is not a valid file- or directory- cap" % name) @@ -104,7 +104,7 @@ # 'name' must be a file URI try: node = self.client.create_node_from_uri(name) - except (TypeError, UnhandledCapTypeError, AssertionError): + except (TypeError, AssertionError): # I think this can no longer be reached raise WebError("'%s' is not a valid file- or directory- cap" % name) diff -rN -u old-tahoe/contrib/fuse/impl_c/blackmatch.py new-tahoe/contrib/fuse/impl_c/blackmatch.py --- old-tahoe/contrib/fuse/impl_c/blackmatch.py 2010-01-23 12:59:10.975000000 +0000 +++ new-tahoe/contrib/fuse/impl_c/blackmatch.py 2010-01-23 12:59:12.773000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env python #----------------------------------------------------------------------------------------------- -from allmydata.uri import CHKFileURI, DirectoryURI, LiteralFileURI +from allmydata.uri import CHKFileURI, DirectoryURI, LiteralFileURI, is_literal_file_uri from allmydata.scripts.common_http import do_http as do_http_req from allmydata.util.hashutil import tagged_hash from allmydata.util.assertutil import precondition @@ -335,7 +335,7 @@ self.fname = self.tfs.cache.tmp_file(os.urandom(20)) if self.fnode is None: log('TFF: [%s] open() for write: no file node, creating new File %s' % (self.name, self.fname, )) - self.fnode = File(0, 'URI:LIT:') + self.fnode = File(0, LiteralFileURI.BASE_STRING) self.fnode.tmp_fname = self.fname # XXX kill this self.parent.add_child(self.name, self.fnode, {}) elif hasattr(self.fnode, 'tmp_fname'): @@ -362,7 +362,7 @@ self.fname = self.fnode.tmp_fname log('TFF: reopening(%s) for reading' % self.fname) else: - if uri.startswith("URI:LIT") or not self.tfs.async: + if is_literal_file_uri(uri) or not self.tfs.async: log('TFF: synchronously fetching file from cache for reading') self.fname = self.tfs.cache.get_file(uri) else: @@ -906,7 +906,7 @@ class TStat(fuse.Stat): # in fuse 0.2, these are set by fuse.Stat.__init__ - # in fuse 0.2-pre3 (hardy) they are not. badness unsues if they're missing + # in fuse 0.2-pre3 (hardy) they are not. badness ensues if they're missing st_mode = None st_ino = 0 st_dev = 0 @@ -1237,7 +1237,7 @@ def get_file(self, uri): self.log('get_file(%s)' % (uri,)) - if uri.startswith("URI:LIT"): + if is_literal_file_uri(uri): return self.get_literal(uri) else: return self.get_chk(uri, async=False)