tahoe-lafs/tahoe-lafs
tahoe-lafs/tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

directories: make initialization of the download cache lazy

Browse Source 
If you open up a directory containing thousands of files, it currently computes the cache filename and checks for the cache file on disk immediately for each immutble file in that directory.  With this patch, it delays those steps until you try to do something with an immutable file that could use the cache.
This commit is contained in:
Zooko O'Whielacronx 2009-07-07 17:40:40 -07:00
parent 903005a528
commit c0d1e7deae
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/allmydata/client.py
View File

@ -428,9 +428,7 @@ class Client(node.Node, pollmixin.PollMixin):
if isinstance(u, LiteralFileURI): if isinstance(u, LiteralFileURI):
node = LiteralFileNode(u, self) # LIT node = LiteralFileNode(u, self) # LIT
else: else:
key = base32.b2a(u.storage_index) node = FileNode(u, self, self.download_cache_dirman) # CHK
cachefile = self.download_cache_dirman.get_file(key)
node = FileNode(u, self, cachefile) # CHK
else: else:
assert IMutableFileURI.providedBy(u), u assert IMutableFileURI.providedBy(u), u
node = MutableFileNode(self).init_from_uri(u) node = MutableFileNode(self).init_from_uri(u)

15
src/allmydata/immutable/filenode.py
View File

@ -68,12 +68,13 @@ class PortionOfFile:
class DownloadCache: class DownloadCache:
implements(IDownloadTarget) implements(IDownloadTarget)
def __init__(self, node, cachefile): def __init__(self, node, cachedirectorymanager):
self._downloader = node._client.getServiceNamed("downloader") self._downloader = node._client.getServiceNamed("downloader")
self._uri = node.get_uri() self._uri = node.get_uri()
self._storage_index = node.get_storage_index() self._storage_index = node.get_storage_index()
self.milestones = set() # of (offset,size,Deferred) self.milestones = set() # of (offset,size,Deferred)
self.cachefile = cachefile self.cachedirectorymanager = cachedirectorymanager
self.cachefile = None
self.download_in_progress = False self.download_in_progress = False
# five states: # five states:
# new FileNode, no downloads ever performed # new FileNode, no downloads ever performed
@ -103,6 +104,8 @@ class DownloadCache:
def read(self, consumer, offset, size): def read(self, consumer, offset, size):
assert offset+size <= self.get_filesize() assert offset+size <= self.get_filesize()
if not self.cachefile:
self.cachefile = self.cachedirectorymanager.get_file(self._storage_index)
f = PortionOfFile(self.cachefile.get_filename(), offset, size) f = PortionOfFile(self.cachefile.get_filename(), offset, size)
d = basic.FileSender().beginFileTransfer(f, consumer) d = basic.FileSender().beginFileTransfer(f, consumer)
d.addCallback(lambda lastSent: consumer) d.addCallback(lambda lastSent: consumer)
@ -142,6 +145,8 @@ class DownloadCache:
umid="8PKOhg", level=log.NOISY) umid="8PKOhg", level=log.NOISY)
def get_filesize(self): def get_filesize(self):
if not self.cachefile:
self.cachefile = self.cachedirectorymanager.get_file(self._storage_index)
try: try:
filesize = os.stat(self.cachefile.get_filename())[stat.ST_SIZE] filesize = os.stat(self.cachefile.get_filename())[stat.ST_SIZE]
except OSError: except OSError:
@ -150,6 +155,8 @@ class DownloadCache:
def open(self, size): def open(self, size):
if not self.cachefile:
self.cachefile = self.cachedirectorymanager.get_file(self._storage_index)
self.f = open(self.cachefile.get_filename(), "wb") self.f = open(self.cachefile.get_filename(), "wb")
def write(self, data): def write(self, data):
@ -176,9 +183,9 @@ class DownloadCache:
class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin): class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin):
def __init__(self, uri, client, cachefile): def __init__(self, uri, client, cachedirectorymanager):
_ImmutableFileNodeBase.__init__(self, uri, client) _ImmutableFileNodeBase.__init__(self, uri, client)
self.download_cache = DownloadCache(self, cachefile) self.download_cache = DownloadCache(self, cachedirectorymanager)
prefix = uri.get_verify_cap().to_string() prefix = uri.get_verify_cap().to_string()
log.PrefixingLogMixin.__init__(self, "allmydata.immutable.filenode", prefix=prefix) log.PrefixingLogMixin.__init__(self, "allmydata.immutable.filenode", prefix=prefix)
self.log("starting", level=log.OPERATIONAL) self.log("starting", level=log.OPERATIONAL)