tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

Rename _turn_delay and friends to make things more clear 

This forces the Uploader and Downloader to implement a _scan_delay
method and makes the naming more consistent with what's actually
happening. Also, fix a few "bugs" in the names of args in the
mocks for some tests.
This commit is contained in:
meejah 2016-10-24 17:09:57 -06:00 committed by Brian Warner
parent ddaa29ce39
commit da4e1589bb
4 changed files with 36 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/allmydata/frontends/magic_folder.py
View File

@ -63,7 +63,7 @@ class MagicFolder(service.MultiService):
name = 'magic-folder'
def __init__(self, client, upload_dircap, collective_dircap, local_path_u, dbfile, umask,
pending_delay=1.0, clock=None, poll_interval=3):
uploader_delay=1.0, clock=None, downloader_delay=3):
precondition_abspath(local_path_u)
service.MultiService.__init__(self)
@ -80,10 +80,10 @@ class MagicFolder(service.MultiService):
upload_dirnode = self._client.create_node_from_uri(upload_dircap)
collective_dirnode = self._client.create_node_from_uri(collective_dircap)
self.uploader = Uploader(client, local_path_u, db, upload_dirnode, pending_delay, clock)
self.uploader = Uploader(client, local_path_u, db, upload_dirnode, uploader_delay, clock)
self.downloader = Downloader(client, local_path_u, db, collective_dirnode,
upload_dirnode.get_readonly_uri(), clock, self.uploader.is_pending, umask,
self.set_public_status, poll_interval=poll_interval)
self.set_public_status, poll_interval=downloader_delay)
self._public_status = (False, ['Magic folder has not yet started'])
def enable_debug_log(self, enabled=True):
@ -204,16 +204,16 @@ class QueueMixin(HookMixin):
This is an infinite loop that processes things out of the _deque.
One iteration runs self._process_deque which calls
_when_queue_is_empty() and then completely drains the _deque
_perform_scan() and then completely drains the _deque
(processing each item). After that we yield for _turn_deque
seconds.
"""
while not self._stopped:
self._log("doing iteration")
d = task.deferLater(self._clock, self._next_scan_delay(), lambda: None)
d = task.deferLater(self._clock, self._scan_delay(), lambda: None)
# adds items to our deque
yield self._when_queue_is_empty()
yield self._perform_scan()
# process anything in our queue
yield self._process_deque()
@ -230,10 +230,10 @@ class QueueMixin(HookMixin):
self._log("stopped")
def _next_scan_delay(self):
return self._turn_delay
def _scan_delay(self):
raise NotImplementedError
def _when_queue_is_empty(self):
def _perform_scan(self):
return
@defer.inlineCallbacks
@ -352,11 +352,11 @@ class Uploader(QueueMixin):
self._upload_dirnode = upload_dirnode
self._inotify = get_inotify_module()
self._notifier = self._inotify.INotify()
self._pending = set() # of unicode relpaths
self._pending = set() # of unicode relpaths
self._pending_delay = pending_delay
self._periodic_full_scan_duration = 10 * 60 # perform a full scan every 10 minutes
self._periodic_callid = None
self._turn_delay = pending_delay
if hasattr(self._notifier, 'set_pending_delay'):
self._notifier.set_pending_delay(pending_delay)
@ -416,6 +416,9 @@ class Uploader(QueueMixin):
# *really* just call this synchronously.
return self._begin_processing(None)
def _scan_delay(self):
return self._pending_delay
def _full_scan(self):
self._periodic_callid = self._clock.callLater(self._periodic_full_scan_duration, self._full_scan)
self._log("FULL SCAN")
@ -769,7 +772,6 @@ class Downloader(QueueMixin, WriteFileMixin):
@defer.inlineCallbacks
def start_downloading(self):
self._log("start_downloading")
self._turn_delay = self._poll_interval
files = self._db.get_all_relpaths()
self._log("all files %s" % files)
@ -941,13 +943,15 @@ class Downloader(QueueMixin, WriteFileMixin):
d.addCallback(_filter_batch_to_deque)
return d
def _scan_delay(self):
return self._poll_interval
@defer.inlineCallbacks
def _when_queue_is_empty(self):
# XXX can we amalgamate all the "scan" stuff and just call it
# directly from QueueMixin?
def _perform_scan(self):
x = None
try:
x = yield self._scan(None)
x = yield self._scan_remote_collective()
self._status_reporter(
True, 'Magic folder is working',
'Last scan: %s' % self.nice_current_time(),
@ -961,9 +965,6 @@ class Downloader(QueueMixin, WriteFileMixin):
)
defer.returnValue(x)
def _scan(self, ign):
return self._scan_remote_collective()
def _process(self, item):
# Downloader
self._log("_process(%r)" % (item,))

8
src/allmydata/test/cli/test_magic_folder.py
View File

@ -122,8 +122,8 @@ class MagicFolderCLITestMixin(CLITestMixin, GridTestMixin, NonASCIIPathMixin):
d = defer.succeed(None)
def _clean(ign):
d = self.magicfolder.finish()
self.magicfolder.uploader._clock.advance(self.magicfolder.uploader._turn_delay + 1)
self.magicfolder.downloader._clock.advance(self.magicfolder.downloader._turn_delay + 1)
self.magicfolder.uploader._clock.advance(self.magicfolder.uploader._pending_delay + 1)
self.magicfolder.downloader._clock.advance(self.magicfolder.downloader._poll_interval + 1)
return d
d.addCallback(_clean)
@ -139,10 +139,10 @@ class MagicFolderCLITestMixin(CLITestMixin, GridTestMixin, NonASCIIPathMixin):
local_path_u=local_magic_dir,
dbfile=dbfile,
umask=0o077,
pending_delay=0.2,
clock=clock,
uploader_delay=0.2,
downloader_delay=0,
)
magicfolder.downloader._turn_delay = 0
magicfolder.setServiceParent(self.get_client(client_num))
magicfolder.ready()

10
src/allmydata/test/test_client.py
View File

@ -322,14 +322,14 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
class MockMagicFolder(service.MultiService):
name = 'magic-folder'
def __init__(self, client, upload_dircap, collective_dircap, local_dir, dbfile, umask, inotify=None,
pending_delay=1.0):
def __init__(self, client, upload_dircap, collective_dircap, local_path_u, dbfile, umask, inotify=None,
uploader_delay=1.0, clock=None, downloader_delay=3):
service.MultiService.__init__(self)
self.client = client
self._umask = umask
self.upload_dircap = upload_dircap
self.collective_dircap = collective_dircap
self.local_dir = local_dir
self.local_dir = local_path_u
self.dbfile = dbfile
self.inotify = inotify
@ -376,8 +376,8 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test
class Boom(Exception):
pass
def BoomMagicFolder(client, upload_dircap, collective_dircap, local_dir, dbfile,
inotify=None, pending_delay=1.0):
def BoomMagicFolder(client, upload_dircap, collective_dircap, local_path_u, dbfile,
umask, inotify=None, uploader_delay=1.0, clock=None, downloader_delay=3):
raise Boom()
self.patch(allmydata.frontends.magic_folder, 'MagicFolder', BoomMagicFolder)

14
src/allmydata/test/test_magic_folder.py
View File

@ -111,13 +111,13 @@ def iterate_downloader(magic):
# can do either of these:
#d = magic.downloader._process_deque()
d = magic.downloader.set_hook('iteration')
magic.downloader._clock.advance(magic.downloader._turn_delay + 1)
magic.downloader._clock.advance(magic.downloader._poll_interval + 1)
return d
def iterate_uploader(magic):
d = magic.uploader.set_hook('iteration')
magic.uploader._clock.advance(magic.uploader._turn_delay + 1)
magic.uploader._clock.advance(magic.uploader._pending_delay + 1)
return d
@defer.inlineCallbacks
@ -308,7 +308,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
self.alice_fileops = FileOperationsHelper(self.alice_magicfolder.uploader, self.inject_inotify)
d0 = self.alice_magicfolder.uploader.set_hook('iteration')
d1 = self.alice_magicfolder.downloader.set_hook('iteration')
self.alice_clock.advance(self.alice_magicfolder.uploader._turn_delay + 1)
self.alice_clock.advance(self.alice_magicfolder.uploader._pending_delay + 1)
d0.addCallback(lambda ign: d1)
d0.addCallback(lambda ign: result)
return d0
@ -332,7 +332,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
self.bob_fileops = FileOperationsHelper(self.bob_magicfolder.uploader, self.inject_inotify)
d0 = self.bob_magicfolder.uploader.set_hook('iteration')
d1 = self.bob_magicfolder.downloader.set_hook('iteration')
self.bob_clock.advance(self.alice_magicfolder.uploader._turn_delay + 1)
self.bob_clock.advance(self.alice_magicfolder.uploader._pending_delay + 1)
d0.addCallback(lambda ign: d1)
d0.addCallback(lambda ign: result)
return d0
@ -346,8 +346,8 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d1 = self.bob_magicfolder.finish()
for mf in [self.alice_magicfolder, self.bob_magicfolder]:
for loader in [mf.uploader, mf.downloader]:
loader._clock.advance(loader._turn_delay + 1)
mf.uploader._clock.advance(mf.uploader._pending_delay + 1)
mf.downloader._clock.advance(mf.downloader._poll_interval + 1)
yield d0
yield d1
@ -541,7 +541,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now, we ONLY want to do the scan, not a full iteration of
# the process loop. So we do just the scan part "by hand" in
# Bob's downloader
yield self.bob_magicfolder.downloader._when_queue_is_empty()
yield self.bob_magicfolder.downloader._perform_scan()
# while we're delving into internals, I guess we might as well
# confirm that we did queue up an item to download
self.assertEqual(1, len(self.bob_magicfolder.downloader._deque))