tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

Fix file-operations/inotify testing 

This re-factors the magic-folder tests to abstract
the whole "do a file operation" so we can properly
send fake (or wait for real) inotify events to the
uploader/downloader. This speeds up the tests quite
a bit and makes test_alice_bob reasonable again (at
about 1.5s instead of over 30s).
This commit is contained in:
meejah 2016-08-16 16:29:37 -06:00
parent b637636ef9
commit 054efe055c
3 changed files with 141 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/allmydata/frontends/magic_folder.py
View File

@ -27,7 +27,6 @@ from allmydata.util.time_format import format_time
from allmydata.immutable.upload import FileName, Data
from allmydata import magicfolderdb, magicpath
defer.setDebugging(True)
IN_EXCL_UNLINK = 0x04000000L
def get_inotify_module():
@ -109,11 +108,14 @@ class MagicFolder(service.MultiService):
self.uploader.start_uploading() # synchronous, returns None
return self.downloader.start_downloading()
@defer.inlineCallbacks
def finish(self):
d = self.uploader.stop()
d2 = self.downloader.stop()
d.addCallback(lambda ign: d2)
return d
# must stop these concurrently so that the clock.advance()s
# work correctly in the tests. Also, it's arguably
# most-correct.
d0 = self.downloader.stop()
d1 = self.uploader.stop()
yield defer.DeferredList([d0, d1])
def remove_service(self):
return service.MultiService.disownServiceParent(self)
@ -133,6 +135,7 @@ class QueueMixin(HookMixin):
'processed': None,
'started': None,
'iteration': None,
'inotify': None,
}
self.started_d = self.set_hook('started')
@ -371,6 +374,7 @@ class Uploader(QueueMixin):
def stop(self):
self._log("stop")
self._stopped = True
self._notifier.stopReading()
self._count('dirs_monitored', -1)
self.periodic_callid.cancel()
@ -378,7 +382,6 @@ class Uploader(QueueMixin):
d = self._notifier.wait_until_stopped()
else:
d = defer.succeed(None)
self._stopped = True
# wait for processing loop to actually exit
d.addCallback(lambda ign: self._processing)
return d
@ -467,6 +470,7 @@ class Uploader(QueueMixin):
return
self._add_pending(relpath_u)
self._call_hook(path, 'inotify')
def _process(self, item):
# Uploader

12
src/allmydata/test/cli/test_magic_folder.py
View File

@ -126,8 +126,16 @@ class MagicFolderCLITestMixin(CLITestMixin, GridTestMixin):
def init_magicfolder(self, client_num, upload_dircap, collective_dircap, local_magic_dir, clock):
dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.get_clientdir(i=client_num))
magicfolder = MagicFolder(self.get_client(client_num), upload_dircap, collective_dircap, local_magic_dir,
dbfile, 0077, pending_delay=0.2, clock=clock)
magicfolder = MagicFolder(
client=self.get_client(client_num),
upload_dircap=upload_dircap,
collective_dircap=collective_dircap,
local_path_u=local_magic_dir,
dbfile=dbfile,
umask=0o077,
pending_delay=0.2,
clock=clock,
)
magicfolder.downloader._turn_delay = 0
magicfolder.setServiceParent(self.get_client(client_num))

239
src/allmydata/test/test_magic_folder.py
View File

@ -130,6 +130,63 @@ def iterate(magic):
yield iterate_downloader(magic)
class FileOperationsHelper(object):
"""
This abstracts all file operations we might do in magic-folder unit-tests.
This is so we can correctly wait for inotify events to 'actually'
propagate. For the mock tests this is easy, since we're sending
them sychronously. For the Real tests we have to wait for the
actual inotify thing.
We could write this as a mixin instead; might fit existing style better?
"""
def __init__(self, uploader, inject_events=False):
self._uploader = uploader
self._inotify = fake_inotify # fixme?
self._fake_inotify = inject_events
def move(self, from_path_u, to_path_u):
from_fname = from_path_u
to_fname = to_path_u
d = self._uploader.set_hook('inotify')
os.rename(from_fname, to_fname)
self._maybe_notify(to_fname, self._inotify.IN_MOVED_TO)
# hmm? we weren't faking IN_MOVED_FROM previously .. but seems like we should have been?
# self._uploader._notifier.event(to_filepath(from_fname), self._inotify.IN_MOVED_FROM)
return d
def write(self, path_u, contents):
fname = path_u
d = self._uploader.set_hook('inotify')
with open(fname, "wb") as f:
f.write(contents)
self._maybe_notify(fname, self._inotify.IN_CLOSE_WRITE)
return d
def mkdir(self, path_u):
fname = path_u
d = self._uploader.set_hook('inotify')
os.mkdir(fname)
self._maybe_notify(fname, self._inotify.IN_CREATE | self._inotify.IN_ISDIR)
return d
def delete(self, path_u):
fname = path_u
d = self._uploader.set_hook('inotify')
os.unlink(fname)
self._maybe_notify(fname, self._inotify.IN_DELETE)
return d
def _maybe_notify(self, fname, mask):
if self._fake_inotify:
self._uploader._notifier.event(to_filepath(fname), self._inotify.IN_DELETE)
class CheckerMixin(object):
"""
Factored out of one of the many test classes.
@ -153,25 +210,17 @@ class CheckerMixin(object):
previously_disappeared = self._get_count('uploader.objects_disappeared')
path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
path = to_filepath(path_u)
if directory:
os.mkdir(path_u)
event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
yield self.fileops.mkdir(path_u)
else:
# We don't use FilePath.setContent() here because it creates a temporary file that
# is renamed into place, which causes events that the test is not expecting.
f = open(path_u, "wb")
try:
f.write(data)
finally:
f.close()
yield self.fileops.write(path_u, data)
if temporary:
os.unlink(path_u)
yield self.notify(path, self.inotify.IN_DELETE, flush=False)
event_mask = self.inotify.IN_CLOSE_WRITE
yield iterate(self.magicfolder)
yield self.fileops.delete(path_u)
yield self.notify(path, event_mask)
yield iterate(self.magicfolder)
encoded_name_u = magicpath.path2magic(name_u)
@ -222,6 +271,7 @@ class CheckerMixin(object):
class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin, CheckerMixin):
inject_inotify = False
def setUp(self):
# super(MagicFolderAliceBobTestMixin, self).setUp() # XXX huh, why isn't this working?
@ -260,6 +310,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
self.alice_magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
self.alice_collective_dircap,
self.alice_magic_dir, self.alice_clock)
self.alice_fileops = FileOperationsHelper(self.alice_magicfolder.uploader, self.inject_inotify)
d0 = self.alice_magicfolder.uploader.set_hook('iteration')
d1 = self.alice_magicfolder.downloader.set_hook('iteration')
self.alice_clock.advance(self.alice_magicfolder.uploader.scan_interval + 1)
@ -283,6 +334,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
self.bob_magicfolder = self.init_magicfolder(1, self.bob_upload_dircap,
self.bob_collective_dircap,
self.bob_magic_dir, self.bob_clock)
self.bob_fileops = FileOperationsHelper(self.bob_magicfolder.uploader, self.inject_inotify)
d0 = self.bob_magicfolder.uploader.set_hook('iteration')
d1 = self.bob_magicfolder.downloader.set_hook('iteration')
self.bob_clock.advance(self.alice_magicfolder.uploader.scan_interval + 1)
@ -292,28 +344,30 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
d.addCallback(get_Bob_magicfolder)
return d
@defer.inlineCallbacks
def tearDown(self):
d = GridTestMixin.tearDown(self)
d.addCallback(lambda ign: self.alice_magicfolder.finish())
d.addCallback(lambda ign: self.bob_magicfolder.finish())
yield GridTestMixin.tearDown(self)
d0 = self.alice_magicfolder.finish()
d1 = self.bob_magicfolder.finish()
for mf in [self.alice_magicfolder, self.bob_magicfolder]:
for loader in [mf.uploader, mf.downloader]:
loader._clock.advance(loader.scan_interval + 1)
# XXX double-check: are self.mktemp() dirs blown away automagically?
return d
yield d0
yield d1
@defer.inlineCallbacks
def test_alice_delete_bob_restore(self):
alice_fname = os.path.join(self.alice_magic_dir, 'blam')
bob_fname = os.path.join(self.bob_magic_dir, 'blam')
alice_up = self.alice_magicfolder.uploader.set_hook('processed')
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
yield self.alice_fileops.write(alice_fname, 'contents0\n')
# alice uploads
yield iterate_uploader(self.alice_magicfolder)
yield alice_up
yield alice_proc
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0)
yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
@ -338,8 +392,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now bob deletes it (bob should upload, alice download)
bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
os.unlink(bob_fname)
yield self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
yield self.bob_fileops.delete(bob_fname)
yield iterate_uploader(self.bob_magicfolder)
yield bob_proc
@ -354,11 +407,14 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
# not *entirely* sure why we need to iterate Alice for the
# real test here. But, we do.
yield iterate(self.alice_magicfolder)
# now alice restores it (alice should upload, bob download)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'new contents\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'new contents\n')
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc
@ -384,8 +440,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc # alice uploads
@ -410,11 +465,10 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now bob deletes it (bob should upload, alice download)
bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
os.unlink(bob_fname)
yield self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
yield self.bob_fileops.delete(bob_fname)
# just after notifying bob, we also delete alice's,
# covering the 'except' flow in _rename_deleted_file()
os.unlink(alice_fname)
yield self.alice_fileops.delete(alice_fname)
yield iterate_uploader(self.bob_magicfolder)
yield bob_proc
@ -435,8 +489,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
bob_fname = os.path.join(self.bob_magic_dir, 'blam')
# alice creates a file, bob downloads it
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate(self.alice_magicfolder)
yield iterate(self.bob_magicfolder)
@ -456,8 +509,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
)
# now bob updates it (bob should upload, alice download)
fileutil.write(bob_fname, 'bob wuz here\n')
yield self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
yield self.bob_fileops.write(bob_fname, 'bob wuz here\n')
yield iterate(self.bob_magicfolder)
yield iterate(self.alice_magicfolder)
@ -474,8 +526,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# bob_fname = os.path.join(self.bob_magic_dir, 'blam')
# Alice creates a file
fileutil.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)]))
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, ''.join(['contents-%04d\n' % i for i in range(1024)]))
yield iterate(self.alice_magicfolder)
# check alice created the file
yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 0)
@ -531,8 +582,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'contents0\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'contents0\n')
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc # alice uploads
@ -558,8 +608,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now alice deletes it (alice should upload, bob download)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
os.unlink(alice_fname)
yield self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
yield self.alice_fileops.delete(alice_fname)
yield iterate_uploader(self.alice_magicfolder)
yield alice_proc
@ -576,8 +625,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# now alice restores the file (with new contents)
alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
fileutil.write(alice_fname, 'alice wuz here\n')
yield self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(alice_fname, 'alice wuz here\n')
yield iterate_uploader(self.alice_magicfolder)
yield iterate_downloader(self.alice_magicfolder) # why?
@ -644,9 +692,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
def Alice_to_write_a_file():
if _debug: print "Alice writes a file\n\n\n\n\n"
self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
yield task.deferLater(reactor, 5, lambda: None)
fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
yield self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
yield self.alice_fileops.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
d.addCallback(_wait_for, Alice_to_write_a_file)
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
@ -669,19 +715,19 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
@defer.inlineCallbacks
def Alice_to_delete_file():
if _debug: print "Alice deletes the file!\n\n\n\n"
yield task.deferLater(reactor, 5, lambda: None)
os.unlink(self.file_path)
yield self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
yield self.alice_fileops.delete(self.file_path)
yield iterate(self.alice_magicfolder)
yield iterate(self.bob_magicfolder)
d.addCallback(_wait_for, Alice_to_delete_file)
@defer.inlineCallbacks
def notify_bob_moved(ign):
# WARNING: this is just directly notifying for the mock
# tests, because in the Real* tests the .backup file will
# me moved into place (from the original)
p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
fileutil.write((p + u'.backup'), "meow, meow meow. meow? meow meow! meow.")
yield self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
yield self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
if self.bob_fileops._fake_inotify:
self.bob_magicfolder.uploader._notifier.event(to_filepath(p + u'.backup'), fake_inotify.IN_MOVED_TO)
yield iterate(self.bob_magicfolder)
d.addCallback(notify_bob_moved)
@ -702,8 +748,10 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
def Alice_to_rewrite_file():
if _debug: print "Alice rewrites file\n"
self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
return self.alice_fileops.write(
self.file_path,
"Alice suddenly sees the white rabbit running into the forest.",
)
d.addCallback(_wait_for, Alice_to_rewrite_file)
d.addCallback(lambda ign: iterate(self.bob_magicfolder))
@ -754,9 +802,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
if _debug: print "Bob rewrites file\n"
self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
if _debug: print "---- bob's file is %r" % (self.file_path,)
fileutil.write(self.file_path, "No white rabbit to be found.")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
d.addCallback(lambda ign: task.deferLater(reactor, 5, lambda: None))
return self.bob_fileops.write(self.file_path, "No white rabbit to be found.")
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
@ -802,16 +848,11 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
# prepare to perform another conflict test
@defer.inlineCallbacks
def Alice_to_write_file2():
# uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
if _debug: print "Alice writes a file2\n"
yield task.deferLater(reactor, 5, lambda: None)
self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
fileutil.write(self.file_path, "something")
d = self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
d = self.alice_fileops.write(self.file_path, "something")
self.bob_clock.advance(4)
yield d
# yield uploaded_d
d.addCallback(lambda ign: task.deferLater(reactor, 5, lambda: None))
d.addCallback(_wait_for, Alice_to_write_file2)
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file2", 0))
@ -837,8 +878,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
if _debug: print "Bob rewrites file2\n"
self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
if _debug: print "---- bob's file is %r" % (self.file_path,)
fileutil.write(self.file_path, "roger roger. what vector?")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
return self.bob_fileops.write(self.file_path, "roger roger. what vector?")
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
@ -914,8 +954,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
def Alice_to_write_file3():
if _debug: print "Alice writes a file\n"
self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
fileutil.write(self.file_path, "something")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
return self.alice_fileops.write(self.file_path, "something")
d.addCallback(_wait_for, Alice_to_write_file3)
d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
@ -928,8 +967,7 @@ class MagicFolderAliceBobTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Rea
if _debug: print "Bob rewrites file3\n"
self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
if _debug: print "---- bob's file is %r" % (self.file_path,)
fileutil.write(self.file_path, "roger roger")
return self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
return self.bob_fileops.write(self.file_path, "roger roger")
d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
@ -989,6 +1027,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
def _wait_until_started(self, ign):
#print "_wait_until_started"
self.magicfolder = self.get_client().getServiceNamed('magic-folder')
self.fileops = FileOperationsHelper(self.magicfolder.uploader, self.inject_inotify)
self.up_clock = task.Clock()
self.down_clock = task.Clock()
self.magicfolder.uploader._clock = self.up_clock
@ -1087,8 +1126,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
@defer.inlineCallbacks
def _check_move_empty_tree(res):
self.mkdir_nonascii(empty_tree_dir)
os.rename(empty_tree_dir, new_empty_tree_dir)
yield self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
yield self.fileops.move(empty_tree_dir, new_empty_tree_dir)
yield iterate(self.magicfolder)
d.addCallback(_check_move_empty_tree)
@ -1103,8 +1141,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
self.mkdir_nonascii(small_tree_dir)
what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
fileutil.write(what_path, "say when")
os.rename(small_tree_dir, new_small_tree_dir)
yield self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
yield self.fileops.move(small_tree_dir, new_small_tree_dir)
yield iterate(self.magicfolder)
# when we add the dir, we queue a scan of it; so we want
# the upload to "go" as well requiring 1 more iteration
@ -1120,8 +1157,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
@defer.inlineCallbacks
def _check_moved_tree_is_watched(res):
another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
fileutil.write(another_path, "file")
yield self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(another_path, "file")
yield iterate(self.magicfolder)
d.addCallback(_check_moved_tree_is_watched)
@ -1147,8 +1183,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
@defer.inlineCallbacks
def create_test_file(filename):
test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
fileutil.write(test_file, "meow %s" % filename)
yield self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(test_file, "meow %s" % filename)
yield iterate(self.magicfolder)
d.addCallback(lambda ign: create_test_file(u"what1"))
@ -1175,16 +1210,14 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
def test_delete(self):
# setup: create a file 'foo'
path = os.path.join(self.local_dir, u'foo')
fileutil.write(path, 'foo\n')
yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(path, 'foo\n')
yield iterate_uploader(self.magicfolder)
self.assertTrue(os.path.exists(path))
node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
# the test: delete the file (and do fake notifies)
os.unlink(path)
yield self.notify(to_filepath(path), self.inotify.IN_DELETE)
yield self.fileops.delete(path)
yield iterate_uploader(self.magicfolder)
self.assertFalse(os.path.exists(path))
@ -1199,14 +1232,12 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
def test_delete_and_restore(self):
# setup: create a file
path = os.path.join(self.local_dir, u'foo')
fileutil.write(path, 'foo\n')
yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(path, 'foo\n')
yield iterate_uploader(self.magicfolder)
self.assertTrue(os.path.exists(path))
# ...and delete the file
os.unlink(path)
yield self.notify(to_filepath(path), self.inotify.IN_DELETE)
yield self.fileops.delete(path)
yield iterate_uploader(self.magicfolder)
self.assertFalse(os.path.exists(path))
@ -1217,8 +1248,7 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
# restore the file, with different contents
path = os.path.join(self.local_dir, u'foo')
fileutil.write(path, 'bar\n')
yield self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
yield self.fileops.write(path, 'bar\n')
yield iterate_uploader(self.magicfolder)
# ensure we still have a DB entry, and that the version is 2
@ -1251,23 +1281,18 @@ class SingleMagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, Reall
class MockTestAliceBob(MagicFolderAliceBobTestMixin, unittest.TestCase):
inject_inotify = True
def setUp(self):
d = super(MockTestAliceBob, self).setUp()
self.inotify = fake_inotify
self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
return d
def notify(self, path, mask, magic=None, flush=True):
if magic is None:
magic = self.magicfolder
magic.uploader._notifier.event(path, mask)
# no flush for the mock test.
return task.deferLater(reactor, 0.1, lambda: None)
class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
"""This can run on any platform, and even if twisted.internet.inotify can't be imported."""
inject_inotify = True
def setUp(self):
d = super(MockTest, self).setUp()
@ -1275,13 +1300,6 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
return d
def notify(self, path, mask, magic=None, flush=True):
if magic is None:
magic = self.magicfolder
magic.uploader._notifier.event(path, mask)
# no flush for the mock test.
return task.deferLater(reactor, 0.1, lambda: None)
def test_errors(self):
self.set_up_grid(oneshare=True)
@ -1425,38 +1443,23 @@ class MockTest(SingleMagicFolderTestMixin, unittest.TestCase):
class RealTest(SingleMagicFolderTestMixin, unittest.TestCase):
"""This is skipped unless both Twisted and the platform support inotify."""
inject_inotify = False
def setUp(self):
d = super(RealTest, self).setUp()
self.inotify = magic_folder.get_inotify_module()
return d
def notify(self, path, mask, magic=None, flush=True):
# Writing to the filesystem causes the notification.
# Actually, there's no way to know when the actual
# notification will occur, and anyway we're not waiting for
# them in any case...so we'll just fudge it and hope 100ms is enough.
delay = 0.1 if sys.platform == "win32" else 0.1
return task.deferLater(reactor, delay, lambda: None)
class RealTestAliceBob(MagicFolderAliceBobTestMixin, unittest.TestCase):
"""This is skipped unless both Twisted and the platform support inotify."""
inject_inotify = False
def setUp(self):
d = super(RealTestAliceBob, self).setUp()
self.inotify = magic_folder.get_inotify_module()
return d
# XXX flush doesn't do anything (anymore?)
def notify(self, path, mask, magic=None, flush=True):
# Writing to the filesystem causes the notification.
# Actually, there's no way to know when the actual
# notification will occur, and anyway we're not waiting for
# them in any case...so we'll just fudge it and hope 100ms is enough.
delay = 0.1 if sys.platform == "win32" else 0.1
return task.deferLater(reactor, delay, lambda: None)
try:
magic_folder.get_inotify_module()