tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

WIP

This commit is contained in:
David Stainton 2015-10-07 18:45:44 +02:00 committed by Daira Hopwood
parent 2057c77ed2
commit 31ad6e86df
3 changed files with 79 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/allmydata/backupdb.py
View File

@ -68,7 +68,25 @@ UPDATERS = {
}
SCHEMA_v3 = MAIN_SCHEMA % (3, ",\nversion INTEGER\n") + TABLE_DIRECTORY
# magic-folder db schema version 3
MAGIC_FOLDER_SCHEMA_v3 = """
CREATE TABLE version
(
version INTEGER -- contains one row, set to %s
);
CREATE TABLE local_files
(
path VARCHAR(1024) PRIMARY KEY, -- index, this is an absolute UTF-8-encoded local filename
-- note that size is before mtime and ctime here, but after in function parameters
size INTEGER, -- os.stat(fn)[stat.ST_SIZE] (NULL if the file has been deleted)
mtime NUMBER, -- os.stat(fn)[stat.ST_MTIME]
ctime NUMBER, -- os.stat(fn)[stat.ST_CTIME]
version INTEGER,
last_downloaded_uri VARCHAR(256) UNIQUE, -- URI:CHK:...
last_downloaded_timestamp NUMBER
);
""" % (3,)
def get_backupdb(dbfile, stderr=sys.stderr,
@ -368,9 +386,29 @@ class BackupDB:
self.connection.commit()
class MagicFolderDB(BackupDB):
class MagicFolderDB():
VERSION = 3
def __init__(self, sqlite_module, connection):
self.sqlite_module = sqlite_module
self.connection = connection
self.cursor = connection.cursor()
def check_file_db_exists(self, path):
"""I will tell you if a given file has an entry in my database or not
by returning True or False.
"""
c = self.cursor
c.execute("SELECT size,mtime,ctime"
" FROM local_files"
" WHERE path=?",
(path,))
row = self.cursor.fetchone()
if not row:
return False
else:
return True
def get_all_relpaths(self):
"""
Retrieve a set of all relpaths of files that have had an entry in magic folder db
@ -380,13 +418,29 @@ class MagicFolderDB(BackupDB):
rows = self.cursor.fetchall()
return set([r[0] for r in rows])
def get_last_downloaded_uri(self, relpath_u):
"""
Return the last downloaded uri recorded in the magic folder db.
If none are found then return None.
"""
c = self.cursor
c.execute("SELECT last_downloaded_uri"
" FROM local_files"
" WHERE path=?",
(relpath_u,))
row = self.cursor.fetchone()
if not row:
return None
else:
return row[0]
def get_local_file_version(self, relpath_u):
"""
Return the version of a local file tracked by our magic folder db.
If no db entry is found then return None.
"""
c = self.cursor
c.execute("SELECT version, fileid"
c.execute("SELECT version"
" FROM local_files"
" WHERE path=?",
(relpath_u,))
@ -397,26 +451,23 @@ class MagicFolderDB(BackupDB):
return row[0]
def did_upload_version(self, filecap, relpath_u, version, pathinfo):
#print "did_upload_version(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo)
print "did_upload_version(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo)
now = time.time()
fileid = self.get_or_allocate_fileid_for_cap(filecap)
try:
self.cursor.execute("INSERT INTO last_upload VALUES (?,?,?)",
(fileid, now, now))
except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
self.cursor.execute("UPDATE last_upload"
" SET last_uploaded=?, last_checked=?"
" WHERE fileid=?",
(now, now, fileid))
try:
print "insert"
self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?)",
(relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version))
(relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, filecap, pathinfo.mtime))
except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
self.cursor.execute("UPDATE local_files"
" SET size=?, mtime=?, ctime=?, fileid=?, version=?"
" WHERE path=?",
(pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version, relpath_u))
print "err... update"
try:
self.cursor.execute("UPDATE local_files"
" SET size=?, mtime=?, ctime=?, version=?, last_downloaded_uri=?, last_downloaded_timestamp=?"
" WHERE path=?",
(pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, filecap, pathinfo.mtime, relpath_u))
except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
print "WTF BBQ OMG"
self.connection.commit()
print "commited"
def is_new_file(self, pathinfo, relpath_u):
"""

12
src/allmydata/frontends/magic_folder.py
View File

@ -51,7 +51,7 @@ class MagicFolder(service.MultiService):
service.MultiService.__init__(self)
db = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3))
db = backupdb.get_backupdb(dbfile, create_version=(backupdb.MAGIC_FOLDER_SCHEMA_v3, 3))
if db is None:
return Failure(Exception('ERROR: Unable to load magic folder db.'))
@ -294,8 +294,10 @@ class Uploader(QueueMixin):
d2 = defer.succeed(None)
if self._db.check_file_db_exists(relpath_u):
d2.addCallback(lambda ign: self._get_metadata(encoded_path_u))
last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u)
current_version = self._db.get_local_file_version(relpath_u) + 1
def set_deleted(metadata):
metadata['last_downloaded_uri'] = last_downloaded_uri
metadata['version'] = current_version
metadata['deleted'] = True
empty_uploadable = Data("", self._client.convergence)
@ -305,6 +307,7 @@ class Uploader(QueueMixin):
filecap = filenode.get_uri()
self._db.did_upload_version(filecap, relpath_u, current_version, pathinfo)
self._count('files_uploaded')
# FIXME consider whether it's correct to retrieve the filenode again.
d2.addCallback(lambda x: self._get_filenode(encoded_path_u))
d2.addCallback(add_db_entry)
@ -330,6 +333,7 @@ class Uploader(QueueMixin):
return upload_d
elif pathinfo.isfile:
version = self._db.get_local_file_version(relpath_u)
last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u)
if version is None:
version = 0
elif self._db.is_new_file(pathinfo, relpath_u):
@ -338,11 +342,13 @@ class Uploader(QueueMixin):
return None
uploadable = FileName(unicode_from_filepath(fp), self._client.convergence)
d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, metadata={"version":version}, overwrite=True)
metadata = { "version":version }
if last_downloaded_uri is not None:
metadata["last_downloaded_uri"] = last_downloaded_uri
d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, metadata=metadata, overwrite=True)
def add_db_entry(filenode):
filecap = filenode.get_uri()
self._db.did_upload_version(filecap, relpath_u, version, pathinfo)
self._count('files_uploaded')
d2.addCallback(add_db_entry)
return d2
else:

2
src/allmydata/test/test_magic_folder.py
View File

@ -39,7 +39,7 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual
def _createdb(self):
dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3))
bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.MAGIC_FOLDER_SCHEMA_v3, 3))
self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,))
self.failUnlessEqual(bdb.VERSION, 3)
return bdb