WIP

src/allmydata/backupdb.py

@@ -68,7 +68,25 @@ UPDATERS = {
 }
 
 
-SCHEMA_v3 = MAIN_SCHEMA % (3, ",\nversion INTEGER\n") + TABLE_DIRECTORY
+# magic-folder db schema version 3
+MAGIC_FOLDER_SCHEMA_v3 = """
+CREATE TABLE version
+(
+ version INTEGER  -- contains one row, set to %s
+);
+
+CREATE TABLE local_files
+(
+ path  VARCHAR(1024) PRIMARY KEY, -- index, this is an absolute UTF-8-encoded local filename
+ -- note that size is before mtime and ctime here, but after in function parameters
+ size  INTEGER,       -- os.stat(fn)[stat.ST_SIZE]   (NULL if the file has been deleted)
+ mtime NUMBER,        -- os.stat(fn)[stat.ST_MTIME]
+ ctime NUMBER,        -- os.stat(fn)[stat.ST_CTIME]
+ version INTEGER,
+ last_downloaded_uri VARCHAR(256) UNIQUE,       -- URI:CHK:... 
+ last_downloaded_timestamp NUMBER
+);
+""" % (3,)
 
 
 def get_backupdb(dbfile, stderr=sys.stderr,
@@ -368,9 +386,29 @@ class BackupDB:
         self.connection.commit()
 
 
-class MagicFolderDB(BackupDB):
+class MagicFolderDB():
     VERSION = 3
 
+    def __init__(self, sqlite_module, connection):
+        self.sqlite_module = sqlite_module
+        self.connection = connection
+        self.cursor = connection.cursor()
+
+    def check_file_db_exists(self, path):
+        """I will tell you if a given file has an entry in my database or not
+        by returning True or False.
+        """
+        c = self.cursor
+        c.execute("SELECT size,mtime,ctime"
+                  " FROM local_files"
+                  " WHERE path=?",
+                  (path,))
+        row = self.cursor.fetchone()
+        if not row:
+            return False
+        else:
+            return True
+
     def get_all_relpaths(self):
         """
         Retrieve a set of all relpaths of files that have had an entry in magic folder db
@@ -380,13 +418,29 @@ class MagicFolderDB(BackupDB):
         rows = self.cursor.fetchall()
         return set([r[0] for r in rows])
 
+    def get_last_downloaded_uri(self, relpath_u):
+        """
+        Return the last downloaded uri recorded in the magic folder db.
+        If none are found then return None.
+        """
+        c = self.cursor
+        c.execute("SELECT last_downloaded_uri"
+                  " FROM local_files"
+                  " WHERE path=?",
+                  (relpath_u,))
+        row = self.cursor.fetchone()
+        if not row:
+            return None
+        else:
+            return row[0]
+
     def get_local_file_version(self, relpath_u):
         """
         Return the version of a local file tracked by our magic folder db.
         If no db entry is found then return None.
         """
         c = self.cursor
-        c.execute("SELECT version, fileid"
+        c.execute("SELECT version"
                   " FROM local_files"
                   " WHERE path=?",
                   (relpath_u,))
@@ -397,26 +451,23 @@ class MagicFolderDB(BackupDB):
             return row[0]
 
     def did_upload_version(self, filecap, relpath_u, version, pathinfo):
-        #print "did_upload_version(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo)
+        print "did_upload_version(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo)
         now = time.time()
-        fileid = self.get_or_allocate_fileid_for_cap(filecap)
-        try:
-            self.cursor.execute("INSERT INTO last_upload VALUES (?,?,?)",
-                                (fileid, now, now))
-        except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
-            self.cursor.execute("UPDATE last_upload"
-                                " SET last_uploaded=?, last_checked=?"
-                                " WHERE fileid=?",
-                                (now, now, fileid))
         try:
+            print "insert"
             self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?)",
-                                (relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version))
+                                (relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, filecap, pathinfo.mtime))
         except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
+            print "err... update"
+            try:
                 self.cursor.execute("UPDATE local_files"
-                                " SET size=?, mtime=?, ctime=?, fileid=?, version=?"
+                                    " SET size=?, mtime=?, ctime=?, version=?, last_downloaded_uri=?, last_downloaded_timestamp=?"
                                     " WHERE path=?",
-                                (pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version, relpath_u))
+                                    (pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, filecap, pathinfo.mtime, relpath_u))
+            except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError):
+                print "WTF BBQ OMG"
         self.connection.commit()
+        print "commited"
 
     def is_new_file(self, pathinfo, relpath_u):
         """

src/allmydata/frontends/magic_folder.py

@@ -51,7 +51,7 @@ class MagicFolder(service.MultiService):
 
         service.MultiService.__init__(self)
 
-        db = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3))
+        db = backupdb.get_backupdb(dbfile, create_version=(backupdb.MAGIC_FOLDER_SCHEMA_v3, 3))
         if db is None:
             return Failure(Exception('ERROR: Unable to load magic folder db.'))
 
@@ -294,8 +294,10 @@ class Uploader(QueueMixin):
                 d2 = defer.succeed(None)
                 if self._db.check_file_db_exists(relpath_u):
                     d2.addCallback(lambda ign: self._get_metadata(encoded_path_u))
+                    last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u)
                     current_version = self._db.get_local_file_version(relpath_u) + 1
                     def set_deleted(metadata):
+                        metadata['last_downloaded_uri'] = last_downloaded_uri
                         metadata['version'] = current_version
                         metadata['deleted'] = True
                         empty_uploadable = Data("", self._client.convergence)
@@ -305,6 +307,7 @@ class Uploader(QueueMixin):
                         filecap = filenode.get_uri()
                         self._db.did_upload_version(filecap, relpath_u, current_version, pathinfo)
                         self._count('files_uploaded')
+
                     # FIXME consider whether it's correct to retrieve the filenode again.
                     d2.addCallback(lambda x: self._get_filenode(encoded_path_u))
                     d2.addCallback(add_db_entry)
@@ -330,6 +333,7 @@ class Uploader(QueueMixin):
                 return upload_d
             elif pathinfo.isfile:
                 version = self._db.get_local_file_version(relpath_u)
+                last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u)
                 if version is None:
                     version = 0
                 elif self._db.is_new_file(pathinfo, relpath_u):
@@ -338,11 +342,13 @@ class Uploader(QueueMixin):
                     return None
 
                 uploadable = FileName(unicode_from_filepath(fp), self._client.convergence)
-                d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, metadata={"version":version}, overwrite=True)
+                metadata = { "version":version }
+                if last_downloaded_uri is not None:
+                    metadata["last_downloaded_uri"] = last_downloaded_uri
+                d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, metadata=metadata, overwrite=True)
                 def add_db_entry(filenode):
                     filecap = filenode.get_uri()
                     self._db.did_upload_version(filecap, relpath_u, version, pathinfo)
-                    self._count('files_uploaded')
                 d2.addCallback(add_db_entry)
                 return d2
             else:

src/allmydata/test/test_magic_folder.py

@@ -39,7 +39,7 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual
 
     def _createdb(self):
         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
-        bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3))
+        bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.MAGIC_FOLDER_SCHEMA_v3, 3))
         self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,))
         self.failUnlessEqual(bdb.VERSION, 3)
         return bdb