From 2ff2a2bb4fdb59155077022e799acb683033530b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 6 Aug 2026 09:27:08 -0700 Subject: [PATCH 1/3] Find the cursor correctly for error reporting --- src/toil/fileStores/cachingFileStore.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/toil/fileStores/cachingFileStore.py b/src/toil/fileStores/cachingFileStore.py index e4b55cf534..5f774b8f9b 100644 --- a/src/toil/fileStores/cachingFileStore.py +++ b/src/toil/fileStores/cachingFileStore.py @@ -1808,28 +1808,28 @@ def _createLinkFromCache(self, cachedPath, localFilePath, symlink=True): (cachedPath,) ) file_id = None - row = cur.fetchone() + row = self.cur.fetchone() while row is not None: logger.critical("File row: %s", row) file_id = row[0] - row = cur.fetchone() + row = self.cur.fetchone() self._read( "SELECT * FROM refs WHERE path = ?", (localFilePath,) ) - row = cur.fetchone() + row = self.cur.fetchone() while row is not None: logger.critical("Our ref row: %s", row) - row = cur.fetchone() + row = self.cur.fetchone() if file_id is not None: self._read( "SELECT * FROM refs WHERE file_id = ?", (file_id,) ) - row = cur.fetchone() + row = self.cur.fetchone() while row is not None: logger.critical("Cache ref row: %s", row) - row = cur.fetchone() + row = self.cur.fetchone() else: logger.critical("Cached file not found in database either") From c6579958041ea562b6cb808159c9ac2c7fc4d3f3 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 6 Aug 2026 18:44:49 -0400 Subject: [PATCH 2/3] Stop deleting every downloaded or uploaded file when a job finishes --- src/toil/fileStores/cachingFileStore.py | 28 ++++++++++++------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/toil/fileStores/cachingFileStore.py b/src/toil/fileStores/cachingFileStore.py index 5f774b8f9b..f3c628a623 100644 --- a/src/toil/fileStores/cachingFileStore.py +++ b/src/toil/fileStores/cachingFileStore.py @@ -997,8 +997,11 @@ def _removeJob(cls, con, cur, jobID): Get rid of the job with the given ID. The job must be owned by us. - Deletes the job's database entry, all its references, and its whole - temporary directory. + Deletes the job's database entry and its whole temporary directory, and + forgets all its refs. + + Any files the job downloaded outside its temporary directory are no + longer our problem. :param sqlite3.Connection con: Connection to the cache database. :param sqlite3.Cursor cur: Cursor in the cache database. @@ -1011,24 +1014,18 @@ def _removeJob(cls, con, cur, jobID): ): jobTemp = row[0] - for row in cls._static_read( - cur, "SELECT path FROM refs WHERE job_id = ?", (jobID,) - ): - try: - # Delete all the reference files. - os.unlink(row[0]) - except OSError: - # May not exist - pass - # And their database entries - cls._static_write(con, cur, [("DELETE FROM refs WHERE job_id = ?", (jobID,))]) - try: # Delete the job's temp directory to the extent that we can. shutil.rmtree(jobTemp) except OSError: pass + # Forget all the refs + cls._static_write(con, cur, [("DELETE FROM refs WHERE job_id = ?", (jobID,))]) + # We definitely don't want to delete the referenced files, because jobs + # are allowed to read/write directly between the job store and other + # shared storage (like the WDL call cache). + # Strike the job from the database cls._static_write(con, cur, [("DELETE FROM jobs WHERE id = ?", (jobID,))]) @@ -1759,6 +1756,7 @@ def _giveAwayDownloadingFile(self, fileStoreID, cachedPath, localFilePath): # Don't fake a delay here; this should be a rename always. # We are giving it away + logger.debug("Giving away %s as %s", cachedPath, localFilePath) shutil.move(cachedPath, localFilePath) # Record that. self._write( @@ -2056,7 +2054,7 @@ def _with_copying_reference_to_upload( ) -> Generator: """ Get a context manager that gives you either the local file path for a - copyuing reference to the given file, or None if that file is not in an + copying reference to the given file, or None if that file is not in an 'uploadable' or 'uploading' state. It is the caller's responsibility to actually do the copy, if they From c68af6e117f4436304f5361005e092fb426165d5 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 6 Aug 2026 18:54:21 -0400 Subject: [PATCH 3/3] Report if missing files in the cache are actually broken symlinks --- src/toil/fileStores/cachingFileStore.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/toil/fileStores/cachingFileStore.py b/src/toil/fileStores/cachingFileStore.py index f3c628a623..b0ded250fb 100644 --- a/src/toil/fileStores/cachingFileStore.py +++ b/src/toil/fileStores/cachingFileStore.py @@ -1800,6 +1800,12 @@ def _createLinkFromCache(self, cachedPath, localFilePath, symlink=True): "Cannot create link to missing cache file %s", cachedPath ) + try: + stats = os.lstat(cachedPath) + logger.critical("Is a broken symlink with stats: %s", stats) + except OSError: + pass + # Dump relevant bits of the database self._read( "SELECT * FROM files WHERE path = ?",