Clean up import_processing (#928)

annotation/annotation_run_files.py

@@ -14,6 +14,11 @@
 from library.utils.file_utils import name_from_filename
 from snpdb.variants_to_vcf import VARIANT_GRID_INFO_DICT, write_contig_sorted_values_to_vcf_file
 
+# Prefix of a run's scratch dir under settings.IMPORT_PROCESSING_DIR (where the bulk inserter writes the
+# CSVs it SQL COPYs from). Here rather than on the inserter so AnnotationRun can name the dir it owns
+# without importing the annotation pipeline.
+ANNOTATION_RUN_IMPORT_PROCESSING_PREFIX = "annotation_run"
+
 
 def get_annotated_filename(annotation_run, vcf_dump_filename) -> str:
     """ Path VEP writes its annotated VCF to for a given dump. Derived from the dump stem, which #1658

annotation/fake_annotation.py

@@ -52,8 +52,7 @@
 
 
 def get_fake_annotation_settings_dict(columns_version: int) -> dict:
-    TEST_IMPORT_PROCESSING_DIR = os.path.join(settings.PRIVATE_DATA_ROOT, 'import_processing',
-                                              "test", str(uuid4()))
+    TEST_IMPORT_PROCESSING_DIR = os.path.join(settings.IMPORT_PROCESSING_DIR, "test", str(uuid4()))
 
     TEST_ANNOTATION = copy.deepcopy(settings.ANNOTATION)
     # phastCons/phyloP custom tracks: v1-v3 fixtures were generated without the bigwig data, so disable

annotation/management/commands/gene_annotation.py

@@ -20,7 +20,10 @@
 from annotation.models.models import SubVersionPartition
 from genes.gene_matching import ReleaseGeneMatcher
 from genes.models import Gene, GeneAnnotationRelease, GnomADGeneConstraint, ReleaseGeneSymbolGene
-from library.django_utils.django_file_utils import get_import_processing_filename
+from library.django_utils.django_file_utils import (
+    get_import_processing_filename,
+    remove_import_processing_dir,
+)
 from ontology.models import (
     ONTOLOGY_RELATIONSHIP_MEDIUM_QUALITY_FILTER,
     GeneDiseaseClassification,
@@ -546,6 +549,8 @@ def _write_records(self, gene_annotation_version: GeneAnnotationVersion, gene_an
         self.stdout.write(f"Inserting file '{csv_filename}' into partition {partition_table}\n")
         sql_copy_csv(csv_filename, partition_table, self.GENE_ANNOTATION_HEADER, delimiter=delimiter)
         self.stdout.write("Done!\n")
+        if settings.IMPORT_PROCESSING_DELETE_TEMP_FILES_ON_SUCCESS:
+            remove_import_processing_dir(gene_annotation_version.pk, prefix='gene_annotation')
 
 
 def bad_gene_annotation():

annotation/management/commands/human_protein_atlas_import.py

@@ -6,12 +6,16 @@
 import os
 
 import pandas as pd
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
 from annotation.models import HumanProteinAtlasAnnotationVersion, HumanProteinAtlasTissueSample
 from genes.models import Gene, GeneSymbol
 from genes.models_enums import AnnotationConsortium
-from library.django_utils.django_file_utils import get_import_processing_filename
+from library.django_utils.django_file_utils import (
+    get_import_processing_filename,
+    remove_import_processing_dir,
+)
 from library.utils import file_sha256sum
 from upload.vcf.sql_copy_files import sql_copy_csv, write_sql_copy_csv
 
@@ -116,3 +120,5 @@ def handle(self, *args, **options):
                       'value']
         sql_copy_csv(csv_filename, partition_table, HPA_HEADER, delimiter=delimiter)
         logging.info("Done!")
+        if settings.IMPORT_PROCESSING_DELETE_TEMP_FILES_ON_SUCCESS:
+            remove_import_processing_dir(version_id, prefix='human_protein_atlas')

annotation/models/models.py

@@ -11,7 +11,6 @@
 import logging
 import os
 import re
-import shutil
 from collections import defaultdict
 from collections.abc import Callable, Iterable
 from contextlib import contextmanager
@@ -37,6 +36,7 @@
 from psqlextra.models import PostgresPartitionedModel
 from psqlextra.types import PostgresPartitioningMethod
 
+from annotation.annotation_run_files import ANNOTATION_RUN_IMPORT_PROCESSING_PREFIX
 from annotation.external_search_terms import (
     get_variant_pubmed_search_terms,
     get_variant_search_terms,
@@ -89,6 +89,7 @@
 )
 from genes.models_enums import AnnotationConsortium
 from library.django_utils import object_is_referenced
+from library.django_utils.django_file_utils import remove_import_processing_dir
 from library.django_utils.data_archive_mixin import DataArchiveMixin
 from library.django_utils.django_partition import RelatedModelsPartitionModel
 from library.genomics import parse_gnomad_coord
@@ -1420,10 +1421,8 @@ def reset_for_retry(self):
         # the upload_attempts>1 cleanup in import_vcf_annotations is skipped - so the leftover files trip
         # write_sql_copy_csv's "don't want to overwrite" guard, which is meant only for genuinely out-of-sync
         # dirs (moved dump / double launch). Done outside the transaction (filesystem op) and after the DB
-        # reset commits, so a rolled-back reset leaves the scratch dir intact. Prefix matches
-        # BulkVEPVCFAnnotationInserter.PREFIX.
-        import_processing_dir = os.path.join(settings.IMPORT_PROCESSING_DIR, f"annotation_run_{self.pk}")
-        shutil.rmtree(import_processing_dir, ignore_errors=True)
+        # reset commits, so a rolled-back reset leaves the scratch dir intact.
+        remove_import_processing_dir(self.pk, prefix=ANNOTATION_RUN_IMPORT_PROCESSING_PREFIX)
 
     def revert_external_to_local(self):
         """ #1568: return an external run to the normal local pipeline. Clears the external flag and dump

annotation/vcf_files/bulk_vep_vcf_annotation_inserter.py

@@ -9,7 +9,6 @@
 import logging
 import operator
 import os
-import shutil
 import time
 from collections import Counter, defaultdict
 from collections.abc import Iterable
@@ -21,6 +20,7 @@
 from django.conf import settings
 
 from annotation import vep_columns as vep_columns_registry
+from annotation.annotation_run_files import ANNOTATION_RUN_IMPORT_PROCESSING_PREFIX
 from annotation.models.models import (
     AnnotationRun,
     VariantAnnotation,
@@ -45,8 +45,8 @@
 from genes.models_enums import AnnotationConsortium
 from library.django_utils import get_model_fields
 from library.django_utils.django_file_utils import (
-    get_import_processing_dir,
     get_import_processing_filename,
+    remove_import_processing_dir,
 )
 from library.genomics import Range, overlap_fraction, parse_gnomad_coord
 from library.log_utils import log_traceback
@@ -172,7 +172,7 @@ class BulkVEPVCFAnnotationInserter:
 
         VEP Fields are where they are copied are defined in ColumnVEPField """
 
-    PREFIX = "annotation_run"
+    PREFIX = ANNOTATION_RUN_IMPORT_PROCESSING_PREFIX
     DB_FIXED_COLUMNS = [
         "version_id",
         "annotation_run_id",
@@ -958,10 +958,7 @@ def _gene_overlap_to_row_data(_header, annotations_list: list):
         pass
 
     def remove_processing_files(self):
-        import_processing_dir = get_import_processing_dir(self.annotation_run.pk, prefix=self.PREFIX)
-        logging.info("********* Deleting '%s' *******", import_processing_dir)
-        # ignore_errors so a missing dir (eg cleaned-up retry) doesn't blow up - we just want it gone
-        shutil.rmtree(import_processing_dir, ignore_errors=True)
+        remove_import_processing_dir(self.annotation_run.pk, prefix=self.PREFIX)
 
     @cached_property
     def gene_identifiers(self):

claude/maps/commands.md

@@ -53,6 +53,7 @@ Generated by `vg map commands` (do not edit; run `scripts/vg map commands` after
 | `import_dbnsfp_gene_annotation` | annotation |  | annotation, genes |
 | `import_gene_annotation` | genes |  | genes, snpdb |
 | `import_lab_info` | snpdb |  | snpdb |
+| `import_processing_cleanup` | upload | Remove import_processing scratch directories whose owner has finished with them (guessed) | annotation, snpdb, upload |
 | `import_sequencing_info` | seqauto |  | seqauto, snpdb |
 | `import_transcript_sequence_fasta` | genes |  |  |
 | `import_vcf` | upload |  | snpdb |

claude/maps/signals.md

@@ -56,6 +56,7 @@ Receivers of Django / third-party signals (post_save, pre_delete, m2m_changed, u
 | post_delete | VariantTag | `snpdb.management.commands.variant_tags:variant_tag_delete` |
 | post_delete | CohortGenotypeCollection | `snpdb.models.models_cohort:cohort_genotype_collection_post_delete_handler` |
 | post_delete | SubCohortVariantCollection | `snpdb.models.models_cohort:post_delete_sub_cohort_variant_collection` |
+| post_delete | UploadPipeline | `upload.models.models:upload_pipeline_post_delete_handler` |
 | post_delete | UploadedPatientRecords | `upload.models.models_uploaded_files:uploaded_patient_records_post_delete_handler` |
 | post_save | ActiveSampleGeneList | `analysis.apps:handle_active_sample_gene_list_created` |
 | post_save | VariantTag | `analysis.apps:variant_tag_create` |