Fix N+1 queries in grids, views and tasks #1723

annotation/transcripts_annotation_selections.py

@@ -275,7 +275,8 @@ def _add_other_annotation_consortium_transcripts(self, variant: Variant):
         # Convert once to explicit, then pass this around
         variant_coordinate = variant.coordinate.as_external_explicit(self.genome_build)
         has_other_annotation_consortium_transcripts = False
-        for transcript_version in TranscriptVersion.objects.filter(**kwargs).order_by("-version"):
+        transcript_version_qs = TranscriptVersion.objects.filter(**kwargs).select_related("gene_version")
+        for transcript_version in transcript_version_qs.order_by("-version"):
             # Don't duplicate ones already available via RefSeq/Ensembl equivalence
             # and only take the highest version we have
             if transcript_version.transcript_id not in existing_other_transcripts:

classification/models/classification_grouping.py

@@ -93,7 +93,8 @@ def get_absolute_url(self) -> str:
     @cached_property
     def allele_origin_dict(self) -> dict[AlleleOriginBucket, 'AlleleOriginGrouping']:
         by_bucket = {}
-        for ao in AlleleOriginGrouping.objects.filter(allele_grouping=self).prefetch_related("classificationgrouping_set"):
+        for ao in AlleleOriginGrouping.objects.filter(allele_grouping=self) \
+                .prefetch_related("classificationgrouping_set__lab__organization"):
             by_bucket[ao.allele_origin_bucket] = ao
         return by_bucket
 
@@ -477,7 +478,8 @@ def gene_symbols(self):
     @staticmethod
     def update_all_dirty():
         # maybe move this out since it does AlleleOriginGroupings too
-        for dirty in ClassificationGrouping.objects.filter(dirty=True).iterator():
+        # update() dirties the allele_origin_grouping straight away, so bring it along
+        for dirty in ClassificationGrouping.objects.filter(dirty=True).select_related("allele_origin_grouping").iterator():
             dirty.update()
         for dirty in AlleleOriginGrouping.objects.filter(dirty=True).iterator():
             dirty.update()

classification/models/clinvar_export_prepare.py

@@ -3,6 +3,7 @@
 from collections.abc import Iterable
 from typing import Optional
 
+from django.db.models import Count, Q
 from django.utils import timezone
 from django.utils.timezone import now
 
@@ -176,9 +177,11 @@ def process_allele(
                 no_condition_count += 1
         clinvar_merger.consolidate()
 
-        total = clinvar_allele.clinvarexport_set.count()
-        in_error = clinvar_allele.clinvarexport_set.filter(status=ClinVarExportStatus.IN_ERROR).count()
-        valid = total - in_error
+        export_counts = clinvar_allele.clinvarexport_set.aggregate(
+            total=Count("pk"),
+            in_error=Count("pk", filter=Q(status=ClinVarExportStatus.IN_ERROR)))
+        in_error = export_counts["in_error"]
+        valid = export_counts["total"] - in_error
 
         clinvar_allele.classifications_missing_condition = no_condition_count
         clinvar_allele.submissions_valid = valid

classification/models/discordance_models.py

@@ -177,23 +177,31 @@ def update(self, clinical_context_change_data: ClinicalContextChangeData, notify
 
         existing_vms = set()
         existing_labs = set()
-        for drc in DiscordanceReportClassification.objects.filter(report=self):
+        drc_qs = DiscordanceReportClassification.objects.filter(report=self) \
+            .select_related("classification_original__classification__lab")
+        for drc in drc_qs:
             existing_vms.add(drc.classification_original.classification_id)
             existing_labs.add(drc.classification_original.classification.lab)
 
         newly_added_labs: set[Lab] = set()
+        added_classification_ids = []
         for vcm_id in self.clinical_context.classifications_qs.values_list('id', flat=True):
             if vcm_id in existing_vms:
                 existing_vms.remove(vcm_id)
             else:
-                vcm = ClassificationModification.objects.get(is_last_published=True, classification=vcm_id)
-                if vcm.classification.lab not in existing_labs:
-                    newly_added_labs.add(vcm.classification.lab)
-
-                DiscordanceReportClassification(
-                    report=self,
-                    classification_original=vcm
-                ).save()
+                added_classification_ids.append(vcm_id)
+
+        added_qs = ClassificationModification.objects.filter(is_last_published=True,
+                                                             classification__in=added_classification_ids) \
+            .select_related("classification__lab")
+        for vcm in added_qs:
+            if vcm.classification.lab not in existing_labs:
+                newly_added_labs.add(vcm.classification.lab)
+
+            DiscordanceReportClassification(
+                report=self,
+                classification_original=vcm
+            ).save()
 
         invalidate_cached_property(self, 'discordance_report_classifications')
         invalidate_cached_property(self, 'involved_labs')

classification/views/allele_grouping_datatables.py

@@ -71,7 +71,9 @@ def render_labs(self, row: CellData) -> set[Lab]:
 
     def c_hgvs_for(self, cg: ClassificationGrouping) -> HGVSDisplay:
         is_preferred_genome_build = True
-        allele_info = cg.latest_classification_modification.classification.allele_info
+        # update() keeps this in step with latest_classification_modification.classification.allele_info,
+        # and reading it here avoids walking that chain as three lazy FK loads per row
+        allele_info = cg.latest_allele_info
         for gb in self.genome_build_prefs:
             if ri := allele_info[gb]:
                 if c_hgvs := ri.c_hgvs_obj:
@@ -84,7 +86,9 @@ def c_hgvs_for(self, cg: ClassificationGrouping) -> HGVSDisplay:
     def render_allele(self, row: CellData) -> JsonDataType:
         allele_group = _allele_group(row.get("allele"))
         # FIXME cache this
-        cgs = ClassificationGrouping.objects.filter(allele_origin_grouping__allele_grouping=allele_group.pk, dirty=False)
+        cgs = ClassificationGrouping.objects.filter(allele_origin_grouping__allele_grouping=allele_group.pk,
+                                                    dirty=False) \
+            .select_related("latest_allele_info__grch37__genome_build", "latest_allele_info__grch38__genome_build")
         all_chgvs = sorted({self.c_hgvs_for(cg) for cg in cgs})
         c_hgvs_json: JsonDataType
         if all_chgvs:

snpdb/variant_sample_information.py

@@ -391,7 +391,7 @@ def _get_classifications_by_sample_id(self, sample_ids: set[int]) -> dict[int, l
         classifications_by_sample_id = defaultdict(list)
         qs = ClassificationModification.latest_for_user(self.user, allele=allele, published=True,
                                                         classification__sample__in=sample_ids)
-        for cm in qs:
+        for cm in qs.select_related("classification__lab"):
             classification = cm.classification
             pills = clinical_significance_pills(classification.summary_typed, classification.allele_origin_bucket)
             classification_json = {
@@ -441,7 +441,10 @@ def _row_to_json(row: dict) -> dict:
     def _get_locus_counts(self) -> list[dict]:
         """ Zygosity counts for every variant at this locus, this variant first """
         counts_by_variant_id = self._get_locus_zygosity_counts()
-        variant_by_id = {v.pk: v for v in Variant.objects.filter(pk__in=counts_by_variant_id)}
+        # str(v) and v.alt.seq below reach through to the locus/sequence rows
+        variant_qs = Variant.objects.filter(pk__in=counts_by_variant_id) \
+            .select_related("locus__contig", "locus__ref", "alt")
+        variant_by_id = {v.pk: v for v in variant_qs}
 
         sorted_rows = []
         for variant_id, zygosity_counts in counts_by_variant_id.items():