Failed Liftover to {{ genome_build }}
+ {% with tool_counts=retry_counts|get_item:genome_build.name %}
+ {% if tool_counts %}
+
A retry re-attempts that tool for every allele it has failed on that still
+ has no {{ genome_build }} variant. Earlier attempts are kept, and shown below.
+
+ {% endif %}
+ {% endwith %}
{% endfor %}
diff --git a/snpdb/tests/test_liftover.py b/snpdb/tests/test_liftover.py
index 2edc930fb..4382dd971 100644
--- a/snpdb/tests/test_liftover.py
+++ b/snpdb/tests/test_liftover.py
@@ -13,11 +13,13 @@
_liftover_using_source_variant_coordinate,
_non_standard_contig_error,
_run_liftover_using_same_contig,
+ allele_can_attempt_liftover,
)
from snpdb.models import (
Allele,
AlleleConversionTool,
AlleleLiftover,
+ AlleleOrigin,
GenomeBuild,
LiftoverRun,
ProcessingStatus,
@@ -87,6 +89,44 @@ def _liftover_using_source_variant_coordinate(self):
self.assertEqual(conversion_tool, AlleleConversionTool.BCFTOOLS_LIFTOVER)
self.assertEqual(variant_coordinate_38, self.expected_vc_38)
+ def test_retry_conversion_tools_overrides_failed_set(self):
+ """ #1273 - a tool that has already failed on an allele is skipped forever unless explicitly retried """
+ grch37 = GenomeBuild.grch37()
+ grch38 = GenomeBuild.grch38()
+ variant_37 = slowly_create_test_variant("3", 128198980, 'A', 'T', grch37)
+ VariantAllele.objects.create(variant=variant_37, genome_build=grch37, allele=self.allele,
+ origin=AlleleOrigin.IMPORTED_TO_DATABASE,
+ allele_linking_tool=AlleleConversionTool.CLINGEN_ALLELE_REGISTRY)
+ for conversion_tool in [AlleleConversionTool.CLINGEN_ALLELE_REGISTRY,
+ AlleleConversionTool.BCFTOOLS_LIFTOVER]:
+ liftover_run = LiftoverRun.objects.create(user=admin_bot(), conversion_tool=conversion_tool,
+ genome_build=grch38)
+ AlleleLiftover.objects.create(allele=self.allele, liftover=liftover_run,
+ status=ProcessingStatus.ERROR)
+
+ _existing, needs_pipeline = _get_build_liftover_dicts([self.allele], grch37, [grch38])
+ self.assertEqual(dict(needs_pipeline), {}, "Every tool failed, so nothing is attempted")
+
+ retry_tools = {AlleleConversionTool.CLINGEN_ALLELE_REGISTRY}
+ _existing, needs_pipeline = _get_build_liftover_dicts([self.allele], grch37, [grch38],
+ retry_conversion_tools=retry_tools)
+ tools = needs_pipeline[grch38]
+ self.assertNotIn(AlleleConversionTool.BCFTOOLS_LIFTOVER, tools, "Tools not retried stay skipped")
+ _allele, variant_coordinate, _error = tools[AlleleConversionTool.CLINGEN_ALLELE_REGISTRY][0]
+ self.assertEqual(variant_coordinate, self.expected_vc_38)
+
+ def test_allele_can_attempt_liftover_with_retry(self):
+ grch38 = GenomeBuild.grch38()
+ for conversion_tool in AlleleConversionTool:
+ liftover_run = LiftoverRun.objects.create(user=admin_bot(), conversion_tool=conversion_tool,
+ genome_build=grch38)
+ AlleleLiftover.objects.create(allele=self.allele, liftover=liftover_run,
+ status=ProcessingStatus.ERROR)
+
+ self.assertFalse(allele_can_attempt_liftover(self.allele, grch38))
+ self.assertTrue(allele_can_attempt_liftover(self.allele, grch38,
+ retry_conversion_tools=list(AlleleConversionTool)))
+
def test_standard_contig_written_to_vcf(self):
self.assertIsNone(_non_standard_contig_error(GenomeBuild.grch37(), self.expected_vc_37))
@@ -128,6 +168,35 @@ def test_allele_id_batches(self):
self.assertEqual(list(_allele_id_batches(allele_qs)), expected)
+class TestFailedLiftoverAlleles(TestCase):
+ """ #1273 - which alleles a per-tool retry re-attempts """
+
+ @classmethod
+ def setUpTestData(cls):
+ grch37 = GenomeBuild.grch37()
+ grch38 = GenomeBuild.grch38()
+ cls.still_missing = create_mock_allele(slowly_create_test_variant("3", 1000, 'A', 'T', grch37), grch37)
+ cls.lifted_over = create_mock_allele(slowly_create_test_variant("3", 2000, 'A', 'T', grch37), grch37)
+ VariantAllele.objects.create(variant=slowly_create_test_variant("3", 2001, 'A', 'T', grch38),
+ genome_build=grch38, allele=cls.lifted_over,
+ origin=AlleleOrigin.LIFTOVER,
+ allele_linking_tool=AlleleConversionTool.BCFTOOLS_LIFTOVER)
+
+ liftover_run = LiftoverRun.objects.create(user=admin_bot(),
+ conversion_tool=AlleleConversionTool.BCFTOOLS_LIFTOVER,
+ genome_build=grch38)
+ for allele in [cls.still_missing, cls.lifted_over]:
+ AlleleLiftover.objects.create(allele=allele, liftover=liftover_run, status=ProcessingStatus.ERROR)
+
+ def test_failed_liftover_for_build(self):
+ grch38 = GenomeBuild.grch38()
+ qs = Allele.failed_liftover_for_build(grch38, AlleleConversionTool.BCFTOOLS_LIFTOVER)
+ self.assertEqual(list(qs), [self.still_missing], "An allele that has since been lifted over is left alone")
+
+ qs = Allele.failed_liftover_for_build(grch38, AlleleConversionTool.CLINGEN_ALLELE_REGISTRY)
+ self.assertEqual(list(qs), [], "Only the tool being retried counts")
+
+
class TestLiftoverQueries(TestCase):
@classmethod
def setUpTestData(cls):
diff --git a/snpdb/views/views_liftover.py b/snpdb/views/views_liftover.py
index 513399f64..cc0cd65fa 100644
--- a/snpdb/views/views_liftover.py
+++ b/snpdb/views/views_liftover.py
@@ -1,5 +1,7 @@
+from typing import Optional
from django.contrib import messages
+from django.db.models import Count
from django.shortcuts import render
from library.django_utils import (
@@ -23,19 +25,23 @@
def liftover_runs(request):
genome_builds = GenomeBuild.builds_with_annotation()
if request.method == 'POST':
- dest_genome_build = None
- for genome_build in genome_builds:
- if f"liftover_to_{genome_build.name}" in request.POST:
- dest_genome_build = genome_build
- break
- if dest_genome_build is None:
- raise ValueError("Could not determine dest genome build from liftover_runs POST")
- messages.add_message(request, messages.INFO, f"Lifting over alleles to {dest_genome_build}")
- liftover_alleles.si(request.user.username, dest_genome_build.name).apply_async()
+ dest_genome_build, retry_conversion_tool = _liftover_post_action(request.POST, genome_builds)
+ if retry_conversion_tool:
+ num_alleles = Allele.failed_liftover_for_build(dest_genome_build, retry_conversion_tool).count()
+ message = f"Retrying {retry_conversion_tool.label} liftover to {dest_genome_build} " \
+ f"for {num_alleles} alleles"
+ else:
+ message = f"Lifting over alleles to {dest_genome_build}"
+ messages.add_message(request, messages.INFO, message)
+ liftover_alleles.si(request.user.username, dest_genome_build.name,
+ retry_conversion_tool.value if retry_conversion_tool else None).apply_async()
alleles_missing_variants = {}
+ retry_counts = {}
for genome_build in genome_builds:
- alleles_missing_variants[genome_build.name] = Allele.missing_variants_for_build(genome_build).count()
+ missing_qs = Allele.missing_variants_for_build(genome_build)
+ alleles_missing_variants[genome_build.name] = missing_qs.count()
+ retry_counts[genome_build.name] = _failed_tool_counts(genome_build, missing_qs)
qs_allele_liftover = AlleleLiftover.objects.all()
@@ -59,6 +65,7 @@ def liftover_runs(request):
context = {
"genome_builds": genome_builds,
"alleles_missing_variants": alleles_missing_variants,
+ "retry_counts": retry_counts,
"processing_status_cols": processing_status_cols,
"tool_status": tool_status,
"failure_rate": failure_rate,
@@ -67,6 +74,26 @@ def liftover_runs(request):
return render(request, "snpdb/liftover/liftover_runs.html", context)
+def _liftover_post_action(post, genome_builds) -> tuple[GenomeBuild, Optional[AlleleConversionTool]]:
+ """ Buttons are named liftover_to_{build} (everything missing) or retry_{build}_{tool} (that tool's failures) """
+ for genome_build in genome_builds:
+ if f"liftover_to_{genome_build.name}" in post:
+ return genome_build, None
+ for act in AlleleConversionTool:
+ if f"retry_{genome_build.name}_{act.value}" in post:
+ return genome_build, act
+ raise ValueError("Could not determine dest genome build from liftover_runs POST")
+
+
+def _failed_tool_counts(genome_build, missing_qs) -> list[tuple[AlleleConversionTool, int]]:
+ """ (tool, number of alleles a retry would re-attempt) for each tool with failures, in enum order """
+ qs = AlleleLiftover.objects.filter(liftover__genome_build=genome_build,
+ status=ProcessingStatus.ERROR,
+ allele__in=missing_qs)
+ counts = dict(qs.values_list("liftover__conversion_tool").annotate(num_alleles=Count("allele", distinct=True)))
+ return [(act, counts[act.value]) for act in AlleleConversionTool if counts.get(act.value)]
+
+
@require_superuser
def view_liftover_run(request, liftover_run_id):
liftover_run = LiftoverRun.objects.get(pk=liftover_run_id)
diff --git a/variantgrid/templates/default_templates/changelog.html b/variantgrid/templates/default_templates/changelog.html
index 507a4bcbc..ed7179c70 100644
--- a/variantgrid/templates/default_templates/changelog.html
+++ b/variantgrid/templates/default_templates/changelog.html
@@ -290,6 +290,10 @@