diff --git a/analysis/fake_variant_tags.py b/analysis/fake_variant_tags.py
index b83fe1c0d..0ec3202fb 100644
--- a/analysis/fake_variant_tags.py
+++ b/analysis/fake_variant_tags.py
@@ -26,8 +26,7 @@
from analysis.models import VariantTag
from analysis.models.enums import TagLocation
-from annotation.models import VariantAnnotation, VariantAnnotationVersion
-from genes.models import TranscriptVersion
+from annotation.fake_data import get_variant_ids_by_gene, zipf_weight
from library.guardian_utils import all_users_group
from snpdb.models import Allele, AlleleConversionTool, AlleleOrigin, GenomeBuild, GlobalSettings, Lab, Organization, \
Tag, TagColor, TagColorsCollection, VariantAllele
@@ -219,13 +218,13 @@ def _create_tags(self):
def _pick_variants(self, genome_build: GenomeBuild, group: str, genes: list[str],
num_variants: int) -> list[FakeVariant]:
""" Real variants, so the gene cards have real symbols to group on and the links all work """
- variant_ids_by_gene = _variant_ids_by_gene(genome_build, genes)
+ variant_ids_by_gene = get_variant_ids_by_gene(genome_build, genes)
tags = [t for t in FAKE_TAGS if t.group in (group, BOTH)]
fake_variants = []
for i, gene_symbol in enumerate(genes):
available = variant_ids_by_gene.get(gene_symbol, [])
- wanted = int(num_variants * _zipf_weight(i) / sum(_zipf_weight(j) for j in range(len(genes))))
+ wanted = int(num_variants * zipf_weight(i) / sum(zipf_weight(j) for j in range(len(genes))))
if len(available) < wanted:
self.stdout.write(f"{gene_symbol}: only {len(available)} annotated variants, wanted {wanted}")
wanted = len(available)
@@ -333,22 +332,6 @@ def _tag_colors_collection() -> TagColorsCollection:
return collection
-def _variant_ids_by_gene(genome_build: GenomeBuild, genes: list[str]) -> dict[str, list[int]]:
- gene_symbol_field = "transcript_version__gene_version__gene_symbol_id"
- transcript_versions_qs = TranscriptVersion.objects.filter(genome_build=genome_build,
- gene_version__gene_symbol__in=genes)
- variant_annotation_qs = VariantAnnotation.objects.filter(version=VariantAnnotationVersion.latest(genome_build),
- transcript_version__in=transcript_versions_qs)
- variant_ids_by_gene = {}
- for gene_symbol, variant_id in variant_annotation_qs.values_list(gene_symbol_field, "variant_id"):
- variant_ids_by_gene.setdefault(gene_symbol, []).append(variant_id)
- return variant_ids_by_gene
-
-
-def _zipf_weight(rank: int) -> float:
- return 1 / (rank + 1) ** 0.8
-
-
def _allocate_events(fake_variants: list[FakeVariant], num_events: int):
""" Everything is tagged once, then the rest are dealt out by weight - the tail is where re-tagging lives """
extra = num_events - len(fake_variants)
diff --git a/annotation/external_annotation.py b/annotation/external_annotation.py
index 1f811f1b7..532bb6567 100644
--- a/annotation/external_annotation.py
+++ b/annotation/external_annotation.py
@@ -152,6 +152,59 @@ def parse_dump_metadata(path) -> dict:
return meta
+class _ExternalDump:
+ """ One dump directory's worth of runs. The callers differ only in how they get hold of their
+ AnnotationRuns - creating them up front, or adopting existing ones - then each run goes through
+ add_run() and the dump is closed off with finish(). """
+
+ def __init__(self, variant_annotation_version: VariantAnnotationVersion, output_dir: str,
+ pipeline_type, min_variants: int, described_as: str):
+ self.variant_annotation_version = variant_annotation_version
+ self.output_dir = output_dir
+ self.pipeline_type = pipeline_type
+ self.min_variants = min_variants
+ self.described_as = described_as # "external" or "existing", for the log lines
+ # Every run in this dump shares the same VAV, so compute the version identity (which runs VEP) once,
+ # lazily on the first run so a no-op dump never invokes VEP.
+ self._identity = None
+ self.annotation_runs = []
+ self.reverted = []
+ os.makedirs(output_dir, exist_ok=True)
+
+ def add_run(self, annotation_run: AnnotationRun):
+ # If already counted off-thread (AnnotationRun.count, #1646), reject a too-small run before dumping.
+ # A counted run here is always non-empty (the count lane finishes count==0 runs itself).
+ if annotation_run.count is not None and annotation_run.count < self.min_variants:
+ _revert_too_small_run(annotation_run, annotation_run.count, self.min_variants, self.reverted)
+ return
+ dump_count = get_runner(self.pipeline_type).dump(annotation_run, dump_dir=self.output_dir)
+ # Range locks are sized without pipeline_type, so an SV dump leaves most locks with zero SVs; such a
+ # run is already FINISHED (get_status: dump_count == 0) with nothing to annotate, so skip its sidecar
+ # rather than emit a no-op .meta.json (thousands of them, for a handful of SVs).
+ if dump_count == 0:
+ return
+ if dump_count < self.min_variants: # uncounted run whose dump turned out too small
+ _revert_too_small_run(annotation_run, dump_count, self.min_variants, self.reverted)
+ return
+ if self._identity is None:
+ self._identity = variant_annotation_version_identity(self.variant_annotation_version)
+ meta_filename = write_dump_metadata(annotation_run, dump_dir=self.output_dir, identity=self._identity)
+ logging.info("Dumped %s AnnotationRun %s: %d variants -> %s (meta %s)",
+ self.described_as, annotation_run.pk, dump_count, annotation_run.vcf_dump_filename,
+ meta_filename)
+ self.annotation_runs.append(annotation_run)
+
+ def finish(self) -> list[AnnotationRun]:
+ write_snakemake_bundle(self.output_dir, self.variant_annotation_version,
+ pipeline_type=self.pipeline_type)
+ if self.reverted:
+ dispatch_annotation_runs.si(self.variant_annotation_version.pk).apply_async()
+ logging.info("Dumped %d %s annotation run(s) for %s into %s (reverted %d too-small run(s) to local)",
+ len(self.annotation_runs), self.described_as, self.variant_annotation_version,
+ self.output_dir, len(self.reverted))
+ return self.annotation_runs
+
+
def dump_external_annotation_runs(variant_annotation_version: VariantAnnotationVersion,
output_dir: str,
pipeline_type=VariantAnnotationPipelineType.STANDARD,
@@ -167,47 +220,16 @@ def dump_external_annotation_runs(variant_annotation_version: VariantAnnotationV
A run holding fewer than `min_variants` variants is reverted to the local pipeline instead of parked
external (see DEFAULT_MIN_EXTERNAL_VARIANTS) - the external round-trip is not worth it for a tiny run. """
_require_sv_offload_supported(pipeline_type)
- os.makedirs(output_dir, exist_ok=True)
- # Every run in this dump shares the same VAV, so compute the version identity (which runs VEP) once,
- # lazily on the first run so a no-op dump never invokes VEP.
- identity = None
- annotation_runs = []
- reverted = []
+ dump = _ExternalDump(variant_annotation_version, output_dir, pipeline_type, min_variants, "external")
while True:
range_lock, _unannotated_count = get_annotation_range_lock_and_unannotated_count(
variant_annotation_version, settings.ANNOTATION_VEP_BATCH_MIN, settings.ANNOTATION_VEP_BATCH_MAX)
if range_lock is None:
break
range_lock.save()
- annotation_run = AnnotationRun.objects.create(annotation_range_lock=range_lock,
- pipeline_type=pipeline_type, external=True)
- # If already counted off-thread (AnnotationRun.count, #1646), reject a too-small run before dumping.
- # A counted run here is always non-empty (the count lane finishes count==0 runs itself).
- if annotation_run.count is not None and annotation_run.count < min_variants:
- _revert_too_small_run(annotation_run, annotation_run.count, min_variants, reverted)
- continue
- dump_count = get_runner(pipeline_type).dump(annotation_run, dump_dir=output_dir)
- # Range locks are sized without pipeline_type, so an SV dump leaves most locks with zero SVs; such a
- # run is already FINISHED (get_status: dump_count == 0) with nothing to annotate, so skip its sidecar
- # rather than emit a no-op .meta.json (thousands of them, for a handful of SVs).
- if dump_count == 0:
- continue
- if dump_count < min_variants: # uncounted run whose dump turned out too small
- _revert_too_small_run(annotation_run, dump_count, min_variants, reverted)
- continue
- if identity is None:
- identity = variant_annotation_version_identity(variant_annotation_version)
- meta_filename = write_dump_metadata(annotation_run, dump_dir=output_dir, identity=identity)
- logging.info("Dumped external AnnotationRun %s: %d variants -> %s (meta %s)",
- annotation_run.pk, dump_count, annotation_run.vcf_dump_filename, meta_filename)
- annotation_runs.append(annotation_run)
-
- write_snakemake_bundle(output_dir, variant_annotation_version, pipeline_type=pipeline_type)
- if reverted:
- dispatch_annotation_runs.si(variant_annotation_version.pk).apply_async()
- logging.info("Dumped %d external annotation run(s) for %s into %s (reverted %d too-small run(s) to local)",
- len(annotation_runs), variant_annotation_version, output_dir, len(reverted))
- return annotation_runs
+ dump.add_run(AnnotationRun.objects.create(annotation_range_lock=range_lock,
+ pipeline_type=pipeline_type, external=True))
+ return dump.finish()
def dump_existing_annotation_runs(variant_annotation_version: VariantAnnotationVersion,
@@ -227,8 +249,8 @@ def dump_existing_annotation_runs(variant_annotation_version: VariantAnnotationV
if leave < 0:
raise ValueError(f"leave must be >= 0, got {leave}")
_require_sv_offload_supported(pipeline_type)
+ dump = _ExternalDump(variant_annotation_version, output_dir, pipeline_type, min_variants, "existing")
- os.makedirs(output_dir, exist_ok=True)
now = timezone.now()
# Mirror the dispatcher's dispatchable filter (annotation_scheduler_task._dispatchable_runs_qs) and its
# lowest-min-variant-first order, so we adopt exactly the runs it would otherwise launch.
@@ -246,11 +268,6 @@ def dump_existing_annotation_runs(variant_annotation_version: VariantAnnotationV
logging.info("dump_existing: %d dispatchable run(s); leaving %d on the local pipeline, dumping %d",
len(kept) + len(candidate_ids), len(kept), len(candidate_ids))
- # Every run in this dump shares the same VAV, so compute the version identity (which runs VEP) once,
- # lazily on the first claimed run so a no-op dump never invokes VEP.
- identity = None
- annotation_runs = []
- reverted = []
for pk in candidate_ids:
# Atomically claim as external only while still dispatchable (same filter as the dispatcher) so we
# never adopt a run it just leased. If we lose the race (0 rows updated) skip it; if we win, the run
@@ -266,33 +283,9 @@ def dump_existing_annotation_runs(variant_annotation_version: VariantAnnotationV
"scheduler?)", pk)
continue
- annotation_run = AnnotationRun.objects.get(pk=pk)
- # If already counted off-thread (AnnotationRun.count, #1646), reject a too-small run before dumping.
- # A counted run here is always non-empty (the count lane finishes count==0 runs itself).
- if annotation_run.count is not None and annotation_run.count < min_variants:
- _revert_too_small_run(annotation_run, annotation_run.count, min_variants, reverted)
- continue
- dump_count = get_runner(pipeline_type).dump(annotation_run, dump_dir=output_dir)
- # A zero-count run is already FINISHED (get_status: dump_count == 0) with nothing to annotate, so
- # skip its sidecar rather than emit a no-op .meta.json (see dump_external_annotation_runs).
- if dump_count == 0:
- continue
- if dump_count < min_variants: # uncounted run whose dump turned out too small
- _revert_too_small_run(annotation_run, dump_count, min_variants, reverted)
- continue
- if identity is None:
- identity = variant_annotation_version_identity(variant_annotation_version)
- meta_filename = write_dump_metadata(annotation_run, dump_dir=output_dir, identity=identity)
- logging.info("Dumped existing AnnotationRun %s: %d variants -> %s (meta %s)",
- annotation_run.pk, dump_count, annotation_run.vcf_dump_filename, meta_filename)
- annotation_runs.append(annotation_run)
-
- write_snakemake_bundle(output_dir, variant_annotation_version, pipeline_type=pipeline_type)
- if reverted:
- dispatch_annotation_runs.si(variant_annotation_version.pk).apply_async()
- logging.info("Dumped %d existing annotation run(s) for %s into %s (reverted %d too-small run(s) to local)",
- len(annotation_runs), variant_annotation_version, output_dir, len(reverted))
- return annotation_runs
+ dump.add_run(AnnotationRun.objects.get(pk=pk))
+
+ return dump.finish()
def verify_annotated_vcf_variant_ids(annotation_run: AnnotationRun, meta: dict):
diff --git a/annotation/fake_data.py b/annotation/fake_data.py
new file mode 100644
index 000000000..5b8576930
--- /dev/null
+++ b/annotation/fake_data.py
@@ -0,0 +1,24 @@
+"""
+Helpers shared by the 'manage.py create_fake_data' subcommands, which live in the app whose data they
+create (@see snpdb.management.commands.create_fake_data).
+"""
+from annotation.models import VariantAnnotation, VariantAnnotationVersion
+from genes.models import TranscriptVersion
+from snpdb.models import GenomeBuild
+
+
+def get_variant_ids_by_gene(genome_build: GenomeBuild, genes: list[str]) -> dict[str, list[int]]:
+ gene_symbol_field = "transcript_version__gene_version__gene_symbol_id"
+ transcript_versions_qs = TranscriptVersion.objects.filter(genome_build=genome_build,
+ gene_version__gene_symbol__in=genes)
+ variant_annotation_qs = VariantAnnotation.objects.filter(version=VariantAnnotationVersion.latest(genome_build),
+ transcript_version__in=transcript_versions_qs)
+ variant_ids_by_gene = {}
+ for gene_symbol, variant_id in variant_annotation_qs.values_list(gene_symbol_field, "variant_id"):
+ variant_ids_by_gene.setdefault(gene_symbol, []).append(variant_id)
+ return variant_ids_by_gene
+
+
+def zipf_weight(rank: int) -> float:
+ """ Long tail - a few genes/records get most of the action, the rest get a little """
+ return 1 / (rank + 1) ** 0.8
diff --git a/classification/fake_reclassifications.py b/classification/fake_reclassifications.py
index 95fffe5ee..c29b7a647 100644
--- a/classification/fake_reclassifications.py
+++ b/classification/fake_reclassifications.py
@@ -29,13 +29,12 @@
from django.utils.timezone import get_current_timezone, now
from guardian.models import GroupObjectPermission
-from annotation.models import VariantAnnotation, VariantAnnotationVersion
+from annotation.fake_data import get_variant_ids_by_gene, zipf_weight
from classification.enums import (ClinicalSignificance, CriteriaEvaluation, ShareLevel, SpecialEKeys,
SubmissionSource)
from classification.models import (Classification, ClassificationModification, ImportedAlleleInfo,
ReclassificationEvent, ReclassificationEventBuilder, ResolvedVariantInfo)
from classification.models.classification_variant_info_models import ImportedAlleleInfoStatus
-from genes.models import TranscriptVersion
from library.guardian_utils import all_users_group
from snpdb.models import GenomeBuild, GenomeBuildPatchVersion, Lab, Organization
@@ -241,7 +240,7 @@ def _create_labs_and_curators(self) -> tuple[dict[str, Lab], list[User]]:
def _pick_records(self, genome_build: GenomeBuild, labs: dict[str, Lab],
num_classifications: int, years: int) -> list[FakeRecord]:
""" Real variants, so the gene chart has real symbols and the records link somewhere sensible """
- variant_ids_by_gene = _variant_ids_by_gene(genome_build, GENES)
+ variant_ids_by_gene = get_variant_ids_by_gene(genome_build, GENES)
lab_weights = [fake_lab.weight for fake_lab in FAKE_LABS]
significances = list(SIGNIFICANCE_BEHAVIOUR)
significance_weights = [SIGNIFICANCE_BEHAVIOUR[s].weight for s in significances]
@@ -251,8 +250,8 @@ def _pick_records(self, genome_build: GenomeBuild, labs: dict[str, Lab],
records = []
for index, gene_symbol in enumerate(GENES):
available = variant_ids_by_gene.get(gene_symbol, [])
- wanted = round(num_classifications * _zipf_weight(index)
- / sum(_zipf_weight(i) for i in range(len(GENES))))
+ wanted = round(num_classifications * zipf_weight(index)
+ / sum(zipf_weight(i) for i in range(len(GENES))))
if len(available) < wanted:
self.stdout.write(f"{gene_symbol}: only {len(available)} annotated variants, wanted {wanted}")
wanted = len(available)
@@ -408,22 +407,6 @@ def _period_start(years: int) -> datetime:
return now().astimezone(get_current_timezone()) - timedelta(days=years * 365)
-def _zipf_weight(rank: int) -> float:
- return 1 / (rank + 1) ** 0.8
-
-
-def _variant_ids_by_gene(genome_build: GenomeBuild, genes: list[str]) -> dict[str, list[int]]:
- gene_symbol_field = "transcript_version__gene_version__gene_symbol_id"
- transcript_versions_qs = TranscriptVersion.objects.filter(genome_build=genome_build,
- gene_version__gene_symbol__in=genes)
- variant_annotation_qs = VariantAnnotation.objects.filter(version=VariantAnnotationVersion.latest(genome_build),
- transcript_version__in=transcript_versions_qs)
- variant_ids_by_gene = {}
- for gene_symbol, variant_id in variant_annotation_qs.values_list(gene_symbol_field, "variant_id"):
- variant_ids_by_gene.setdefault(gene_symbol, []).append(variant_id)
- return variant_ids_by_gene
-
-
def _summary(step: FakeStep) -> dict:
return {"clinical_significance": step.significance, "date": step.curated_on.isoformat()}
diff --git a/library/graphs/chromosomes_graph.py b/library/graphs/chromosomes_graph.py
index 85fd67b37..abfa67350 100644
--- a/library/graphs/chromosomes_graph.py
+++ b/library/graphs/chromosomes_graph.py
@@ -4,6 +4,7 @@
from collections import defaultdict
+import numpy as np
import pandas as pd
from library.genomics import format_chrom
@@ -81,3 +82,30 @@ def plot_chromosomes(ax, cytoband_filename, has_chr=False, **kwargs):
ax.set_yticklabels(yticklabels)
ax.set_xticks([])
return chrom_ranges
+
+
+def plot_chromosome_bin_values(ax, chrom_ranges, bin_values_by_chrom, bin_size, cmap, vmax,
+ padding, alpha):
+ """ Draw a heatmap band down each chromosome from its per-bin values, all sharing one colour scale.
+
+ Returns one of the QuadMeshes, for the figure to hang a colorbar off. """
+ quadmesh = None
+ for chrom, (_xranges, yranges) in chrom_ranges.items():
+ bins = bin_values_by_chrom[chrom]
+ num_bins = len(bins) + 1
+ x_pos = np.arange(num_bins) * bin_size
+
+ # Get into right dimensions
+ bins = [bins]
+ # pcolor says x,y should have dimensions 1 greater than colors
+ x_pos = [x_pos, x_pos]
+ y_top = np.empty(num_bins)
+ y_top.fill(yranges[0])
+ y_bottom = np.empty(num_bins)
+ y_bottom.fill(yranges[0] + yranges[1])
+ y_pos = [y_top + padding, y_bottom - padding]
+
+ masked_c = np.ma.masked_invalid(np.array(bins))
+ quadmesh = ax.pcolormesh(np.array(x_pos), np.array(y_pos), masked_c, cmap=cmap, alpha=alpha)
+ quadmesh.set_clim(vmin=0, vmax=vmax)
+ return quadmesh
diff --git a/snpdb/graphs/chromosome_density_graph.py b/snpdb/graphs/chromosome_density_graph.py
index fc80a6f42..0c6fe5e3b 100644
--- a/snpdb/graphs/chromosome_density_graph.py
+++ b/snpdb/graphs/chromosome_density_graph.py
@@ -5,7 +5,7 @@
from django.db import connection
from library.genomics import get_genomic_size_description
-from library.graphs.chromosomes_graph import plot_chromosomes
+from library.graphs.chromosomes_graph import plot_chromosome_bin_values, plot_chromosomes
from library.utils import sha256sum_str
from library.utils.database_utils import get_queryset_select_from_where_parts
from patients.models_enums import Zygosity
@@ -85,31 +85,9 @@ def plot(self, ax):
vmax = max(vmax, np.max(bin_counts[~np.isnan(bin_counts)]))
chrom_densities[chrom] = bin_counts
- for chrom, (xranges, yranges) in chrom_ranges.items():
- bins = chrom_densities[chrom]
- num_bins = len(bins) + 1
- x_pos = np.arange(num_bins) * BIN_SIZE
-
- # Get into right dimensions
- bins = [bins]
- # pcolor says x,y should have dimensions 1 greater than colors
- x_pos = [x_pos, x_pos]
- y_top = np.empty(num_bins)
- y_top.fill(yranges[0])
- y_bottom = np.empty(num_bins)
- y_bottom.fill(yranges[0] + yranges[1])
- y_pos = [y_top + density_padding,
- y_bottom - density_padding]
-
- x = np.array(x_pos)
- y = np.array(y_pos)
- c = np.array(bins)
- # TODO: log??
-
- masked_c = np.ma.masked_invalid(c)
- quadmesh = ax.pcolormesh(x, y, masked_c, cmap=self.cmap, alpha=density_alpha)
- quadmesh.set_clim(vmin=0, vmax=vmax)
- self.im = quadmesh # Just need one
+ # TODO: log??
+ self.im = plot_chromosome_bin_values(ax, chrom_ranges, chrom_densities, BIN_SIZE, self.cmap,
+ vmax, density_padding, density_alpha)
title = self.get_title()
if title:
diff --git a/snpdb/graphs/homozygosity_percent_graph.py b/snpdb/graphs/homozygosity_percent_graph.py
index e4b4820e0..497df6f4a 100644
--- a/snpdb/graphs/homozygosity_percent_graph.py
+++ b/snpdb/graphs/homozygosity_percent_graph.py
@@ -6,7 +6,7 @@
from django.db.models import Count, ExpressionWrapper, F, IntegerField
from library.genomics import get_genomic_size_description
-from library.graphs.chromosomes_graph import plot_chromosomes
+from library.graphs.chromosomes_graph import plot_chromosome_bin_values, plot_chromosomes
from library.utils import sha256sum_str
from snpdb.graphs.graphcache import CacheableGraph
from snpdb.models import Sample, Variant
@@ -110,30 +110,8 @@ def plot(self, ax):
logging.debug("HomozygosityPercentGraph: %d bins", num_bins)
- for chrom, (xranges, yranges) in chrom_ranges.items():
- bins = chrom_homo_percent[chrom]
- num_bins = len(bins) + 1
- x_pos = np.arange(num_bins) * BIN_SIZE
-
- # Get into right dimensions
- bins = [bins]
- # pcolor says x,y should have dimensions 1 greater than colors
- x_pos = [x_pos, x_pos]
- y_top = np.empty(num_bins)
- y_top.fill(yranges[0])
- y_bottom = np.empty(num_bins)
- y_bottom.fill(yranges[0] + yranges[1])
- y_pos = [y_top + density_padding,
- y_bottom - density_padding]
-
- x = np.array(x_pos)
- y = np.array(y_pos)
- c = np.array(bins)
-
- masked_c = np.ma.masked_invalid(c)
- quadmesh = ax.pcolormesh(x, y, masked_c, cmap=self.cmap, alpha=density_alpha)
- quadmesh.set_clim(vmin=0, vmax=100.0)
- self.im = quadmesh # Just need one
+ self.im = plot_chromosome_bin_values(ax, chrom_ranges, chrom_homo_percent, BIN_SIZE, self.cmap,
+ 100.0, density_padding, density_alpha)
bin_size_description = get_genomic_size_description(BIN_SIZE)
ax.set_title(f"{self.sample.name}\n(min depth {MIN_DEPTH}, min {MIN_VARIANTS_PER_BIN} variants per {bin_size_description})")
diff --git a/snpdb/grids.py b/snpdb/grids.py
index 56208b27e..253ba3db9 100644
--- a/snpdb/grids.py
+++ b/snpdb/grids.py
@@ -47,7 +47,7 @@
VariantZygosityCountCollection,
)
from snpdb.sample_filters import get_sample_ontology_q, get_sample_qc_gene_list_gene_symbol_q
-from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder
+from snpdb.views.datatable_view import DC, CellData, DatatableConfig, RichColumn, SortOrder
from uicore.templatetags.js_tags import jsonify_for_js
from variantgrid.perm_path import get_visible_url_names
@@ -360,9 +360,21 @@ def filter_queryset(self, qs: QuerySet[Cohort]) -> QuerySet[Cohort]:
return qs
-class TriosListColumns(DatatableConfig[Trio]):
+class FamilyGroupListColumns(DatatableConfig[DC]):
+ """ Trios/Quads listing - same grid bar the extra family members """
+ MODEL: type[DC]
+ GRID_NAME: str
+ # (field prefix, label, has an affected column)
+ FAMILY_MEMBERS = [("mother", "Mother", True), ("father", "Father", True), ("proband", "Proband", False)]
+
def __init__(self, request: HttpRequest):
super().__init__(request)
+ member_columns = []
+ for member, label, has_affected in self.FAMILY_MEMBERS:
+ member_columns.append(RichColumn(key=f'{member}__sample__name', label=label, orderable=True))
+ if has_affected:
+ member_columns.append(RichColumn(key=f'{member}_affected', label=f'{label} Affected',
+ orderable=True))
self.rich_columns = [
RichColumn(key='id', visible=False),
RichColumn(key='name', label='Name', orderable=True,
@@ -371,57 +383,31 @@ def __init__(self, request: HttpRequest):
RichColumn(key='user__username', label='User', orderable=True),
RichColumn(key='modified', client_renderer='TableFormat.timestamp', orderable=True,
default_sort=SortOrder.DESC),
- RichColumn(key='mother__sample__name', label='Mother', orderable=True),
- RichColumn(key='mother_affected', label='Mother Affected', orderable=True),
- RichColumn(key='father__sample__name', label='Father', orderable=True),
- RichColumn(key='father_affected', label='Father Affected', orderable=True),
- RichColumn(key='proband__sample__name', label='Proband', orderable=True),
+ *member_columns,
RichColumn(key='id', name='delete', label='', orderable=False,
renderer=self.render_delete,
client_renderer='TableFormat.deleteRow'),
]
- def get_initial_queryset(self) -> QuerySet[Trio]:
- return Trio.filter_for_user(self.user)
+ def get_initial_queryset(self) -> QuerySet[DC]:
+ return self.MODEL.filter_for_user(self.user)
- def filter_queryset(self, qs: QuerySet[Trio]) -> QuerySet[Trio]:
- user_grid_config = UserGridConfig.get(self.user, 'Trios')
+ def filter_queryset(self, qs: QuerySet[DC]) -> QuerySet[DC]:
+ user_grid_config = UserGridConfig.get(self.user, self.GRID_NAME)
if not user_grid_config.show_group_data:
qs = qs.filter(user=self.user)
return qs
-class QuadsListColumns(DatatableConfig[Quad]):
- def __init__(self, request: HttpRequest):
- super().__init__(request)
- self.rich_columns = [
- RichColumn(key='id', visible=False),
- RichColumn(key='name', label='Name', orderable=True,
- renderer=self.view_primary_key,
- client_renderer='TableFormat.linkUrl'),
- RichColumn(key='user__username', label='User', orderable=True),
- RichColumn(key='modified', client_renderer='TableFormat.timestamp', orderable=True,
- default_sort=SortOrder.DESC),
- RichColumn(key='mother__sample__name', label='Mother', orderable=True),
- RichColumn(key='mother_affected', label='Mother Affected', orderable=True),
- RichColumn(key='father__sample__name', label='Father', orderable=True),
- RichColumn(key='father_affected', label='Father Affected', orderable=True),
- RichColumn(key='proband__sample__name', label='Proband', orderable=True),
- RichColumn(key='sibling__sample__name', label='Sibling', orderable=True),
- RichColumn(key='sibling_affected', label='Sibling Affected', orderable=True),
- RichColumn(key='id', name='delete', label='', orderable=False,
- renderer=self.render_delete,
- client_renderer='TableFormat.deleteRow'),
- ]
+class TriosListColumns(FamilyGroupListColumns[Trio]):
+ MODEL = Trio
+ GRID_NAME = 'Trios'
- def get_initial_queryset(self) -> QuerySet[Quad]:
- return Quad.filter_for_user(self.user)
- def filter_queryset(self, qs: QuerySet[Quad]) -> QuerySet[Quad]:
- user_grid_config = UserGridConfig.get(self.user, 'Quads')
- if not user_grid_config.show_group_data:
- qs = qs.filter(user=self.user)
- return qs
+class QuadsListColumns(FamilyGroupListColumns[Quad]):
+ MODEL = Quad
+ GRID_NAME = 'Quads'
+ FAMILY_MEMBERS = [*FamilyGroupListColumns.FAMILY_MEMBERS, ("sibling", "Sibling", True)]
class GenomicIntervalsListColumns(DatatableConfig[GenomicIntervalsCollection]):
@@ -445,7 +431,9 @@ def get_initial_queryset(self) -> QuerySet[GenomicIntervalsCollection]:
return get_objects_for_user(self.user, 'snpdb.view_genomicintervalscollection', accept_global_perms=False)
-class CustomColumnsCollectionColumns(DatatableConfig[CustomColumnsCollection]):
+class NamedCollectionColumns(DatatableConfig[DC]):
+ """ A user's named collections - nothing to show but who made it and when """
+ MODEL: type[DC]
def __init__(self, request):
super().__init__(request)
@@ -462,8 +450,12 @@ def __init__(self, request):
default_sort=SortOrder.DESC),
]
- def get_initial_queryset(self) -> QuerySet[CustomColumnsCollection]:
- return CustomColumnsCollection.filter_for_user(self.user)
+ def get_initial_queryset(self) -> QuerySet[DC]:
+ return self.MODEL.filter_for_user(self.user)
+
+
+class CustomColumnsCollectionColumns(NamedCollectionColumns[CustomColumnsCollection]):
+ MODEL = CustomColumnsCollection
def server_side_format_clingen_allele(row, field):
@@ -615,25 +607,8 @@ def get_queryset_field_names(self):
return field_names
-class TagColorsCollectionColumns(DatatableConfig[TagColorsCollection]):
-
- def __init__(self, request):
- super().__init__(request)
- self.user = request.user
-
- self.rich_columns = [
- RichColumn(key="id", visible=False),
- RichColumn(key="name", label="Name", orderable=True,
- renderer=self.view_primary_key,
- client_renderer='TableFormat.linkUrl'),
- RichColumn(key="user__username", label="User", orderable=True),
- RichColumn(key="created", client_renderer='TableFormat.timestamp', orderable=True),
- RichColumn(key="modified", client_renderer='TableFormat.timestamp', orderable=True,
- default_sort=SortOrder.DESC),
- ]
-
- def get_initial_queryset(self) -> QuerySet[TagColorsCollection]:
- return TagColorsCollection.filter_for_user(self.user)
+class TagColorsCollectionColumns(NamedCollectionColumns[TagColorsCollection]):
+ MODEL = TagColorsCollection
class LiftoverRunColumns(DatatableConfig[LiftoverRun]):
diff --git a/snpdb/models/models_cohort.py b/snpdb/models/models_cohort.py
index 4764a5348..4f069a27c 100644
--- a/snpdb/models/models_cohort.py
+++ b/snpdb/models/models_cohort.py
@@ -764,17 +764,9 @@ class Meta:
unique_together = ("collection", "variant")
-class Trio(GuardianPermissionsAutoInitialSaveMixin, PreviewModelMixin, SortByPKMixin, TimeStampedModel):
- """ A simple pedigree used frequently for Mendellian disease (TrioNode in analysis)
- and karyomapping """
- name = models.TextField(blank=True)
- user = models.ForeignKey(User, null=True, on_delete=CASCADE)
- cohort = models.ForeignKey(Cohort, on_delete=CASCADE)
- mother = models.ForeignKey(CohortSample, related_name='trio_mother', on_delete=CASCADE)
- mother_affected = models.BooleanField(default=False)
- father = models.ForeignKey(CohortSample, related_name='trio_father', on_delete=CASCADE)
- father_affected = models.BooleanField(default=False)
- proband = models.ForeignKey(CohortSample, related_name='trio_proband', on_delete=CASCADE)
+class FamilyGroupMixin:
+ """ Shared by Trio and Quad - permissions and display that don't care how many members there are.
+ Subclasses provide get_cohort_samples() and their own urls. """
@classmethod
def get_permission_class(cls):
@@ -784,16 +776,12 @@ def get_permission_class(cls):
def preview_icon(cls) -> str:
return "fa-solid fa-people-roof"
- @classmethod
- def preview_if_url_visible(cls) -> str:
- return "trios"
-
@property
def preview(self) -> 'PreviewData':
return self.preview_with(identifier=str(self))
def get_permission_object(self):
- # Trio permissions based on cohort
+ # Permissions are based on the cohort
return self.cohort
@classmethod
@@ -808,33 +796,52 @@ def genome_build(self):
def data_archived(self) -> bool:
return self.cohort.data_archived
- def get_cohort_samples(self):
- return [self.mother, self.father, self.proband]
-
def get_samples(self):
return Sample.objects.filter(cohortsample__in=self.get_cohort_samples()).order_by("pk")
- def get_absolute_url(self):
- return reverse('view_trio', kwargs={"pk": self.pk})
-
- def get_listing_url(self):
- return reverse('trios')
+ @staticmethod
+ def _member_details(member, affected: bool) -> str:
+ return f"{member} ({'affected' if affected else 'unaffected'})"
@property
def mother_details(self):
- affected = "affected" if self.mother_affected else "unaffected"
- return f"{self.mother} ({affected})"
+ return self._member_details(self.mother, self.mother_affected)
@property
def father_details(self):
- affected = "affected" if self.father_affected else "unaffected"
- return f"{self.father} ({affected})"
+ return self._member_details(self.father, self.father_affected)
def __str__(self):
- return self.name or f"Trio {self.pk}"
+ return self.name or f"{type(self).__name__} {self.pk}"
+
+
+class Trio(FamilyGroupMixin, GuardianPermissionsAutoInitialSaveMixin, PreviewModelMixin, SortByPKMixin, TimeStampedModel):
+ """ A simple pedigree used frequently for Mendellian disease (TrioNode in analysis)
+ and karyomapping """
+ name = models.TextField(blank=True)
+ user = models.ForeignKey(User, null=True, on_delete=CASCADE)
+ cohort = models.ForeignKey(Cohort, on_delete=CASCADE)
+ mother = models.ForeignKey(CohortSample, related_name='trio_mother', on_delete=CASCADE)
+ mother_affected = models.BooleanField(default=False)
+ father = models.ForeignKey(CohortSample, related_name='trio_father', on_delete=CASCADE)
+ father_affected = models.BooleanField(default=False)
+ proband = models.ForeignKey(CohortSample, related_name='trio_proband', on_delete=CASCADE)
+
+ @classmethod
+ def preview_if_url_visible(cls) -> str:
+ return "trios"
+
+ def get_cohort_samples(self):
+ return [self.mother, self.father, self.proband]
+
+ def get_absolute_url(self):
+ return reverse('view_trio', kwargs={"pk": self.pk})
+
+ def get_listing_url(self):
+ return reverse('trios')
-class Quad(GuardianPermissionsAutoInitialSaveMixin, PreviewModelMixin, SortByPKMixin, TimeStampedModel):
+class Quad(FamilyGroupMixin, GuardianPermissionsAutoInitialSaveMixin, PreviewModelMixin, SortByPKMixin, TimeStampedModel):
"""Mother + Father + Proband + Sibling.
Extends the Trio concept to 4 family members. The sibling (typically
@@ -852,66 +859,22 @@ class Quad(GuardianPermissionsAutoInitialSaveMixin, PreviewModelMixin, SortByPKM
sibling = models.ForeignKey(CohortSample, related_name='quad_sibling', on_delete=CASCADE)
sibling_affected = models.BooleanField(default=False)
- @classmethod
- def get_permission_class(cls):
- return Cohort
-
- @classmethod
- def preview_icon(cls) -> str:
- return "fa-solid fa-people-roof"
-
@classmethod
def preview_if_url_visible(cls) -> str:
return "quads"
- @property
- def preview(self) -> 'PreviewData':
- return self.preview_with(identifier=str(self))
-
- def get_permission_object(self):
- return self.cohort
-
- @classmethod
- def _filter_from_permission_object_qs(cls, queryset):
- return cls.objects.filter(cohort__in=queryset)
-
- @property
- def genome_build(self):
- return self.cohort.genome_build
-
- @property
- def data_archived(self) -> bool:
- return self.cohort.data_archived
-
def get_cohort_samples(self):
return [self.mother, self.father, self.proband, self.sibling]
- def get_samples(self):
- return Sample.objects.filter(cohortsample__in=self.get_cohort_samples()).order_by("pk")
-
def get_absolute_url(self):
return reverse('view_quad', kwargs={"pk": self.pk})
def get_listing_url(self):
return reverse('quads')
- @property
- def mother_details(self):
- affected = "affected" if self.mother_affected else "unaffected"
- return f"{self.mother} ({affected})"
-
- @property
- def father_details(self):
- affected = "affected" if self.father_affected else "unaffected"
- return f"{self.father} ({affected})"
-
@property
def sibling_details(self):
- affected = "affected" if self.sibling_affected else "unaffected"
- return f"{self.sibling} ({affected})"
-
- def __str__(self):
- return self.name or f"Quad {self.pk}"
+ return self._member_details(self.sibling, self.sibling_affected)
# This has to be in this file so we don't end up with circular references
diff --git a/snpdb/templatetags/model_helpers.py b/snpdb/templatetags/model_helpers.py
index b03cf5b81..1d0a2ad05 100644
--- a/snpdb/templatetags/model_helpers.py
+++ b/snpdb/templatetags/model_helpers.py
@@ -12,15 +12,10 @@
register = template.Library()
-@register.filter()
-def as_table(model):
- ret = ""
-
- rows = get_model_fields_and_formatted_values_tuples_list(model)
- for name, field in rows:
+def _display_rows(model):
+ """ (label, css data type, escaped value) per model field """
+ for name, field in get_model_fields_and_formatted_values_tuples_list(model):
data_type = field.__class__.__name__
- value = str(field)
-
name = name.replace('_', ' ')
if str(field).isdigit():
@@ -28,32 +23,22 @@ def as_table(model):
value = format(int(field), ',')
else:
value = escape(field)
-
- row = f'
{name}
{value}
'
- ret += row
- return ret
+ yield name, data_type, value
@register.filter()
-def as_p(model):
- ret = ""
-
- rows = get_model_fields_and_formatted_values_tuples_list(model)
- for name, field in rows:
- data_type = field.__class__.__name__
- value = str(field)
-
- name = name.replace('_', ' ')
+def as_table(model):
+ # Callers apply |safe at the template, so this stays a plain str
+ return "".join(f'
{name}
'
+ f'
{value}
'
+ for name, data_type, value in _display_rows(model))
- if str(field).isdigit():
- data_type = 'num'
- value = format(int(field), ',')
- else:
- value = escape(field)
- row = f'