From daaf948e7d9e9a18b7c370f0062a32877b703e6a Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 7 Jul 2026 14:29:11 +0100 Subject: [PATCH 001/108] feat(status): add status tracking utilities - Added r/utils/status.R to track R section completion statuses per-organism and record these to a JSON file for parsing in Python. --- src/comms/r/utils/status.R | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/comms/r/utils/status.R diff --git a/src/comms/r/utils/status.R b/src/comms/r/utils/status.R new file mode 100644 index 0000000..ebe31da --- /dev/null +++ b/src/comms/r/utils/status.R @@ -0,0 +1,48 @@ +#!/bin/R +# status.R: per-organism status tracking utilities + +library(jsonlite) + +# new_status_tracker: initialise an empty tracker for a section +new_status_tracker <- function(section) { + list(section = section, organisms = list(), reasons = list()) +} + +# record_ok: mark organism as having produced at least one successful output +record_ok <- function(tracker, organism) { + current <- tracker$organisms[[organism]] + if (is.null(current) || current == "skipped") { + tracker$organisms[[organism]] <- "ok" + } + tracker +} + +# record_skip: mark organism as skipped for this attempt (insufficient data) +record_skip <- function(tracker, organism, reason) { + if (is.null(tracker$organisms[[organism]])) { + tracker$organisms[[organism]] <- "skipped" + } + tracker$reasons[[organism]] <- c(tracker$reasons[[organism]], reason) + tracker +} + +# record_fail: mark organism as failed (unhandled error) +record_fail <- function(tracker, organism, reason) { + tracker$organisms[[organism]] <- "failed" + tracker$reasons[[organism]] <- c(tracker$reasons[[organism]], reason) + tracker +} + +# write_status: serialise a tracker to /_status.json +write_status <- function(tracker, output_dir) { + reasons_flat <- lapply(tracker$reasons, function(r) paste(r, collapse = "; ")) + payload <- list( + section = tracker$section, + organisms = tracker$organisms, + reasons = if (length(reasons_flat)) reasons_flat else setNames(list(), character(0)) + ) + writeLines( + toJSON(payload, auto_unbox = TRUE, pretty = TRUE), + file.path(output_dir, "_status.json") + ) +} \ No newline at end of file From ceb004ee72cbff45569ca20ff8383b6ceb781820 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 7 Jul 2026 14:43:32 +0100 Subject: [PATCH 002/108] feat(pca): implement status reporting - Modified r/sections/pca.R to implement status reporting per-organism, and to add a sample count guard to prevent attempts to analyse with too few samples for clustering. --- src/comms/r/sections/pca.R | 73 +++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/src/comms/r/sections/pca.R b/src/comms/r/sections/pca.R index 4517c75..5797811 100644 --- a/src/comms/r/sections/pca.R +++ b/src/comms/r/sections/pca.R @@ -20,6 +20,7 @@ script_dir <- local({ # Import utility functions source(file.path(script_dir, "..", "utils", "import.R")) +source(file.path(script_dir, "..", "utils", "status.R")) source(file.path(script_dir, "..", "utils", "theme.R")) # Load libraries @@ -28,6 +29,9 @@ library(ggfortify) library(ggrepel) library(svglite) +# Define variable for minimum samples required to run clustering +MIN_SAMPLES_FOR_PCA <- max(min_reps, 2) + # Import files ref_info <- loadRefInfo(ref_info_path) cont_info <- loadContInfo(cont_csv_path) @@ -38,36 +42,55 @@ dnsaf_cols <- colnames(results_wide)[startsWith(colnames(results_wide), "dNSAF_" sample_meta <- buildSampleMetadata(str_remove(dnsaf_cols, "dNSAF_"), samples) organisms <- unique(sample_meta$organism) +status <- new_status_tracker("pca") + for (org in organisms) { - org_meta <- filter(sample_meta, organism == org) - org_cols <- org_meta$dnsaf_col - label_map <- setNames(org_meta$sample_id, org_meta$dnsaf_col) + tryCatch({ + org_meta <- filter(sample_meta, organism == org) + org_cols <- org_meta$dnsaf_col + label_map <- setNames(org_meta$sample_id, org_meta$dnsaf_col) + + if (length(org_cols) < MIN_SAMPLES_FOR_PCA) { + reason <- sprintf("only %d sample(s) available (need >= %d for clustering)", length(org_cols), MIN_SAMPLES_FOR_PCA) + message(sprintf("PCA %s: %s - skipping", org, reason)) + status <<- record_skip(status, org, reason) + next + } - org_data <- results_wide %>% - filter(rowSums(select(., all_of(org_cols)) > 0) > 0) + org_data <- results_wide %>% + filter(rowSums(select(., all_of(org_cols)) > 0) > 0) - pca_mat <- org_data %>% - select(proteinId, all_of(org_cols)) %>% - column_to_rownames("proteinId") %>% - rename_with(~label_map[.]) %>% - t() + pca_mat <- org_data %>% + select(proteinId, all_of(org_cols)) %>% + column_to_rownames("proteinId") %>% + rename_with(~label_map[.]) %>% + t() - k <- max(2, min(length(unique(org_meta$fraction)), nrow(pca_mat) - 1)) - pca_data <- clara(pca_mat, k=k, metric="euclidean", stand=FALSE, samples=500, sampsize=nrow(pca_mat), pamLike=TRUE, correct.d=TRUE) + k <- max(2, min(length(unique(org_meta$fraction)), nrow(pca_mat) - 1)) + pca_data <- clara(pca_mat, k=k, metric="euclidean", stand=FALSE, samples=500, sampsize=nrow(pca_mat), pamLike=TRUE, correct.d=TRUE) - pca_plot <- autoplot(pca_data, frame=TRUE, frame.type="t", size=5) + - theme_comms() + - geom_text_repel(label=rownames(pca_mat), size=4, box.padding=0.5, point.padding=0.75, direction="both", force=15, max.overlaps=Inf) + - scale_color_manual(values=COMMS_COLOURS) + - scale_fill_manual(values=COMMS_COLOURS) + - labs(colour="Cluster", fill="Cluster", title=sprintf("PCA — %s", org)) - svglite(file.path(output_dir, sprintf("pca_%s.svg", org)), width=10, height=8) - print(pca_plot); dev.off() + pca_plot <- autoplot(pca_data, frame=TRUE, frame.type="t", size=5) + + theme_comms() + + geom_text_repel(label=rownames(pca_mat), size=4, box.padding=0.5, point.padding=0.75, direction="both", force=15, max.overlaps=Inf) + + scale_color_manual(values = COMMS_COLOURS) + + scale_fill_manual(values = COMMS_COLOURS) + + labs(colour="Cluster", fill="Cluster", title=sprintf("PCA — %s", org)) + svglite(file.path(output_dir, sprintf("pca_%s.svg", org)), width=10, height=8) + print(pca_plot); dev.off() - dist_mat <- dist(scale(pca_mat), method="euclidean") - hc <- hclust(dist_mat, method="average") - svglite(file.path(output_dir, sprintf("dendrogram_%s.svg", org)), width=10, height=6) - plot(hc, main=sprintf("Sample clustering — %s", org), xlab="", sub="", ylab="Distance", cex=0.9) - dev.off() + dist_mat <- dist(scale(pca_mat), method="euclidean") + hc <- hclust(dist_mat, method="average") + svglite(file.path(output_dir, sprintf("dendrogram_%s.svg", org)), width=10, height=6) + plot(hc, main=sprintf("Sample clustering — %s", org), xlab="", sub="", ylab="Distance", cex=0.9) + dev.off() + + status <<- record_ok(status, org) + }, error = function(e) { + while (dev.cur() != 1) dev.off() + message(sprintf("PCA %s: error — %s", org, conditionMessage(e))) + status <<- record_fail(status, org, conditionMessage(e)) + }) } + +write_status(status, output_dir) message("PCA section complete") \ No newline at end of file From 97967926370e61b2e901170310a48d5822b1ac58 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 7 Jul 2026 16:47:43 +0100 Subject: [PATCH 003/108] feat(da): implement status reporting - Modified r/sections/da.R to implement status reporting for skipping section and to ensure one fraction error does not fail the entire pipeline. --- src/comms/r/sections/da.R | 126 +++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/src/comms/r/sections/da.R b/src/comms/r/sections/da.R index 8adc634..aec234e 100644 --- a/src/comms/r/sections/da.R +++ b/src/comms/r/sections/da.R @@ -24,6 +24,7 @@ script_dir <- local({ source(file.path(script_dir, "..", "utils", "import.R")) source(file.path(script_dir, "..", "utils", "limma_da.R")) source(file.path(script_dir, "..", "utils", "normalise.R")) +source(file.path(script_dir, "..", "utils", "status.R")) source(file.path(script_dir, "..", "utils", "theme.R")) # Load libraries @@ -47,76 +48,91 @@ if (length(treatments) != 2) stop("DA section requires exactly two treatment lev # Initialise list for differentially abundant results da_results_all <- list() +status <- new_status_tracker("da") + for (org in organisms) { org_meta <- filter(sample_meta, organism == org) fractions <- unique(org_meta$fraction) for (frac in fractions) { - frac_meta <- filter(org_meta, fraction==frac) - frac_cols <- frac_meta$dnsaf_col - - frac_data <- results_wide %>% - select(proteinId, proteinAnnotation, all_of(frac_cols)) %>% - filter(rowSums(select(., all_of(frac_cols)) > 0) > 0) - - for (trt in treatments) { - trt_cols <- filter(frac_meta, treatment==trt)$dnsaf_col - frac_data[[paste0("n_", trt)]] <- rowSums(select(frac_data, all_of(trt_cols)) > 0) - } - frac_data <- filter(frac_data, if_any(starts_with("n_"), ~. >= min_reps)) - - if (nrow(frac_data) < 5) { - message(sprintf("DA %s %s: too few proteins after replicate filter (%d) — skipping", org, frac, nrow(frac_data))); next - } - - log_mat <- frac_data %>% - select(proteinId, all_of(frac_cols)) %>% - column_to_rownames("proteinId") %>% - as.matrix() %>% - logdNSAF() - treatment_vec <- frac_meta %>% - arrange(match(dnsaf_col, frac_cols)) %>% - pull(treatment) - - if (length(unique(treatment_vec)) < 2) { - message(sprintf("DA %s %s: fewer than 2 treatment levels in this fraction — skipping", org, frac)) - next - } - - da_res <- runLimmaDA(log_mat, treatment_vec) %>% - classifyDA(lfc_threshold, fdr_threshold) %>% - left_join(select(frac_data, proteinId, proteinAnnotation), by="proteinId") - - key <- paste(org, frac, sep="_") - da_results_all[[key]] <- da_res - - # Generate volcano plot - top_labels <- filter(da_res, Abundance != "Unchanged") %>% slice_min(adj_pval, n=20) - volcano <- ggplot(da_res, aes(x=log2FC, y=-log10(adj_pval), colour=Abundance)) + - geom_point(alpha=0.7, size=1.5) + - geom_hline(yintercept=-log10(fdr_threshold), linetype="dashed", colour="grey50") + - geom_vline(xintercept=c(-lfc_threshold, lfc_threshold), linetype="dashed", colour="grey50") + - geom_text_repel(data=top_labels, aes(label=proteinAnnotation), size=3, max.overlaps=15) + - scale_colour_manual(values=c("Increased"="#CC6677","Decreased"="#88CCEE","Unchanged"="grey70")) + - theme_comms() + - labs(title=sprintf("DA — %s %s (%s vs %s)", org, frac, treatments[2], treatments[1]), x=expression(log[2](FC)), y=expression(-log[10](adj.p))) - svglite(file.path(output_dir, sprintf("volcano_%s_%s.svg", frac, org)), width=10, height=7) - print(volcano); dev.off() + tryCatch({ + frac_meta <- filter(org_meta, fraction == frac) + frac_cols <- frac_meta$dnsaf_col + + frac_data <- results_wide %>% + select(proteinId, proteinAnnotation, all_of(frac_cols)) %>% + filter(rowSums(select(., all_of(frac_cols)) > 0) > 0) + + for (trt in treatments) { + trt_cols <- filter(frac_meta, treatment == trt)$dnsaf_col + frac_data[[paste0("n_", trt)]] <- rowSums(select(frac_data, all_of(trt_cols)) > 0) + } + frac_data <- filter(frac_data, if_any(starts_with("n_"), ~. >= min_reps)) + + if (nrow(frac_data) < 5) { + reason <- sprintf("too few proteins after replicate filter (%d) in fraction %s", nrow(frac_data), frac) + message(sprintf("DA %s %s: %s — skipping", org, frac, reason)) + status <<- record_skip(status, org, reason) + next + } + + log_mat <- frac_data %>% + select(proteinId, all_of(frac_cols)) %>% + column_to_rownames("proteinId") %>% + as.matrix() %>% + logdNSAF() + treatment_vec <- frac_meta %>% + arrange(match(dnsaf_col, frac_cols)) %>% + pull(treatment) + + if (length(unique(treatment_vec)) < 2) { + reason <- sprintf("fewer than 2 treatment levels in fraction %s", frac) + message(sprintf("DA %s %s: %s — skipping", org, frac, reason)) + status <<- record_skip(status, org, reason) + next + } + + da_res <- runLimmaDA(log_mat, treatment_vec) %>% + classifyDA(lfc_threshold, fdr_threshold) %>% + left_join(select(frac_data, proteinId, proteinAnnotation), by = "proteinId") + + key <- paste(org, frac, sep = "_") + da_results_all[[key]] <- da_res + + top_labels <- filter(da_res, Abundance != "Unchanged") %>% + slice_min(adj_pval, n=20) + volcano <- ggplot(da_res, aes(x=log2FC, y=-log10(adj_pval), colour=Abundance)) + + geom_point(alpha=0.7, size=1.5) + + geom_hline(yintercept=-log10(fdr_threshold), linetype="dashed", colour="grey50") + + geom_vline(xintercept=c(-lfc_threshold, lfc_threshold), linetype="dashed", colour="grey50") + + geom_text_repel(data=top_labels, aes(label=proteinAnnotation), size=3, max.overlaps=15) + + scale_colour_manual(values=c("Increased"="#CC6677", "Decreased"="#88CCEE", "Unchanged"="grey70")) + + theme_comms() + + labs(title=sprintf("DA — %s %s (%s vs %s)", org, frac, treatments[2], treatments[1]), x=expression(log[2](FC)), y=expression(-log[10](adj.p))) + svglite(file.path(output_dir, sprintf("volcano_%s_%s.svg", frac, org)), width=10, height=7) + print(volcano); dev.off() + + status <<- record_ok(status, org) + }, error = function(e) { + while (dev.cur() != 1) dev.off() + message(sprintf("DA %s %s: error — %s", org, frac, conditionMessage(e))) + status <<- record_fail(status, org, conditionMessage(e)) + }) } } +write_status(status, output_dir) + # Venn diagrams per organism for (org in organisms) { org_results <- da_results_all[str_starts(names(da_results_all), org)] - da_up_sets <- lapply(org_results, function(x) filter(x, Abundance == "Increased")$proteinId) + da_up_sets <- lapply(org_results, function(x) filter(x, Abundance == "Increased")$proteinId) da_down_sets <- lapply(org_results, function(x) filter(x, Abundance == "Decreased")$proteinId) names(da_up_sets) <- names(da_down_sets) <- str_remove(names(org_results), paste0(org, "_")) if (length(da_up_sets) >= 2) { - venn_up <- venn.diagram(da_up_sets, filename=NULL, disable.logging=TRUE, - category.names=names(da_up_sets)) + venn_up <- venn.diagram(da_up_sets, filename=NULL, disable.logging=TRUE, category.names=names(da_up_sets)) ggsave(file.path(output_dir, sprintf("venn_da_up_%s.svg", org)), venn_up) - venn_down <- venn.diagram(da_down_sets, filename=NULL, disable.logging=TRUE, - category.names=names(da_down_sets)) + venn_down <- venn.diagram(da_down_sets, filename=NULL, disable.logging=TRUE, category.names=names(da_down_sets)) ggsave(file.path(output_dir, sprintf("venn_da_down_%s.svg", org)), venn_down) } } From 75bc50dfaa59a69dfeefb71fcaa5a27df14224ec Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 7 Jul 2026 16:48:50 +0100 Subject: [PATCH 004/108] feat(qc): implement status reporting - Modified r/sections/qc.R to implement status reporting and add a sample count guard for generating upset plots and presence/absence heatmaps. --- src/comms/r/sections/qc.R | 141 +++++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 57 deletions(-) diff --git a/src/comms/r/sections/qc.R b/src/comms/r/sections/qc.R index 2c303ba..ab625d9 100644 --- a/src/comms/r/sections/qc.R +++ b/src/comms/r/sections/qc.R @@ -21,6 +21,7 @@ script_dir <- local({ # Import utility functions source(file.path(script_dir, "..", "utils", "import.R")) source(file.path(script_dir, "..", "utils", "normalise.R")) +source(file.path(script_dir, "..", "utils", "status.R")) source(file.path(script_dir, "..", "utils", "theme.R")) # Load libraries @@ -38,63 +39,89 @@ dnsaf_cols <- colnames(results_wide)[startsWith(colnames(results_wide), "dNSAF_" sample_meta <- buildSampleMetadata(str_remove(dnsaf_cols, "dNSAF_"), samples) organisms <- unique(sample_meta$organism) +status <- new_status_tracker("qc") + for (org in organisms) { - org_meta <- filter(sample_meta, organism == org) - org_cols <- org_meta$dnsaf_col - label_map <- setNames(org_meta$sample_id, org_meta$dnsaf_col) - - org_data <- results_wide %>% - filter(rowSums(select(., all_of(org_cols)) > 0) > 0) - - # Per-sample dNSAF density plot - dnsaf_long <- org_data %>% - select(proteinId, all_of(org_cols)) %>% - pivot_longer(-proteinId, names_to="Sample", values_to="dNSAF") %>% - filter(dNSAF > 0) %>% - mutate(log_dNSAF=log(dNSAF), Sample=label_map[Sample]) - density_plot <- ggplot(dnsaf_long, aes(x=log_dNSAF, colour=Sample)) + - geom_density() + theme_comms() + - labs(x="log(dNSAF)", y="Density", title=sprintf("Per-sample dNSAF distributions — %s", org)) + - theme(legend.position="bottom") - svglite(file.path(output_dir, sprintf("dnsaf_distributions_%s.svg", org)), width=10, height=6) - print(density_plot); dev.off() - - # Total spectral counts per sample - spec_counts <- bind_rows(lapply(org_cols, function(col) { - nm <- str_remove(col, "dNSAF_") - if (!nm %in% names(results_list)) return(NULL) - tibble(Sample=label_map[col], TotalSpectra=sum(results_list[[nm]]$`RAW`, na.rm=TRUE)) - })) %>% compact() %>% bind_rows() - counts_plot <- ggplot(spec_counts, aes(x=Sample, y=TotalSpectra)) + - geom_col(fill="#88CCEE") + theme_comms() + - theme(axis.text.x=element_text(angle=45, hjust=1)) + - labs(x=NULL, y="Total spectral counts", title=sprintf("Spectral counts per sample — %s", org)) - svglite(file.path(output_dir, sprintf("spectral_counts_per_sample_%s.svg", org)), width=10, height=5) - print(counts_plot); dev.off() - - # Missing-value upset plot - presence_matrix <- org_data %>% - select(all_of(org_cols)) %>% - mutate(across(everything(), ~as.integer(. > 0))) - colnames(presence_matrix) <- label_map[colnames(presence_matrix)] - svglite(file.path(output_dir, sprintf("missing_values_upset_%s.svg", org)), width=12, height=7) - upset(as.data.frame(presence_matrix), nsets=ncol(presence_matrix), order.by="freq", mainbar.y.label="Proteins", sets.x.label="Proteins detected") - dev.off() - - # Presence/absence heatmap - svglite(file.path(output_dir, sprintf("presence_absence_heatmap_%s.svg", org)), width=10, height=8) - pheatmap(as.matrix(presence_matrix), color=c("white", "#117733"), legend_breaks=c(0, 1), legend_labels=c("Absent", "Present"), main=sprintf("Protein presence/absence — %s", org), fontsize=10) - dev.off() - - # QC summary Excel - n_detected <- org_data %>% - summarise(across(all_of(org_cols), ~sum(. > 0))) %>% - pivot_longer(everything(), names_to="dnsaf_col", values_to="ProteinsDetected") %>% - mutate(Sample=label_map[dnsaf_col]) %>% - select(Sample, ProteinsDetected) - qc_summary <- left_join(spec_counts, n_detected, by="Sample") - wb <- wb_workbook() - wb$add_worksheet(org); wb$add_data(org, qc_summary) - wb_save(wb, file.path(output_dir, sprintf("qc_summary_%s.xlsx", org))) + tryCatch({ + org_meta <- filter(sample_meta, organism == org) + org_cols <- org_meta$dnsaf_col + label_map <- setNames(org_meta$sample_id, org_meta$dnsaf_col) + + org_data <- results_wide %>% + filter(rowSums(select(., all_of(org_cols)) > 0) > 0) + + if (nrow(org_data) == 0) { + reason <- "no proteins detected for this organism" + message(sprintf("QC %s: %s — skipping", org, reason)) + status <<- record_skip(status, org, reason) + next + } + + # Per-sample dNSAF density plot (works fine with a single sample) + dnsaf_long <- org_data %>% + select(proteinId, all_of(org_cols)) %>% + pivot_longer(-proteinId, names_to="Sample", values_to="dNSAF") %>% + filter(dNSAF > 0) %>% + mutate(log_dNSAF=log(dNSAF), Sample=label_map[Sample]) + density_plot <- ggplot(dnsaf_long, aes(x=log_dNSAF, colour=Sample)) + + geom_density() + theme_comms() + + labs(x="log(dNSAF)", y="Density", title=sprintf("Per-sample dNSAF distributions — %s", org)) + + theme(legend.position="bottom") + svglite(file.path(output_dir, sprintf("dnsaf_distributions_%s.svg", org)), width=10, height=6) + print(density_plot); dev.off() + + # Total spectral counts per sample (also fine with a single sample) + spec_counts <- bind_rows(lapply(org_cols, function(col) { + nm <- str_remove(col, "dNSAF_") + if (!nm %in% names(results_list)) return(NULL) + tibble(Sample=label_map[col], TotalSpectra=sum(results_list[[nm]]$`RAW`, na.rm=TRUE)) + })) %>% compact() %>% bind_rows() + counts_plot <- ggplot(spec_counts, aes(x=Sample, y=TotalSpectra)) + + geom_col(fill="#88CCEE") + theme_comms() + + theme(axis.text.x=element_text(angle=45, hjust=1)) + + labs(x=NULL, y="Total spectral counts", title=sprintf("Spectral counts per sample — %s", org)) + svglite(file.path(output_dir, sprintf("spectral_counts_per_sample_%s.svg", org)), width=10, height=5) + print(counts_plot); dev.off() + + # Missing-value upset plot and presence/absence heatmap need >= 2 samples to mean anything + if (length(org_cols) >= 2) { + presence_matrix <- org_data %>% + select(all_of(org_cols)) %>% + mutate(across(everything(), ~as.integer(. > 0))) + colnames(presence_matrix) <- label_map[colnames(presence_matrix)] + + svglite(file.path(output_dir, sprintf("missing_values_upset_%s.svg", org)), width = 12, height = 7) + upset(as.data.frame(presence_matrix), nsets = ncol(presence_matrix), order.by = "freq", + mainbar.y.label = "Proteins", sets.x.label = "Proteins detected") + dev.off() + + svglite(file.path(output_dir, sprintf("presence_absence_heatmap_%s.svg", org)), width = 10, height = 8) + pheatmap(as.matrix(presence_matrix), color = c("white", "#117733"), legend_breaks = c(0, 1), + legend_labels = c("Absent", "Present"), + main = sprintf("Protein presence/absence — %s", org), fontsize = 10) + dev.off() + } else { + message(sprintf("QC %s: only 1 sample — skipping upset plot and presence/absence heatmap", org)) + } + + # QC summary Excel + n_detected <- org_data %>% + summarise(across(all_of(org_cols), ~sum(. > 0))) %>% + pivot_longer(everything(), names_to = "dnsaf_col", values_to = "ProteinsDetected") %>% + mutate(Sample = label_map[dnsaf_col]) %>% + select(Sample, ProteinsDetected) + qc_summary <- left_join(spec_counts, n_detected, by = "Sample") + wb <- wb_workbook() + wb$add_worksheet(org); wb$add_data(org, qc_summary) + wb_save(wb, file.path(output_dir, sprintf("qc_summary_%s.xlsx", org))) + + status <<- record_ok(status, org) + }, error = function(e) { + while (dev.cur() != 1) dev.off() + message(sprintf("QC %s: error — %s", org, conditionMessage(e))) + status <<- record_fail(status, org, conditionMessage(e)) + }) } + +write_status(status, output_dir) message("QC section complete") \ No newline at end of file From f55007012f39d0dd4331ebc157332ff9ae0b855e Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 7 Jul 2026 16:52:44 +0100 Subject: [PATCH 005/108] feat(concordance): implement status reporting - Modified r/sections/concordance.R to implement status reporting for skipping section and to ensure one fraction error does not fail the entire pipeline. --- src/comms/r/sections/concordance.R | 125 +++++++++++++++++------------ 1 file changed, 72 insertions(+), 53 deletions(-) diff --git a/src/comms/r/sections/concordance.R b/src/comms/r/sections/concordance.R index 40dad61..828d61e 100644 --- a/src/comms/r/sections/concordance.R +++ b/src/comms/r/sections/concordance.R @@ -25,6 +25,7 @@ script_dir <- local({ source(file.path(script_dir, "..", "utils", "import.R")) source(file.path(script_dir, "..", "utils", "limma_da.R")) source(file.path(script_dir, "..", "utils", "normalise.R")) +source(file.path(script_dir, "..", "utils", "status.R")) source(file.path(script_dir, "..", "utils", "theme.R")) # Load libraries @@ -51,68 +52,86 @@ lfq_data <- loadLfqFiles(lfq_dir) concordance_stats <- list() organisms <- unique(sample_meta$organism) -concordance_stats <- list() +status <- new_status_tracker("concordance") for (org in organisms) { org_meta <- filter(sample_meta, organism == org) fractions <- unique(org_meta$fraction) for (frac in fractions) { - frac_meta <- filter(org_meta, fraction == frac) - frac_cols <- frac_meta$dnsaf_col - - frac_data <- results_wide %>% - select(proteinId, all_of(frac_cols)) %>% - filter(rowSums(select(., -proteinId) > 0) > 0) - log_mat_dnsaf <- frac_data %>% - column_to_rownames("proteinId") %>% - as.matrix() %>% - logdNSAF() - treatment_vec <- frac_meta %>% - arrange(match(dnsaf_col, frac_cols)) %>% - pull(treatment) - if (length(unique(treatment_vec)) < 2) { - message(sprintf("DA %s %s: fewer than 2 treatment levels in this fraction — skipping", org, frac)) - next - } - da_dnsaf <- runLimmaDA(log_mat_dnsaf, treatment_vec) %>% - classifyDA(lfc_threshold, fdr_threshold) %>% - select(proteinId, log2FC_dNSAF=log2FC, adj_pval_dNSAF=adj_pval, - Abundance_dNSAF=Abundance) - lfq_frac <- filter(lfq_data, Fraction==frac) - if (nrow(lfq_frac) == 0) { - message(sprintf("Concordance %s %s: no LFQ data found — skipping", org, frac)); next - } - - lfq_sample_cols <- setdiff(colnames(lfq_frac), c("proteinId", "Fraction")) - lfq_treatment_vec <- tibble(sample_id = lfq_sample_cols) %>% - inner_join(samples, by = "sample_id") %>% - pull(treatment) - - log_mat_lfq <- lfq_frac %>% - select(proteinId, all_of(lfq_sample_cols)) %>% - column_to_rownames("proteinId") %>% - as.matrix() %>% - log2() - da_lfq <- runLimmaDA(log_mat_lfq, lfq_treatment_vec) %>% - classifyDA(lfc_threshold, fdr_threshold) %>% - select(proteinId, log2FC_LFQ=log2FC, adj_pval_LFQ=adj_pval, Abundance_LFQ=Abundance) - combined <- inner_join(da_dnsaf, da_lfq, by="proteinId") - key <- paste(org, frac, sep = "_") - concordance_stats[[key]] <- combined - r_val <- cor(combined$log2FC_dNSAF, combined$log2FC_LFQ, use = "complete.obs") - scatter <- ggplot(combined, aes(x=log2FC_dNSAF, y=log2FC_LFQ)) + - geom_point(aes(colour=Abundance_dNSAF), alpha=0.6) + - geom_smooth(method="lm", se=FALSE, colour="black", linewidth=0.5) + - scale_colour_manual(values=c("Increased"="#CC6677","Decreased"="#88CCEE","Unchanged"="grey70")) + - theme_comms() + - labs(title=sprintf("LFQ vs dNSAF concordance — %s %s", org, frac), x=expression(log[2](FC)~dNSAF), y=expression(log[2](FC)~LFQ), colour="DA (dNSAF)") + - annotate("text", x=Inf, y=-Inf, hjust=1.1, vjust=-0.5, size=3.5, label=sprintf("r = %.2f (n=%d proteins)", r_val, nrow(combined))) - svglite(file.path(output_dir, sprintf("lfq_vs_dnsaf_%s_%s.svg", frac, org)), width=8, height=7) - print(scatter); dev.off() + tryCatch({ + frac_meta <- filter(org_meta, fraction == frac) + frac_cols <- frac_meta$dnsaf_col + + frac_data <- results_wide %>% + select(proteinId, all_of(frac_cols)) %>% + filter(rowSums(select(., -proteinId) > 0) > 0) + + log_mat_dnsaf <- frac_data %>% + column_to_rownames("proteinId") %>% + as.matrix() %>% + logdNSAF() + treatment_vec <- frac_meta %>% + arrange(match(dnsaf_col, frac_cols)) %>% + pull(treatment) + if (length(unique(treatment_vec)) < 2) { + reason <- sprintf("fewer than 2 treatment levels in fraction %s", frac) + message(sprintf("Concordance %s %s: %s — skipping", org, frac, reason)) + status <<- record_skip(status, org, reason) + next + } + da_dnsaf <- runLimmaDA(log_mat_dnsaf, treatment_vec) %>% + classifyDA(lfc_threshold, fdr_threshold) %>% + select(proteinId, log2FC_dNSAF=log2FC, adj_pval_dNSAF=adj_pval, Abundance_dNSAF=Abundance) + + lfq_frac <- filter(lfq_data, Fraction == frac) + if (nrow(lfq_frac) == 0) { + reason <- sprintf("no LFQ data for fraction %s", frac) + message(sprintf("Concordance %s %s: %s — skipping", org, frac, reason)) + status <<- record_skip(status, org, reason) + next + } + + lfq_sample_cols <- setdiff(colnames(lfq_frac), c("proteinId", "Fraction")) + lfq_treatment_vec <- tibble(sample_id=lfq_sample_cols) %>% + inner_join(samples, by="sample_id") %>% + pull(treatment) + + log_mat_lfq <- lfq_frac %>% + select(proteinId, all_of(lfq_sample_cols)) %>% + column_to_rownames("proteinId") %>% + as.matrix() %>% + log2() + da_lfq <- runLimmaDA(log_mat_lfq, lfq_treatment_vec) %>% + classifyDA(lfc_threshold, fdr_threshold) %>% + select(proteinId, log2FC_LFQ=log2FC, adj_pval_LFQ=adj_pval, Abundance_LFQ=Abundance) + + combined <- inner_join(da_dnsaf, da_lfq, by="proteinId") + key <- paste(org, frac, sep = "_") + concordance_stats[[key]] <- combined + + r_val <- cor(combined$log2FC_dNSAF, combined$log2FC_LFQ, use="complete.obs") + scatter <- ggplot(combined, aes(x=log2FC_dNSAF, y=log2FC_LFQ)) + + geom_point(aes(colour=Abundance_dNSAF), alpha=0.6) + + geom_smooth(method="lm", se=FALSE, colour="black", linewidth=0.5) + + scale_colour_manual(values=c("Increased"="#CC6677", "Decreased"="#88CCEE", "Unchanged"="grey70")) + + theme_comms() + + labs(title=sprintf("LFQ vs dNSAF concordance — %s %s", org, frac), x=expression(log[2](FC)~dNSAF), y=expression(log[2](FC)~LFQ), colour="DA (dNSAF)") + + annotate("text", x=Inf, y =-Inf, hjust=1.1, vjust=-0.5, size=3.5, label=sprintf("r = %.2f (n=%d proteins)", r_val, nrow(combined))) + svglite(file.path(output_dir, sprintf("lfq_vs_dnsaf_%s_%s.svg", frac, org)), width=8, height=7) + print(scatter); dev.off() + + status <<- record_ok(status, org) + }, error = function(e) { + while (dev.cur() != 1) dev.off() + message(sprintf("Concordance %s %s: error — %s", org, frac, conditionMessage(e))) + status <<- record_fail(status, org, conditionMessage(e)) + }) } } +write_status(status, output_dir) + # Export .xlsx spreadsheet wb <- wb_workbook() for (key in names(concordance_stats)) { From 5c69610a74780a79db89e8710deed70acc66b9dd Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 7 Jul 2026 17:01:15 +0100 Subject: [PATCH 006/108] feat(ev-markers): implement status reporting - Modified r/sections/aux/ev-markers.R to implement status reporting. --- src/comms/r/sections/aux/ev-markers.R | 184 ++++++++++++++------------ 1 file changed, 99 insertions(+), 85 deletions(-) diff --git a/src/comms/r/sections/aux/ev-markers.R b/src/comms/r/sections/aux/ev-markers.R index b2d7c82..0afb7be 100644 --- a/src/comms/r/sections/aux/ev-markers.R +++ b/src/comms/r/sections/aux/ev-markers.R @@ -21,6 +21,7 @@ script_dir <- local({ # Import utility functions source(file.path(script_dir, "..", "..", "utils", "import.R")) source(file.path(script_dir, "..", "..", "utils", "normalise.R")) +source(file.path(script_dir, "..", "..", "utils", "state.R")) source(file.path(script_dir, "..", "..", "utils", "theme.R")) # Load libraries @@ -56,94 +57,107 @@ categorise_marker <- function(annotation) { organisms <- unique(sample_meta$organism) all_marker_tables <- list() +status <- new_status_tracker("ev-markers") for (org in organisms) { - org_meta <- filter(sample_meta, organism == org) - org_cols <- org_meta$dnsaf_col - # EV markers are only meaningful for the primary organism: skip any organism whose dNSAF columns contain no primary-organism proteins - primary_check <- results_wide %>% - filter(startsWith(proteinId, organism_prefix)) %>% - filter(rowSums(select(., all_of(org_cols)) > 0) > 0) - - if (nrow(primary_check) == 0) { - message(sprintf("EV markers: no primary organism proteins in %s columns — skipping", org)) - next - } - - fractions <- unique(org_meta$fraction) - - org_data <- results_wide %>% - filter(startsWith(proteinId, organism_prefix)) %>% - filter(rowSums(select(., all_of(org_cols)) > 0) > 0) - - # Per-fraction mean dNSAF - for (frac in fractions) { - cols <- intersect(filter(org_meta, fraction == frac)$dnsaf_col, colnames(org_data)) - org_data[[paste0("avg_", frac)]] <- - if (length(cols) > 0) rowMeans(select(org_data, all_of(cols)), na.rm=TRUE) else NA_real_ - } - - marker_table <- org_data %>% - rowwise() %>% - mutate(MISEVCategory=categorise_marker(proteinAnnotation)) %>% - ungroup() %>% - filter(!is.na(MISEVCategory)) %>% - mutate(MISEVCategory=factor(MISEVCategory, levels=MISEV_LEVELS)) %>% - arrange(MISEVCategory) - - if (nrow(marker_table) == 0) { - message(sprintf("EV markers %s: no marker proteins found — skipping", org)); next - } - - # Enrichment ratios - ev_frac <- fractions[str_detect(tolower(fractions), "ev")][1] - wcl_frac <- fractions[str_detect(tolower(fractions), "wcl")][1] - awf_frac <- fractions[str_detect(tolower(fractions), "awf|cr")][1] - if (!is.na(ev_frac) && !is.na(wcl_frac)) - marker_table <- mutate(marker_table, log2_EV_vs_WCL=log2((.data[[paste0("avg_", ev_frac)]] + 1e-10) / (.data[[paste0("avg_", wcl_frac)]] + 1e-10)) - if (!is.na(ev_frac) && !is.na(awf_frac)) - marker_table <- mutate(marker_table, log2_EV_vs_AWF=log2((.data[[paste0("avg_", ev_frac)]] + 1e-10) / (.data[[paste0("avg_", awf_frac)]] + 1e-10))) - - # Per-protein heatmap with category gaps - avg_cols <- intersect(paste0("avg_", fractions), colnames(marker_table)) - heatmap_mat <- marker_table %>% - select(proteinAnnotation, all_of(avg_cols)) %>% - column_to_rownames("proteinId") %>% - as.matrix() %>% - logdNSAF() - colnames(heatmap_mat) <- str_remove(colnames(heatmap_mat), "avg_") - ann_row <- data.frame(Category=as.character(marker_table$MISEVCategory), row.names=marker_table$proteinAnnotation) - gaps_row <- marker_table %>% - count(MISEVCategory) %>% - arrange(MISEVCategory) %>% - pull(n) %>% - cumsum() %>% - head(-1) - svglite(file.path(output_dir, sprintf("marker_heatmap_%s.svg", org)), width=12, height=max(6, nrow(heatmap_mat) * 0.35)) - pheatmap(heatmap_mat, annotation_row=ann_row, gaps_row=gaps_row, cluster_rows=FALSE, colour=colorRampPalette(c("#88CCEE", "white", "#CC6677"))(50), main=sprintf("MISEV2023 markers — log(dNSAF) — %s", org), fontsize_row=8, fontsize_col=10, border_colour=NA) - dev.off() - - # Aggregated category heatmap (3 × fraction×treatment) - agg_mat <- marker_table %>% - select(MISEVCategory, all_of(org_cols)) %>% - pivot_longer(-MISEVCategory, names_to="dnsaf_col", values_to="dNSAF") %>% - left_join(select(org_meta, dnsaf_col, fraction, treatment), by="dnsaf_col") %>% - mutate(log_dNSAF=logdNSAF(dNSAF)) %>% - group_by(MISEVCategory, fraction, treatment) %>% - summarise(mean_log_dNSAF=mean(log_dNSAF, na.rm=TRUE), .groups="drop") %>% - mutate(col_label=paste(fraction, treatment, sep="_")) %>% - select(MISEVCategory, col_label, mean_log_dNSAF) %>% - pivot_wider(names_from=col_label, values_from=mean_log_dNSAF) %>% - arrange(MISEVCategory) %>% - column_to_rownames("MISEVCategory") %>% - as.matrix() - - svglite(file.path(output_dir, sprintf("marker_category_heatmap_%s.svg", org)), width=8, height=4) - pheatmap(agg_mat, cluster_rows=FALSE, cluster_cols=FALSE, colour=colorRampPalette(c("#88CCEE", "white", "#CC6677"))(50), main=sprintf("Mean log(dNSAF) by MISEV category — %s", org), fontsize=10, border_colour=NA) - dev.off() - - all_marker_tables[[org]] <- marker_table + tryCatch({ + org_meta <- filter(sample_meta, organism == org) + org_cols <- org_meta$dnsaf_col + + primary_check <- results_wide %>% + filter(startsWith(proteinId, organism_prefix)) %>% + filter(rowSums(select(., all_of(org_cols)) > 0) > 0) + + if (nrow(primary_check) == 0) { + reason <- "no primary-organism proteins detected in this organism's samples" + message(sprintf("EV markers %s: %s — skipping", org, reason)) + status <<- record_skip(status, org, reason) + next + } + + fractions <- unique(org_meta$fraction) + org_data <- results_wide %>% + filter(startsWith(proteinId, organism_prefix)) %>% + filter(rowSums(select(., all_of(org_cols)) > 0) > 0) + + for (frac in fractions) { + cols <- intersect(filter(org_meta, fraction == frac)$dnsaf_col, colnames(org_data)) + org_data[[paste0("avg_", frac)]] <- + if (length(cols) > 0) rowMeans(select(org_data, all_of(cols)), na.rm = TRUE) else NA_real_ + } + + marker_table <- org_data %>% + rowwise() %>% + mutate(MISEVCategory=categorise_marker(proteinAnnotation)) %>% + ungroup() %>% + filter(!is.na(MISEVCategory)) %>% + mutate(MISEVCategory=factor(MISEVCategory, levels=MISEV_LEVELS)) %>% + arrange(MISEVCategory) + + if (nrow(marker_table) == 0) { + reason <- "no marker proteins found" + message(sprintf("EV markers %s: %s — skipping", org, reason)) + status <<- record_skip(status, org, reason) + next + } + + ev_frac <- fractions[str_detect(tolower(fractions), "ev")][1] + wcl_frac <- fractions[str_detect(tolower(fractions), "wcl")][1] + awf_frac <- fractions[str_detect(tolower(fractions), "awf|cr")][1] + if (!is.na(ev_frac) && !is.na(wcl_frac)) + marker_table <- mutate(marker_table, log2_EV_vs_WCL=log2((.data[[paste0("avg_", ev_frac)]] + 1e-10) / (.data[[paste0("avg_", wcl_frac)]] + 1e-10))) + if (!is.na(ev_frac) && !is.na(awf_frac)) + marker_table <- mutate(marker_table, log2_EV_vs_AWF=log2((.data[[paste0("avg_", ev_frac)]] + 1e-10) / (.data[[paste0("avg_", awf_frac)]] + 1e-10))) + + avg_cols <- intersect(paste0("avg_", fractions), colnames(marker_table)) + heatmap_mat <- marker_table %>% + select(proteinAnnotation, all_of(avg_cols)) %>% + column_to_rownames("proteinId") %>% + as.matrix() %>% + logdNSAF() + colnames(heatmap_mat) <- str_remove(colnames(heatmap_mat), "avg_") + + ann_row <- data.frame(Category=as.character(marker_table$MISEVCategory), row.names=marker_table$proteinAnnotation) + gaps_row <- marker_table %>% + count(MISEVCategory) %>% + arrange(MISEVCategory) %>% + pull(n) %>% + cumsum() %>% + head(-1) + + svglite(file.path(output_dir, sprintf("marker_heatmap_%s.svg", org)), width=12, height=max(6, nrow(heatmap_mat) * 0.35)) + pheatmap(heatmap_mat, annotation_row=ann_row, gaps_row=gaps_row, cluster_rows=FALSE, colour=colorRampPalette(c("#88CCEE", "white", "#CC6677"))(50), main=sprintf("MISEV2023 markers — log(dNSAF) — %s", org), fontsize_row=8, fontsize_col=10, border_colour=NA) + dev.off() + + agg_mat <- marker_table %>% + select(MISEVCategory, all_of(org_cols)) %>% + pivot_longer(-MISEVCategory, names_to="dnsaf_col", values_to="dNSAF") %>% + left_join(select(org_meta, dnsaf_col, fraction, treatment), by="dnsaf_col") %>% + mutate(log_dNSAF=logdNSAF(dNSAF)) %>% + group_by(MISEVCategory, fraction, treatment) %>% + summarise(mean_log_dNSAF=mean(log_dNSAF, na.rm=TRUE), .groups="drop") %>% + mutate(col_label=paste(fraction, treatment, sep="_")) %>% + select(MISEVCategory, col_label, mean_log_dNSAF) %>% + pivot_wider(names_from=col_label, values_from=mean_log_dNSAF) %>% + arrange(MISEVCategory) %>% + column_to_rownames("MISEVCategory") %>% + as.matrix() + + svglite(file.path(output_dir, sprintf("marker_category_heatmap_%s.svg", org)), width=8, height=4) + pheatmap(agg_mat, cluster_rows=FALSE, cluster_cols=FALSE, colour=colorRampPalette(c("#88CCEE", "white", "#CC6677"))(50), main=sprintf("Mean log(dNSAF) by MISEV category — %s", org), fontsize=10, border_colour=NA) + dev.off() + + all_marker_tables[[org]] <- marker_table + status <<- record_ok(status, org) + }, error = function(e) { + while (dev.cur() != 1) dev.off() + message(sprintf("EV markers %s: error — %s", org, conditionMessage(e))) + status <<- record_fail(status, org, conditionMessage(e)) + }) } + +write_status(status, output_dir) + # Export .xlsx — one sheet per organism wb <- wb_workbook() for (org in names(all_marker_tables)) { From 61f781475ed790f80a15b71e21ce9ff443d112db Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 09:27:04 +0100 Subject: [PATCH 007/108] feat(report): rebuild logging logic to use status - Modified commands/report.py to rebuild logging logic to use new status reports, allowing more granular reporting of results. --- src/comms/commands/report.py | 118 ++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 23 deletions(-) diff --git a/src/comms/commands/report.py b/src/comms/commands/report.py index 05b2952..0b4d0de 100644 --- a/src/comms/commands/report.py +++ b/src/comms/commands/report.py @@ -3,7 +3,7 @@ ''' # -- Import external dependencies -import shutil, subprocess, sys +import json, shutil, subprocess, sys from datetime import datetime from importlib.resources import files as pkg_files from pathlib import Path @@ -18,24 +18,64 @@ # -- Initialise Rich console console = Console() -# -- Define helper dictionary matching sections to R scripts and whether they require LFQ data -_SECTIONS: dict[str, tuple[str, bool]] = { - # Core sections - 'qc': ('qc.R', False), - 'pca': ('pca.R', False), - 'da': ('da.R', False), - 'secondary-species': ('secondary-species.R', False), - 'concordance': ('concordance.R', True), +# -- Define helper dictionary matching sections to R scripts, LFQ requirement, and whether the section reports per-organism status +_SECTIONS: dict[str, tuple[str, bool, bool]] = { + # Core sections: (script, needs_lfq, per_organism) + 'qc': ('qc.R', False, True), + 'pca': ('pca.R', False, True), + 'da': ('da.R', False, True), + 'secondary-species': ('secondary-species.R', False, False), + 'concordance': ('concordance.R', True, True), # Auxiliary sections - 'ev-markers': ('aux/ev-markers.R', False), + 'ev-markers': ('aux/ev-markers.R', False, True), } +# -- Define helper dictionary for potential per-organism section statuses +_ORGANISM_STATUSES = {'ok', 'skipped', 'failed'} + # -- _resolve_r_script: returns Path to R script def _resolve_r_script(script_name: str) -> Path: '''Locate R script based on provided script name''' return pkg_files('comms').joinpath(f'r/sections/{script_name}') -# -- _run_r_script: returns boolean indicating if command was run successfully +# -- _read_status: returns tuple of dicts (containing organisms, reasons) parsed from a section's _status.json +def _read_status(output_subdir: Path) -> tuple[dict[str, str], dict[str, str]]: + status_path = output_subdir / '_status.json' + if not status_path.exists(): + return {}, {} + try: + payload = json.loads(status_path.read_text()) + except Exception as e: + logMsg.debug(f'Could not parse {status_path}: {e}') + return {}, {} + organisms = {k: v for k, v in payload.get('organisms', {}).items() if v in _ORGANISM_STATUSES} + reasons = dict(payload.get('reasons', {})) + return organisms, reasons + +# -- _section_status: returns string (either 'succeeded', 'partial', 'failed' or 'skipped') +def _section_status(proc_ok: bool, organisms: dict[str, str]) -> str: + if not organisms: + # No structured status available (legacy/non-organism script, or crash before writing status) + return 'failed' if not proc_ok else 'skipped' + ok = sum(1 for s in organisms.values() if s == 'ok') + failed = sum(1 for s in organisms.values() if s == 'failed') + if failed == 0: + return 'succeeded' if ok > 0 else 'skipped' + return 'partial' if ok > 0 else 'failed' + +# -- _log_organism_outcomes: returns None but outputs logging messages +def _log_organism_outcomes(section: str, organisms: dict[str, str], reasons: dict[str, str]) -> None: + for org, status in organisms.items(): + reason = reasons.get(org) + suffix = f' ({reason})' if reason else '' + if status == 'ok': + logMsg.progress(f'{section} — {org}: succeeded') + elif status == 'skipped': + logMsg.progress(f'{section} — {org}: skipped{suffix}') + else: + logMsg.progress(f'{section} — {org}: failed{suffix}') + +# -- _run_r_section: returns boolean indicating if the R process itself exited cleanly def _run_r_section( section: str, script_name: str, @@ -56,7 +96,12 @@ def _run_r_section( return True # -- _write_index: return none, but write index -def _write_index(output_dir: Path, params: dict, results: dict[str, bool]) -> None: +def _write_index( + output_dir: Path, + params: dict, + section_status: dict[str, str], + organism_results: dict[str, dict[str, str]], +) -> None: lines = [ '# comms report', f'\nGenerated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}', @@ -65,9 +110,16 @@ def _write_index(output_dir: Path, params: dict, results: dict[str, bool]) -> No for k,v in params.items(): lines.append(f'- **{k}**: `{v}`') lines.append(f'\n## Sections\n') - for sec, ok in results.items(): - status = '✓ SUCCEEDED' if ok else '✗ FAILED' - lines.append(f'- {sec}: {status}') + status_glyphs = { + 'succeeded': '✓ SUCCEEDED', + 'partial': '◐ PARTIAL', + 'failed': '✗ FAILED', + 'skipped': '- SKIPPED', + } + for sec, status in section_status.items(): + lines.append(f'- {sec}: {status_glyphs[status]}') + for org, org_status in organism_results.get(sec, {}).items(): + lines.append(f' - {org}: {org_status}') (output_dir / 'index.md').write_text('\n'.join(lines)) # -- run_report: return None, but run report section R scripts and output script @@ -147,22 +199,30 @@ def run_report( organism_prefix, str(min_reps), ] - results: dict[str, bool] = {} + section_status: dict[str, str] = {} + organism_results: dict[str, dict[str, str]] = {} for sec in sections: logMsg.progress(f'Running section: {sec}') - script, needs_lfq = _SECTIONS[sec] + script, needs_lfq, per_organism = _SECTIONS[sec] extra: list[str] = [] if sec == 'da': extra = [str(lfc_threshold), str(fdr_threshold)] elif sec == 'concordance': extra = [str(lfq_dir), str(lfc_threshold), str(fdr_threshold)] - results[sec] = _run_r_section( + output_subdir = output_dir / sec.replace('-', '_') + proc_ok = _run_r_section( section = sec, script_name=script, - output_subdir=output_dir / sec.replace('-', '_'), + output_subdir=output_subdir, positional_args=common_args+extra, rscript=rscript, ) + organisms, reasons = _read_status(output_subdir) if per_organism else ({}, {}) + section_status[sec] = _section_status(proc_ok, organisms) + organism_results[sec] = organisms + if organisms: + _log_organism_outcomes(sec, organisms, reasons) + _write_index( output_dir, { @@ -174,8 +234,20 @@ def run_report( 'lfc_threshold': lfc_threshold, 'fdr_threshold': fdr_threshold, }, - results) - n_ok = sum(results.values()) - n_fail = len(results) - n_ok - logMsg.info(f'Report complete: {n_ok} succeeded, {n_fail} failed') + section_status, + organism_results, + ) + + n_succeeded = sum(1 for s in section_status.values() if s == 'succeeded') + n_partial = sum(1 for s in section_status.values() if s == 'partial') + n_failed = sum(1 for s in section_status.values() if s == 'failed') + n_skipped = sum(1 for s in section_status.values() if s == 'skipped') + parts = [f'{n_succeeded} succeeded'] + if n_partial: + parts.append(f'{n_partial} partial (at least one organism failed)') + if n_failed: + parts.append(f'{n_failed} failed') + if n_skipped: + parts.append(f'{n_skipped} skipped (no organism had sufficient data)') + logMsg.info(f'Report complete: {", ".join(parts)}') logMsg.debug(f'Finished command: report') \ No newline at end of file From 7b7dd27e141b537e37e5958c8d169db3b95cb84d Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 09:36:18 +0100 Subject: [PATCH 008/108] test(report): update report unit tests - Modified tests/unit/test_report.py to update index writing test to use new function signature. --- tests/unit/test_report.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index b8c779c..007b228 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -28,25 +28,25 @@ def test_aux_script_resolved(self): # -- Define tests for writing index class TestWriteIndex: def test_creates_index_file(self, tmp_path): - _write_index(tmp_path, {'quantify_dir': '/tmp/q'}, {'qc': True, 'da': False}) + _write_index(tmp_path, {'quantify_dir': '/tmp/q'}, {'qc': 'succeeded', 'da': 'failed'}, {'qc': {'org1': 'succeeded', 'org2': 'failed'}}) assert (tmp_path / 'index.md').exists() def test_index_contains_section_names(self, tmp_path): - _write_index(tmp_path, {}, {'qc': True, 'da': False}) + _write_index(tmp_path, {}, {'qc': 'succeeded', 'da': 'failed'}, {'qc': {'org1': 'succeeded', 'org2': 'failed'}}) content = (tmp_path / 'index.md').read_text() assert 'qc' in content assert 'da' in content def test_failed_section_marked_with_failed(self, tmp_path): - _write_index(tmp_path, {}, {'qc': False}) + _write_index(tmp_path, {}, {'qc': 'failed'}, {'qc': {'org1': 'failed', 'org2': 'failed'}}) assert 'FAILED' in (tmp_path / 'index.md').read_text() def test_passed_section_marked_with_checkmark(self, tmp_path): - _write_index(tmp_path, {}, {'qc': True}) + _write_index(tmp_path, {}, {'qc': 'succeeded'}, {'qc': {'org1': 'succeeded', 'org2': 'succeeded'}}) assert '✓' in (tmp_path / 'index.md').read_text() def test_parameters_included_in_index(self, tmp_path): - _write_index(tmp_path, {'organism_prefix': 'Mtrun'}, {}) + _write_index(tmp_path, {'organism_prefix': 'Mtrun'}, {}, {}) assert 'organism_prefix' in (tmp_path / 'index.md').read_text() # -- Define shared fixtures From 58b9dc2540fa8a867a27fffea71667028e046947 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 09:53:04 +0100 Subject: [PATCH 009/108] feat(R): add R package dependencies file - Added r/dependencies.R to list all required R package dependencies for running comms report sections. --- src/comms/r/dependencies.R | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/comms/r/dependencies.R diff --git a/src/comms/r/dependencies.R b/src/comms/r/dependencies.R new file mode 100644 index 0000000..d00f2a0 --- /dev/null +++ b/src/comms/r/dependencies.R @@ -0,0 +1,7 @@ +#!/bin/R +# dependencies.R: list of all R packages required by comms report command + +R_DEPENDENCIES <- list( + cran = c("tidyverse", "openxlsx2", "svglite", "ggrepel", "ggfortify", "cluster", "UpSetR", "pheatmap", "VennDiagram", "iq"), + bioc = c("limma") +) \ No newline at end of file From f6ac310a14ced6e86884101fc2c5f534f024b09b Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 09:54:07 +0100 Subject: [PATCH 010/108] fix(R): add jsonlite dependency - Modified r/dependencies.R to add jsonlite dependency. --- src/comms/r/dependencies.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/r/dependencies.R b/src/comms/r/dependencies.R index d00f2a0..a6f0cc3 100644 --- a/src/comms/r/dependencies.R +++ b/src/comms/r/dependencies.R @@ -2,6 +2,6 @@ # dependencies.R: list of all R packages required by comms report command R_DEPENDENCIES <- list( - cran = c("tidyverse", "openxlsx2", "svglite", "ggrepel", "ggfortify", "cluster", "UpSetR", "pheatmap", "VennDiagram", "iq"), + cran = c("tidyverse", "openxlsx2", "svglite", "ggrepel", "ggfortify", "cluster", "UpSetR", "pheatmap", "VennDiagram", "iq", "jsonlite"), bioc = c("limma") ) \ No newline at end of file From 243f6eabbc66ff3b367d63e2f244fb074923a6dc Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 09:57:24 +0100 Subject: [PATCH 011/108] refactor(R): move dependency-related scripts - Moved r/dependencies.R and r/install_deps.R to r/deps/... to contain dependency-related scripts in single subdirectory. --- src/comms/r/{ => deps}/dependencies.R | 0 src/comms/r/{ => deps}/install_deps.R | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/comms/r/{ => deps}/dependencies.R (100%) rename src/comms/r/{ => deps}/install_deps.R (100%) diff --git a/src/comms/r/dependencies.R b/src/comms/r/deps/dependencies.R similarity index 100% rename from src/comms/r/dependencies.R rename to src/comms/r/deps/dependencies.R diff --git a/src/comms/r/install_deps.R b/src/comms/r/deps/install_deps.R similarity index 100% rename from src/comms/r/install_deps.R rename to src/comms/r/deps/install_deps.R From 116ab978e9663ecfd55e8c915d4b3618e737ac7b Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 09:59:27 +0100 Subject: [PATCH 012/108] feat(R): add dependency check script - Added r/deps/check_deps.R to check if required dependencies are installed. --- src/comms/r/deps/check_deps.R | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/comms/r/deps/check_deps.R diff --git a/src/comms/r/deps/check_deps.R b/src/comms/r/deps/check_deps.R new file mode 100644 index 0000000..7a77f70 --- /dev/null +++ b/src/comms/r/deps/check_deps.R @@ -0,0 +1,22 @@ +#!/bin/R +# check_deps.R: report which R dependencies are installed (as JSON to stdout) + +script_dir <- local({ + args <- commandArgs(trailingOnly = FALSE) + script <- grep("^--file=", args, value = TRUE) + dirname(normalizePath(sub("^--file=", "", script))) +}) +source(file.path(script_dir, "dependencies.R")) + +all_packages <- c(R_DEPENDENCIES$cran, R_DEPENDENCIES$bioc) +installed <- character(0) +missing <- character(0) +for (pkg in all_packages) { + ok <- tryCatch(requireNamespace(pkg, quietly = TRUE), error = function(e) FALSE) + if (ok) installed <- c(installed, pkg) else missing <- c(missing, pkg) +} +cat(sprintf( + '{"installed":[%s],"missing":[%s]}', + paste(sprintf('"%s"', installed), collapse = ","), + paste(sprintf('"%s"', missing), collapse = ",") +)) \ No newline at end of file From 99104fccbf649fff6a6b9de94d2129d5d4eb5293 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 10:00:30 +0100 Subject: [PATCH 013/108] feat(R): update dependency installation script - Modified r/deps/install_deps.R to only install missing dependencies. --- src/comms/r/deps/install_deps.R | 40 ++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/comms/r/deps/install_deps.R b/src/comms/r/deps/install_deps.R index 492ed15..1deaed9 100644 --- a/src/comms/r/deps/install_deps.R +++ b/src/comms/r/deps/install_deps.R @@ -1,9 +1,37 @@ #!/bin/R -# install_deps.R: install all required R dependencies for comms report command +# install_deps.R: install R dependencies for comms report command (only installing missing packages) -cran_packages <- c("tidyverse", "openxlsx2", "svglite", "ggrepel", "ggfortify", "cluster", "UpSetR", "pheatmap", "VennDiagram", "iq") -bioc_packages <- c("limma") +script_dir <- local({ + args <- commandArgs(trailingOnly = FALSE) + script <- grep("^--file=", args, value = TRUE) + dirname(normalizePath(sub("^--file=", "", script))) +}) +source(file.path(script_dir, "dependencies.R")) -install.packages(cran_packages, repos = "https://cloud.r-project.org") -if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager", repos = "https://cloud.r-project.org") -BiocManager::install(bioc_packages, ask = FALSE, update = FALSE) \ No newline at end of file +is_missing <- function(pkg) !requireNamespace(pkg, quietly = TRUE) + +missing_cran <- Filter(is_missing, R_DEPENDENCIES$cran) +missing_bioc <- Filter(is_missing, R_DEPENDENCIES$bioc) + +if (length(missing_cran) == 0 && length(missing_bioc) == 0) { + message("All R report dependencies are already installed.") + quit(status = 0) +} + +if (length(missing_cran) > 0) { + message(sprintf("Installing %d CRAN package(s): %s", length(missing_cran), paste(missing_cran, collapse = ", "))) + install.packages(missing_cran, repos = "https://cloud.r-project.org") +} +if (length(missing_bioc) > 0) { + message(sprintf("Installing %d Bioconductor package(s): %s", length(missing_bioc), paste(missing_bioc, collapse = ", "))) + if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager", repos = "https://cloud.r-project.org") + BiocManager::install(missing_bioc, ask = FALSE, update = FALSE) +} + +still_missing <- Filter(is_missing, c(R_DEPENDENCIES$cran, R_DEPENDENCIES$bioc)) +if (length(still_missing) > 0) { + message(sprintf("Still missing after install attempt: %s", paste(still_missing, collapse = ", "))) + quit(status = 1) +} + +message("All R report dependencies installed successfully.") \ No newline at end of file From 0f6083dd3cc162a2f1c4b53f90618a3224516ced Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 10:16:29 +0100 Subject: [PATCH 014/108] feat(r-utils): add wrapper for R dependency utils - Added utils/installrdeps.py to wrap R dependency utilities in Python for CLI access. --- src/comms/utils/installrdeps.py | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/comms/utils/installrdeps.py diff --git a/src/comms/utils/installrdeps.py b/src/comms/utils/installrdeps.py new file mode 100644 index 0000000..56925a9 --- /dev/null +++ b/src/comms/utils/installrdeps.py @@ -0,0 +1,53 @@ +''' +comMS R dependency wrapper utilities +''' + +# -- Import external dependencies +import json, shutil, subprocess +from importlib.resources import files as pkg_files +from pathlib import Path + +# -- Import internal functions +from comms.utils.log import logMsg + +# -- _r_script: returns Path to a script under comms/r/ +def _r_script(name: str) -> Path: + return pkg_files('comms').joinpath(f'r/{name}') + +# -- check_r_dependencies: returns {'installed': [...], 'missing': [...]}, or None if Rscript itself isn't callable / the check failed +def check_r_dependencies(rscript: str = 'Rscript') -> dict[str, list[str]] | None: + if shutil.which(rscript) is None: + logMsg.debug(f'Rscript not callable: {rscript}') + return None + script_path = _r_script('/deps/check_deps.R') + result = subprocess.run([rscript, '--vanilla', str(script_path)], capture_output=True, text=True) + if result.returncode != 0: + logMsg.debug(f'Dependency check failed: {result.stderr}') + return None + try: + return json.loads(result.stdout.strip()) + except Exception as e: + logMsg.debug(f'Could not parse depencency check output: {e}') + return None + +# -- install_r_dependencies: runs install_deps.R, streaming its messages through logMsg.info; returns True on success (including "nothing to do") +def install_r_dependencies(rscript: str = 'Rscript') -> bool: + if shutil.which(rscript) is None: + logMsg.error(f'Rscript not callable: {rscript}') + return False + script_path = _r_script('deps/install_deps.R') + logMsg.info('Installing R report dependencies (this can take a few minutes)...') + process = subprocess.Popen( + [rscript, '--vanilla', str(script_path)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + for line in process.stdout: + line = line.rstrip() + if line: + logMsg.info(line) + process.wait() + if process.returncode != 0: + logMsg.error('R dependency installation failed; see above output') + return False + logMsg.info('All R report dependencies installed') + return True \ No newline at end of file From 12b3ca74d3105595fd17a2d49c4b96ae39b92c72 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 10:27:38 +0100 Subject: [PATCH 015/108] feat(cli): add r-utils command to utilities - Added cli/rutils.py to create new r-utils command for checking and installing R dependencies. - Modified cli/cli.py to register new r-utils command. --- src/comms/cli/cli.py | 2 ++ src/comms/cli/rutils.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/comms/cli/rutils.py diff --git a/src/comms/cli/cli.py b/src/comms/cli/cli.py index 52c3cf5..cb47e7f 100644 --- a/src/comms/cli/cli.py +++ b/src/comms/cli/cli.py @@ -21,6 +21,7 @@ from comms.cli.pipeline import commsPipeline from comms.cli.config import commsConfig from comms.cli.license import commsLicense +from comms.cli.rutils import commsRUtils from comms.cli.uninstall import commsUninstall from comms.cli.version import commsVersion @@ -51,6 +52,7 @@ comms.add_typer(commsReport) comms.add_typer(commsConfig, name='config', help='Manage comMS configuration', rich_help_panel='comMS Configuration') comms.add_typer(commsLicense) +comms.add_typer(commsRUtils, name='r-utils', help='Check or install required R dependencies', rich_help_panel='Utilities') comms.add_typer(commsUninstall) comms.add_typer(commsVersion) diff --git a/src/comms/cli/rutils.py b/src/comms/cli/rutils.py new file mode 100644 index 0000000..9427be9 --- /dev/null +++ b/src/comms/cli/rutils.py @@ -0,0 +1,32 @@ +''' +comMS CLI subcommand for managing R dependencies +''' + +# -- Import external dependencies +import typer +from typing import Annotated, List, Optional + +# -- Import internal functions +from comms.utils import installrdeps as rUtils +from comms.utils.log import logMsg + +# -- Initialise Typer class +commsRUtils = typer.Typer(add_completion=False, invoke_without_command=True) + +# -- Define rUtils callback +@commsRUtils.callback(invoke_without_command=False) +def rutils_callback(ctx: typer.Context) -> None: + logMsg('R Utilities') + logMsg.debug(f'Starting R utility: {ctx.invoked_subcommand}') + +# -- Define R utility command: check +@commsRUtils.command(rich_help_panel='R Utilities') +def check(): + '''Check that all required R dependencies are installed''' + rUtils.check_r_dependencies() + +# -- Define R utility command: install +@commsRUtils.command(rich_help_panel='R Utilities') +def install(): + '''Install any missing R dependencies''' + rUtils.install_r_dependencies() \ No newline at end of file From b718c0afaa890b6ee7fb8f1e61e5e278b8c46427 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 12:16:34 +0100 Subject: [PATCH 016/108] feat(r-utils): print dependency checks to terminal - Modified utils/installrdeps.py to print installed/missing dependencies to terminal, and to prompt for confirmation before installing if called directly. --- src/comms/utils/installrdeps.py | 57 ++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/src/comms/utils/installrdeps.py b/src/comms/utils/installrdeps.py index 56925a9..ea19916 100644 --- a/src/comms/utils/installrdeps.py +++ b/src/comms/utils/installrdeps.py @@ -6,6 +6,7 @@ import json, shutil, subprocess from importlib.resources import files as pkg_files from pathlib import Path +from rich import print # -- Import internal functions from comms.utils.log import logMsg @@ -14,20 +15,39 @@ def _r_script(name: str) -> Path: return pkg_files('comms').joinpath(f'r/{name}') +# -- _print_dependency_table: returns None but prints a table to output which lists installed/unavailable dependencies +def _print_dependency_table(deps_dict: dict[str, list[str]]) -> None: + installed_deps = deps_dict['installed'] + missing_deps = deps_dict['missing'] + if len(installed_deps) > 0: + print(f'[bold]Installed packages ({len(installed_deps)})[/bold]') + for d in installed_deps: + print(f'\t[bold green]✓[/bold green] {d}') + if len(missing_deps) > 0: + print(f'[bold]Missing packages ({len(missing_deps)})[/bold]') + for d in missing_deps: + print(f'\t[bold red]✗[/bold red] {d}') + print('\nTo install missing packages, run [bold]comms r-utils install[/bold]') + # -- check_r_dependencies: returns {'installed': [...], 'missing': [...]}, or None if Rscript itself isn't callable / the check failed def check_r_dependencies(rscript: str = 'Rscript') -> dict[str, list[str]] | None: if shutil.which(rscript) is None: - logMsg.debug(f'Rscript not callable: {rscript}') + logMsg.error(f'Rscript not callable: {rscript}') return None + logMsg.debug(f'R available at {rscript}') script_path = _r_script('/deps/check_deps.R') result = subprocess.run([rscript, '--vanilla', str(script_path)], capture_output=True, text=True) + logMsg.debug(f'R dependency check ran successfully') if result.returncode != 0: - logMsg.debug(f'Dependency check failed: {result.stderr}') + logMsg.warn(f'Dependency check failed: {result.stderr}') return None + # Parse returned result as JSON try: - return json.loads(result.stdout.strip()) + parsed = json.loads(result.stdout.strip()) + _print_dependency_table(parsed) + return parsed except Exception as e: - logMsg.debug(f'Could not parse depencency check output: {e}') + logMsg.error(f'Could not parse depencency check output: {e}') return None # -- install_r_dependencies: runs install_deps.R, streaming its messages through logMsg.info; returns True on success (including "nothing to do") @@ -50,4 +70,31 @@ def install_r_dependencies(rscript: str = 'Rscript') -> bool: logMsg.error('R dependency installation failed; see above output') return False logMsg.info('All R report dependencies installed') - return True \ No newline at end of file + return True + +# -- install_r_dependencies_terminal: wrapper for install_r_dependencies to print confirm installation +def install_r_dependencies_terminal(rscript: str = 'Rscript') -> None: + # Check that Rscript is available + if shutil.which(rscript) is None: + logMsg.error(f'Rscript not callable: {rscript}') + raise SystemExit(1) + # Check if anything needs installing + try: + script_path = _r_script('/deps/check_deps.R') + result = subprocess.run([rscript, '--vanilla', str(script_path)], capture_output=True, text=True) + parsed = json.loads(result.stdout.strip()) + missing = parsed['missing'] + if len(missing) > 0: + logMsg.info(f'{len(missing)} {'dependencies need' if len(missing) > 1 else 'dependency needs'} to be installed: {', '.join(d for d in missing)}') + while True: + user_confirmation = input('Install these packages? (y/N)').lower() + if user_confirmation in ['', 'n']: + logMsg.info(f'Cancelled dependency installation.') + break + if user_confirmation == 'y': + install_r_dependencies() + break + else: + logMsg.info(f'No dependencies missing') + except Exception as e: + logMsg.error(f'Error while checking for missing dependencies: {e}') \ No newline at end of file From bb0807d208465bf1e3354a5813df7a21f2cab383 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 12:22:30 +0100 Subject: [PATCH 017/108] feat(rutils): update rutils function call - Modified cli/rutils.py to update function called by install subcommand to use terminal version that prompts for confirmation before installing. --- src/comms/cli/rutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/cli/rutils.py b/src/comms/cli/rutils.py index 9427be9..e85c875 100644 --- a/src/comms/cli/rutils.py +++ b/src/comms/cli/rutils.py @@ -29,4 +29,4 @@ def check(): @commsRUtils.command(rich_help_panel='R Utilities') def install(): '''Install any missing R dependencies''' - rUtils.install_r_dependencies() \ No newline at end of file + rUtils.install_r_dependencies_terminal() \ No newline at end of file From 9a4fce30b93f8e4de1bf9a6e1da0932b9f6da159 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 15:19:14 +0100 Subject: [PATCH 018/108] style(r-utils): remove initial debug log message - Modified cli/rutils.py to remove initial debug-level log message. --- src/comms/cli/rutils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/comms/cli/rutils.py b/src/comms/cli/rutils.py index e85c875..616590c 100644 --- a/src/comms/cli/rutils.py +++ b/src/comms/cli/rutils.py @@ -16,8 +16,7 @@ # -- Define rUtils callback @commsRUtils.callback(invoke_without_command=False) def rutils_callback(ctx: typer.Context) -> None: - logMsg('R Utilities') - logMsg.debug(f'Starting R utility: {ctx.invoked_subcommand}') + logMsg('r-utils') # -- Define R utility command: check @commsRUtils.command(rich_help_panel='R Utilities') From 5249667af710fc216d9393d2410fe43f95327c86 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 15:20:20 +0100 Subject: [PATCH 019/108] style(r-utils): update r-utils formatting - Modified utils/installrdeps.py to update style of output text to use logging messages and to use single line. --- src/comms/utils/installrdeps.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/comms/utils/installrdeps.py b/src/comms/utils/installrdeps.py index ea19916..0a2d38f 100644 --- a/src/comms/utils/installrdeps.py +++ b/src/comms/utils/installrdeps.py @@ -20,14 +20,10 @@ def _print_dependency_table(deps_dict: dict[str, list[str]]) -> None: installed_deps = deps_dict['installed'] missing_deps = deps_dict['missing'] if len(installed_deps) > 0: - print(f'[bold]Installed packages ({len(installed_deps)})[/bold]') - for d in installed_deps: - print(f'\t[bold green]✓[/bold green] {d}') + logMsg.info(f'[bold green]✓ Installed packages ({len(installed_deps)})[/bold green]: {", ".join(installed_deps)}') if len(missing_deps) > 0: - print(f'[bold]Missing packages ({len(missing_deps)})[/bold]') - for d in missing_deps: - print(f'\t[bold red]✗[/bold red] {d}') - print('\nTo install missing packages, run [bold]comms r-utils install[/bold]') + logMsg.info(f'[bold red]✗ Missing packages ({len(missing_deps)})[/bold red]: {", ".join(missing_deps)}') + logMsg.info('To install missing packages, run [bold]comms r-utils install[/bold]') # -- check_r_dependencies: returns {'installed': [...], 'missing': [...]}, or None if Rscript itself isn't callable / the check failed def check_r_dependencies(rscript: str = 'Rscript') -> dict[str, list[str]] | None: @@ -85,7 +81,7 @@ def install_r_dependencies_terminal(rscript: str = 'Rscript') -> None: parsed = json.loads(result.stdout.strip()) missing = parsed['missing'] if len(missing) > 0: - logMsg.info(f'{len(missing)} {'dependencies need' if len(missing) > 1 else 'dependency needs'} to be installed: {', '.join(d for d in missing)}') + logMsg.info(f'{len(missing)} {"dependencies need" if len(missing) > 1 else "dependency needs"} to be installed: {", ".join(d for d in missing)}') while True: user_confirmation = input('Install these packages? (y/N)').lower() if user_confirmation in ['', 'n']: From c7ff64d283054cb2a972d75dd6e2d5dcfedf7849 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 15:23:01 +0100 Subject: [PATCH 020/108] style(r-utils): update log message formatting - Modified utils/installrdeps.py to ensure consistent log message formatting. --- src/comms/utils/installrdeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/utils/installrdeps.py b/src/comms/utils/installrdeps.py index 0a2d38f..1061838 100644 --- a/src/comms/utils/installrdeps.py +++ b/src/comms/utils/installrdeps.py @@ -85,7 +85,7 @@ def install_r_dependencies_terminal(rscript: str = 'Rscript') -> None: while True: user_confirmation = input('Install these packages? (y/N)').lower() if user_confirmation in ['', 'n']: - logMsg.info(f'Cancelled dependency installation.') + logMsg.info(f'Cancelled dependency installation') break if user_confirmation == 'y': install_r_dependencies() From e3498e4ddc99f940b59b0d49922335f37f898155 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:29:10 +0100 Subject: [PATCH 021/108] feat(log): add an input level to log for inputs - Modified utils/log.py to add a new level for inputs, which allows inputs via rich.prompt and records to both streams with the same style as other log messages. --- src/comms/utils/log.py | 56 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/comms/utils/log.py b/src/comms/utils/log.py index 5e4d48f..2346e4d 100644 --- a/src/comms/utils/log.py +++ b/src/comms/utils/log.py @@ -2,10 +2,11 @@ Shared utility functions: logging ''' # -- Import external dependencies -import atexit, logging, shutil, sys, tempfile +import atexit, logging, shutil, sys, tempfile, time from pathlib import Path from rich.console import Console, ConsoleRenderable from rich.logging import RichHandler +from rich.prompt import Prompt from rich.text import Text # -- Define level colours for RichHandler @@ -13,6 +14,7 @@ 'DEBUG': 'color(67)', 'PROGRESS': 'color(75)', 'INFO': 'color(33)', + 'INPUT': 'color(28)', 'WARNING': 'color(178)', 'ERROR': 'color(160)', 'CRITICAL': 'color(124)', @@ -28,10 +30,23 @@ def _progress(self, message, *args, **kwargs): logging.Logger.progress = _progress +# -- Register custom INPUT logging level +INPUT = 45 +logging.addLevelName(INPUT, 'INPUT') + +def _input(self, message, *args, **kwargs): + if self.isEnabledFor(INPUT): + self._log(INPUT, message, args, **kwargs) + +logging.Logger.input = _input + + # -- Define custom RichHandler subclass (CommsRichHandler) to allow custom formatting class CommsRichHandler(RichHandler): def emit(self, record: logging.LogRecord) -> None: log_state._emitted = True + if getattr(record, '_suppress_console', False): + return super().emit(record) def render_message(self, record: logging.LogRecord, message: str) -> 'ConsoleRenderable': level_colour = _LEVEL_COLOURS.get(record.levelname, 'white') @@ -61,6 +76,45 @@ def info(cls, msg: str): if cls._instance: cls._instance.logger.info(msg) @classmethod + def input(cls, msg: str, **prompt_kwargs) -> str | None: + if not cls._instance: + return None + logger = cls._instance.logger + if not logger.isEnabledFor(INPUT): + return None + interactive = sys.stdin.isatty() + if interactive: + level_colour = _LEVEL_COLOURS.get('INPUT', 'white') + console = Console(stderr=True) + timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + styled_prompt = ( + f"[dim]{timestamp}[/dim] | " + f"[bold]{logger.name}[/bold] | " + f"[bold {level_colour}]INPUT[/] | [white]{msg}[/]" + ) + prompt_obj = Prompt( + styled_prompt, + console=console, + choices=prompt_kwargs.get('choices'), + show_default=prompt_kwargs.get('show_default', True), + show_choices=prompt_kwargs.get('show_choices', True), + ) + default = prompt_kwargs.get('default', ...) + answer = prompt_obj(default=default, stream=prompt_kwargs.get('stream')) + # Work out how many terminal rows the prompt (and the typed answer, if echoed) actually occupied, so wrapped prompts get fully erased rather than leaving fragments behind + rendered = prompt_obj.make_prompt(default) + total_len = len(rendered.plain) + len(str(answer)) + width = console.width or 80 + rows = max(1, -(-total_len // width)) # ceil division + for _ in range(rows): + console.file.write("\x1b[1A\x1b[2K") + console.file.write("\r") + console.file.flush() + else: + answer = prompt_kwargs.get('default') + logger.input(f"{msg}: {answer}") + return answer + @classmethod def warn(cls, msg: str): if cls._instance: cls._instance.logger.warning(msg) From 3ba52a73ccc5decef6c7fcece89745e8a6201e91 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:30:26 +0100 Subject: [PATCH 022/108] style(r-utils): update log levels - Modified utils/installrdeps.py to update levels used in logging, and to use new log input method. --- src/comms/utils/installrdeps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/comms/utils/installrdeps.py b/src/comms/utils/installrdeps.py index 1061838..dc909d6 100644 --- a/src/comms/utils/installrdeps.py +++ b/src/comms/utils/installrdeps.py @@ -60,7 +60,7 @@ def install_r_dependencies(rscript: str = 'Rscript') -> bool: for line in process.stdout: line = line.rstrip() if line: - logMsg.info(line) + logMsg.progress(line) process.wait() if process.returncode != 0: logMsg.error('R dependency installation failed; see above output') @@ -83,7 +83,7 @@ def install_r_dependencies_terminal(rscript: str = 'Rscript') -> None: if len(missing) > 0: logMsg.info(f'{len(missing)} {"dependencies need" if len(missing) > 1 else "dependency needs"} to be installed: {", ".join(d for d in missing)}') while True: - user_confirmation = input('Install these packages? (y/N)').lower() + user_confirmation = logMsg.input('Install these packages? [dim](y/N)[/dim]', choices=['y','n'], default='n', case_sensitive=False, show_default=False, show_choices=False).lower() if user_confirmation in ['', 'n']: logMsg.info(f'Cancelled dependency installation') break From 2ed08d134dd8d79d4081ff2be7fe8e0e43fbb25c Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:33:50 +0100 Subject: [PATCH 023/108] feat(sheet): add sample sheet parser utility - Modified utils/sheet.py to add a utility function for parsing TSV files (ie sample sheets) to SampleRow classes. --- src/comms/utils/sheet.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/comms/utils/sheet.py b/src/comms/utils/sheet.py index 2cf4cad..fa49170 100644 --- a/src/comms/utils/sheet.py +++ b/src/comms/utils/sheet.py @@ -32,4 +32,26 @@ def render_sample_sheet(rows) -> str: lines.append('\t'.join( [r.sample_id, r.raw_file, r.treatment, r.fraction, replicate, r.batch] )) - return '\n'.join(lines) + '\n' \ No newline at end of file + return '\n'.join(lines) + '\n' + +# -- parse_sample_sheet: parse a TSV text into a list of SampleRow +def parse_sample_sheet(text: str) -> list['SampleRow']: + lines = [line for line in text.splitlines() if line.strip()] + if not lines: + return [] + header = [h.strip() for h in lines[0].split('\t')] + rows: list[SampleRow] = [] + for line in lines[1:]: + values = line.split('\t') + record = dict(zip(header, values)) + replicate_text = record.get('replicate', '').strip() + rows.append(SampleRow( + sample_id=record.get('sample_id', '').strip(), + raw_file=record.get('raw_file', '').strip(), + treatment=record.get('treatment', '').strip(), + fraction=record.get('fraction', '').strip(), + replicate=int(replicate_text) if replicate_text else None, + batch=record.get('batch', '').strip(), + replicate_overridden=bool(replicate_text), + )) + return rows \ No newline at end of file From c4daebf211f97ddf858c3b0df3c89358b8bbc9e3 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:40:32 +0100 Subject: [PATCH 024/108] feat(experiment): add existing experiment check - Modified commands/experiment.py to add a helper function to check if an experiment already exists at a given path. --- src/comms/commands/experiment.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index dbf1ff4..098bedc 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -3,17 +3,35 @@ ''' # -- Import external dependencies -import tomli_w, typer +import tomli_w, tomllib, typer from datetime import datetime, timezone from pathlib import Path from rich import print from typing import Literal # -- Import internal functions -from comms.utils.log import logMsg -from comms.utils.sheet import SampleRow, render_sample_sheet from comms.commands.config import _apply_protocol_flags, _apply_organism, _writeConfigTo +from comms.utils.context import _normalise_dirs +from comms.utils.log import logMsg from comms.utils.settings import loadDefaultConfig +from comms.utils.sheet import SampleRow, render_sample_sheet, parse_sample_sheet + +# -- _existing_experiment: returns (root, comms_dir, metadata, config, sample_rows) if experiment_dir already holds a saved experiment else None +def _existing_experiment(experiment_dir: Path): + root, comms_dir = _normalise_dirs(experiment_dir) + meta_path = comms_dir / 'experiment.toml' + config_path = comms_dir / 'config.toml' + if not (meta_path.exists() and config_path.exists()): + return None + with meta_path.open('rb') as f: + metadata = tomllib.load(f) + with config_path.open('rb') as f: + config = tomllib.load(f) + sheet_path = comms_dir / 'sample_sheet.tsv' + rows: list[SampleRow] = [] + if sheet_path.exists(): + rows = parse_sample_sheet(sheet_path.read_text(encoding='utf-8')) + return root, comms_dir, metadata, config, rows # -- launch_experiment_gui: opens the PySide6 experiment setup window def launch_experiment_gui() -> None: From 9bf4c1b7550108d35e61cb252bd4c733f1f5f64b Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:47:27 +0100 Subject: [PATCH 025/108] feat(experiment): update experiment logic for edit - Modified commands/experiment.py to update logic to allow editing an existing experiment using headless mode. --- src/comms/commands/experiment.py | 108 +++++++++++++++++++------------ 1 file changed, 68 insertions(+), 40 deletions(-) diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index 098bedc..731e573 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -3,7 +3,7 @@ ''' # -- Import external dependencies -import tomli_w, tomllib, typer +import re, tomli_w, tomllib, typer from datetime import datetime, timezone from pathlib import Path from rich import print @@ -44,9 +44,11 @@ def launch_experiment_gui() -> None: logMsg.info(f'Launching experiment setup GUI') raise SystemExit(run_app()) -# -- _prompt_list: return a list of strings by repeated prompting -def _prompt_list(label: str) -> list[str]: - items: list[str] = [] +# -- _prompt_list: return a list of strings by repeated prompting, prepopulated with any existing items +def _prompt_list(label: str, existing: list[str] | None = None) -> list[str]: + items: list[str] = list(existing or []) + if items: + print(f'Current {label}(s): {", ".join(items)}') while True: value = typer.prompt(f'Add a {label} (blank to finish)', default='', show_default=False) value = value.strip() @@ -56,26 +58,40 @@ def _prompt_list(label: str) -> list[str]: items.append(value) return items -# -- _choose: prompt until the user picks one of the allowed options -def _choose(label: str, options: list[str]) -> str: +# -- _choose: prompt until the user picks one of the allowed options, prepopulated with any existing values +def _choose(label: str, options: list[str], default: str | None = None) -> str: while True: - choice = typer.prompt(f'{label} {options}') + choice = typer.prompt(f'{label} {options}', default=default, show_default=default is not None) if choice in options: return choice -# -- run_experiment_headless: build a sample sheet, config and metadata via prompts -def run_experiment_headless() -> None: +# -- run_experiment_headless: build a sample sheet, config and metadata via prompts, or edit an existing experiment +def run_experiment_headless(experiment_dir: Path | None = None) -> None: logMsg('experiment') logMsg.debug('Starting command: experiment') - logMsg.info('Starting headless experiment setup') - - name = typer.prompt('Experiment name') - base_dir = Path(typer.prompt('Save experiment to (directory)')).expanduser() - bin_dir = typer.prompt('Bin directory (blank to auto-resolve)', default='', show_default=False).strip() - database = typer.prompt('Combined database FASTA').strip() + existing = _existing_experiment(experiment_dir) if experiment_dir else None + edit_mode = existing is not None + if edit_mode: + root, comms_dir, metadata, config, existing_rows = existing + logMsg.info(f'Existing experiment found at {comms_dir}, editing in place') + else: + metadata, config, existing_rows = {}, {}, [] + logMsg.info('Starting headless experiment edit' if edit_mode else 'Starting headless experiment setup') - treatments = _prompt_list('treatment') - fractions = _prompt_list('fraction') + name = typer.prompt( + 'Experiment name', + default=metadata.get('experiment', {}).get('name', ''), show_default=edit_mode, + ) + if edit_mode: + base_dir = root + else: + base_dir = Path(typer.prompt('Save experiment to (directory)', default=str(experiment_dir) if experiment_dir else None, show_default=experiment_dir is not None)).expanduser() + bin_dir = typer.prompt('Bin directory (blank to auto-resolve)', default=metadata.get('experiment', {}).get('bin_dir', ''), show_default=edit_mode).strip() + database = typer.prompt('Combined database FASTA', default=metadata.get('files', {}).get('database', ''), show_default=edit_mode).strip() + existing_treatments = sorted({r.treatment for r in existing_rows if r.treatment}) + existing_fractions = sorted({r.fraction for r in existing_rows if r.fraction}) + treatments = _prompt_list('treatment', existing=existing_treatments) + fractions = _prompt_list('fraction', existing=existing_fractions) if not treatments or not fractions: logMsg.error('At least one treatment and one fraction are required') raise SystemExit(1) @@ -92,35 +108,41 @@ def run_experiment_headless() -> None: logMsg.error(f'No .RAW or .mzML files found in {input_dir}') raise SystemExit(1) + existing_by_raw = {r.raw_file: r for r in existing_rows} rows: list[SampleRow] = [] counters: dict[tuple[str, str], int] = {} for f in files: print(f'\n[bold]{f.name}[/bold]') - treatment = _choose('Treatment', treatments) - fraction = _choose('Fraction', fractions) + prior = existing_by_raw.get(f.name) + treatment = _choose('Treatment', treatments, default=prior.treatment if prior else None) + fraction = _choose('Fraction', fractions, default=prior.fraction if prior else None) key = (treatment, fraction) counters[key] = counters.get(key, 0) + 1 rows.append(SampleRow( - sample_id=f.stem, raw_file=f.name, - treatment=treatment, fraction=fraction, replicate=counters[key], + sample_id=prior.sample_id if prior else f.stem, + raw_file=f.name, + treatment=treatment, + fraction=fraction, + replicate=counters[key], )) # Config: reuse the same helpers as the GUI's ConfigPanel + index_cfg = config.get('index', {}) + search_cfg = config.get('search', {}) cfg = loadDefaultConfig() cfg = _apply_protocol_flags( cfg, - iodo=typer.confirm('Cysteine carbamidomethylation (static)?', default=False), - ox=typer.confirm('Methionine oxidation (variable)?', default=True), - phos=typer.confirm('STY phosphorylation (variable)?', default=False), - n_cyc=typer.confirm('N-terminal Gln cyclisation?', default=True), - n_ace=typer.confirm('Protein N-terminal acetylation?', default=True), - clip_met=typer.confirm('Clip N-terminal methionine?', default=True), - low_res=typer.confirm('Low-resolution instrument (ion trap)?', default=False), + iodo=typer.confirm('Cysteine carbamidomethylation (static)?', default='C+0' not in index_cfg.get('fixed_mods', '')), + ox=typer.confirm('Methionine oxidation (variable)?', default=bool(re.search(r'M\+15\.9949', index_cfg.get('mods_spec', ''))) if edit_mode else True), + phos=typer.confirm('STY phosphorylation (variable)?', default=bool(re.search(r'STY\+79\.966331', index_cfg.get('mods_spec', ''))) if edit_mode else False), + n_cyc=typer.confirm('N-terminal Gln cyclisation?', default=bool(index_cfg.get('nterm_peptide_mods_spec', '')) if edit_mode else True), + n_ace=typer.confirm('Protein N-terminal acetylation?', default=bool(index_cfg.get('nterm_protein_mods_spec', '')) if edit_mode else True), + clip_met=typer.confirm('Clip N-terminal methionine?', default=index_cfg.get('clip_n_met', True) if edit_mode else True), + low_res=typer.confirm('Low-resolution instrument (ion trap)?', default=(search_cfg.get('score_function') == 'combined-p-value') if edit_mode else False), ) - cfg.setdefault('index', {})['custom_mods'] = '' - # Analysis mode: single- or multi-species - organisms: dict[str, str] = {} - multispecies = typer.confirm('Multispecies analysis (per-organism FDR)?', default=False) + cfg.setdefault('index', {})['custom_mods'] = index_cfg.get('custom_mods', '') + organisms: dict[str, str] = dict(config.get('organism', {})) if edit_mode else {} + multispecies = typer.confirm('Multispecies analysis (per-organism FDR)?', default=bool(organisms)) if multispecies: while True: label = typer.prompt('Organism label (blank to finish)', default='', show_default=False).strip() @@ -129,17 +151,24 @@ def run_experiment_headless() -> None: pattern = typer.prompt(f'Header pattern for {label}').strip() if pattern: organisms[label] = pattern + else: + organisms = {} cfg = _apply_organism(cfg, organisms) if multispecies: - cfg['percolator']['shared_psm'] = typer.prompt(f'Shared PSM handling policy', default='drop', type=Literal['drop','include'], show_choices=True, show_default=True).strip() - # Report settings + cfg['percolator']['shared_psm'] = typer.prompt('Shared PSM handling policy', default=config.get('percolator', {}).get('shared_psm', 'drop'), type=Literal['drop', 'include'], show_choices=True, show_default=True).strip() + report_meta = metadata.get('report', {}) organism_prefix = '' - include_report = typer.confirm('Create report?', default=True) + include_report = typer.confirm('Create report?', default=report_meta.get('enabled', True)) if include_report: - reference = typer.prompt('Reference protein annotation file (blank to skip)', default='', show_default=False).strip() - contaminants = typer.prompt('Contaminants list CSV path (blank to skip)', default='', show_default=False).strip() + reference = typer.prompt('Reference protein annotation file (blank to skip)', default=report_meta.get('ref_info', ''), show_default=edit_mode).strip() + contaminants = typer.prompt('Contaminants list CSV path (blank to skip)', default=report_meta.get('cont_csv', ''), show_default=edit_mode).strip() if multispecies: - organism_prefix = typer.prompt('Primary organism ID prefix', default='', show_default=False).strip() + organism_prefix = typer.prompt('Primary organism ID prefix', default=report_meta.get('organism_prefix', ''), show_default=edit_mode).strip() + status = rdepsFuncs.check_r_dependencies() + if status is not None and status['missing']: + if typer.confirm(f"Install missing R report dependencies now? ({', '.join(status['missing'])})", default=True): + rdepsFuncs.install_r_dependencies() + # Write all three files out_dir = base_dir / 'comms' out_dir.mkdir(parents=True, exist_ok=True) @@ -171,9 +200,8 @@ def run_experiment_headless() -> None: meta['report']['organism_prefix'] = organism_prefix else: meta.setdefault('report', {})['enabled'] = False - # Save metadata file with (out_dir / 'experiment.toml').open('wb') as f: tomli_w.dump(meta, f) - logMsg.info(f'Experiment written to {out_dir}') + logMsg.info(f'Experiment {"updated" if edit_mode else "written"} at {out_dir}') print(f'\nRun the pipeline with:\n' f'\t[bold]comms pipeline {sheet_path} --database --input {input_dir} --experiment-dir {base_dir}[/bold]\n') \ No newline at end of file From 599e9ed00361bf411da312395503dafdc1feacd9 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:51:00 +0100 Subject: [PATCH 026/108] feat(experiment): update cli command to accept dir - Modified cli/cli.py to update experiment command to accept an experiment directory if provided. --- src/comms/cli/cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/comms/cli/cli.py b/src/comms/cli/cli.py index cb47e7f..167fe01 100644 --- a/src/comms/cli/cli.py +++ b/src/comms/cli/cli.py @@ -59,16 +59,20 @@ # -- Register experiment command @comms.command(rich_help_panel='comMS Configuration') def experiment( + experiment_dir: Annotated[ + Optional[Path], + typer.Argument(help='Existing experiment directory to edit (leave blank to create a new experiment)') + ] = None, headless: Annotated[ bool, typer.Option('--headless', help='Run setup in terminal instead of GUI') ] = False, ): - '''Set up a comMS experiment (sample sheet + config + metadata)''' + '''Set up a comMS experiment (sample sheet + config + metadata), or edit an existing one''' if headless: - experimentFuncs.run_experiment_headless() + experimentFuncs.run_experiment_headless(experiment_dir) else: - experimentFuncs.launch_experiment_gui() + experimentFuncs.launch_experiment_gui(experiment_dir) # ==================== # Top-level callback: --verbose / --debug flags From 00919b7cdb2758bfd9bb2042c5db06f32945f915 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:51:41 +0100 Subject: [PATCH 027/108] fix(log): allow case_sensitive kwarg in input - Modified utils/log.py to allow case_sensitive kwarg for rich.prompt to be passed through to logMsg.input. --- src/comms/utils/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/comms/utils/log.py b/src/comms/utils/log.py index 2346e4d..6faefdd 100644 --- a/src/comms/utils/log.py +++ b/src/comms/utils/log.py @@ -99,6 +99,7 @@ def input(cls, msg: str, **prompt_kwargs) -> str | None: show_default=prompt_kwargs.get('show_default', True), show_choices=prompt_kwargs.get('show_choices', True), ) + prompt_obj.case_sensitive = prompt_kwargs.get('case_sensitive', True) default = prompt_kwargs.get('default', ...) answer = prompt_obj(default=default, stream=prompt_kwargs.get('stream')) # Work out how many terminal rows the prompt (and the typed answer, if echoed) actually occupied, so wrapped prompts get fully erased rather than leaving fragments behind From 0b21ae02da4b52819505be28aad91e438b884b9c Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 16:53:10 +0100 Subject: [PATCH 028/108] fix(cli): add required libraries to cli - Modified cli/cli.py to add import statements for required libraries. --- src/comms/cli/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/comms/cli/cli.py b/src/comms/cli/cli.py index 167fe01..4075d27 100644 --- a/src/comms/cli/cli.py +++ b/src/comms/cli/cli.py @@ -4,7 +4,8 @@ # -- Import external dependencies import logging, typer -from typing import Annotated +from pathlib import Path +from typing import Annotated, Optional # -- Import internal utility functions from comms.utils.settings import initComms From 4b176e4f4403c8738cb43c96d30e8ae967803a76 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 17:09:12 +0100 Subject: [PATCH 029/108] style(experiment): update experiment input logging - Modified commands/experiment.py to update inputs in headless mode to use new logMsg.input class method. --- src/comms/commands/experiment.py | 68 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index 731e573..7f45442 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -3,15 +3,15 @@ ''' # -- Import external dependencies -import re, tomli_w, tomllib, typer +import re, tomli_w, tomllib from datetime import datetime, timezone from pathlib import Path from rich import print -from typing import Literal # -- Import internal functions from comms.commands.config import _apply_protocol_flags, _apply_organism, _writeConfigTo from comms.utils.context import _normalise_dirs +from comms.utils.installrdeps import check_r_dependencies, install_r_dependencies from comms.utils.log import logMsg from comms.utils.settings import loadDefaultConfig from comms.utils.sheet import SampleRow, render_sample_sheet, parse_sample_sheet @@ -50,7 +50,7 @@ def _prompt_list(label: str, existing: list[str] | None = None) -> list[str]: if items: print(f'Current {label}(s): {", ".join(items)}') while True: - value = typer.prompt(f'Add a {label} (blank to finish)', default='', show_default=False) + value = logMsg.input(f'Add a {label} (blank to finish)', default='', show_default=False) value = value.strip() if not value: break @@ -60,10 +60,15 @@ def _prompt_list(label: str, existing: list[str] | None = None) -> list[str]: # -- _choose: prompt until the user picks one of the allowed options, prepopulated with any existing values def _choose(label: str, options: list[str], default: str | None = None) -> str: - while True: - choice = typer.prompt(f'{label} {options}', default=default, show_default=default is not None) - if choice in options: - return choice + formatted_options = [f'{opt} (default)' if opt==default else opt for opt in options] + label = f'{label} [dim]\[{", ".join(formatted_options)}][/dim]' + return logMsg.input(label, choices=options, default=default, show_default=False, show_choices=False) + +# -- _confirm: yes/no prompt via logMsg.input, returned as a bool +def _confirm(msg: str, default: bool) -> bool: + msg = f'{msg} [dim]({"Y/n" if default else "y/N"})[/dim]' + answer = logMsg.input(msg, choices=['y', 'n'], default='y' if default else 'n', case_sensitive=False, show_choices=False, show_default=False) + return str(answer).strip().lower() == 'y' # -- run_experiment_headless: build a sample sheet, config and metadata via prompts, or edit an existing experiment def run_experiment_headless(experiment_dir: Path | None = None) -> None: @@ -78,16 +83,13 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: metadata, config, existing_rows = {}, {}, [] logMsg.info('Starting headless experiment edit' if edit_mode else 'Starting headless experiment setup') - name = typer.prompt( - 'Experiment name', - default=metadata.get('experiment', {}).get('name', ''), show_default=edit_mode, - ) + name = logMsg.input('Experiment name', default=metadata.get('experiment', {}).get('name', ''), show_default=edit_mode) if edit_mode: base_dir = root else: - base_dir = Path(typer.prompt('Save experiment to (directory)', default=str(experiment_dir) if experiment_dir else None, show_default=experiment_dir is not None)).expanduser() - bin_dir = typer.prompt('Bin directory (blank to auto-resolve)', default=metadata.get('experiment', {}).get('bin_dir', ''), show_default=edit_mode).strip() - database = typer.prompt('Combined database FASTA', default=metadata.get('files', {}).get('database', ''), show_default=edit_mode).strip() + base_dir = Path(logMsg.input('Save experiment to (directory)', default=str(experiment_dir) if experiment_dir else None, show_default=experiment_dir is not None)).expanduser() + bin_dir = logMsg.input('Bin directory (blank to auto-resolve)', default=metadata.get('experiment', {}).get('bin_dir', ''), show_default=edit_mode).strip() + database = logMsg.input('Combined database FASTA', default=metadata.get('files', {}).get('database', ''), show_default=edit_mode).strip() existing_treatments = sorted({r.treatment for r in existing_rows if r.treatment}) existing_fractions = sorted({r.fraction for r in existing_rows if r.fraction}) treatments = _prompt_list('treatment', existing=existing_treatments) @@ -96,7 +98,7 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: logMsg.error('At least one treatment and one fraction are required') raise SystemExit(1) - input_dir = Path(typer.prompt('Directory of .RAW / .mzML files')).expanduser() + input_dir = Path(logMsg.input('Directory of .RAW / .mzML files')).expanduser() input_files = _prompt_list('data file') files = [] for f in input_files: @@ -132,42 +134,42 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: cfg = loadDefaultConfig() cfg = _apply_protocol_flags( cfg, - iodo=typer.confirm('Cysteine carbamidomethylation (static)?', default='C+0' not in index_cfg.get('fixed_mods', '')), - ox=typer.confirm('Methionine oxidation (variable)?', default=bool(re.search(r'M\+15\.9949', index_cfg.get('mods_spec', ''))) if edit_mode else True), - phos=typer.confirm('STY phosphorylation (variable)?', default=bool(re.search(r'STY\+79\.966331', index_cfg.get('mods_spec', ''))) if edit_mode else False), - n_cyc=typer.confirm('N-terminal Gln cyclisation?', default=bool(index_cfg.get('nterm_peptide_mods_spec', '')) if edit_mode else True), - n_ace=typer.confirm('Protein N-terminal acetylation?', default=bool(index_cfg.get('nterm_protein_mods_spec', '')) if edit_mode else True), - clip_met=typer.confirm('Clip N-terminal methionine?', default=index_cfg.get('clip_n_met', True) if edit_mode else True), - low_res=typer.confirm('Low-resolution instrument (ion trap)?', default=(search_cfg.get('score_function') == 'combined-p-value') if edit_mode else False), + iodo=_confirm('Cysteine carbamidomethylation (static)?', default='C+0' not in index_cfg.get('fixed_mods', '')), + ox=_confirm('Methionine oxidation (variable)?', default=bool(re.search(r'M\+15\.9949', index_cfg.get('mods_spec', ''))) if edit_mode else True), + phos=_confirm('STY phosphorylation (variable)?', default=bool(re.search(r'STY\+79\.966331', index_cfg.get('mods_spec', ''))) if edit_mode else False), + n_cyc=_confirm('N-terminal Gln cyclisation?', default=bool(index_cfg.get('nterm_peptide_mods_spec', '')) if edit_mode else True), + n_ace=_confirm('Protein N-terminal acetylation?', default=bool(index_cfg.get('nterm_protein_mods_spec', '')) if edit_mode else True), + clip_met=_confirm('Clip N-terminal methionine?', default=index_cfg.get('clip_n_met', True) if edit_mode else True), + low_res=_confirm('Low-resolution instrument (ion trap)?', default=(search_cfg.get('score_function') == 'combined-p-value') if edit_mode else False), ) cfg.setdefault('index', {})['custom_mods'] = index_cfg.get('custom_mods', '') organisms: dict[str, str] = dict(config.get('organism', {})) if edit_mode else {} - multispecies = typer.confirm('Multispecies analysis (per-organism FDR)?', default=bool(organisms)) + multispecies = _confirm('Multispecies analysis (per-organism FDR)?', default=bool(organisms)) if multispecies: while True: - label = typer.prompt('Organism label (blank to finish)', default='', show_default=False).strip() + label = logMsg.input('Organism label (blank to finish)', default='', show_default=False).strip() if not label: break - pattern = typer.prompt(f'Header pattern for {label}').strip() + pattern = logMsg.input(f'Header pattern for {label}').strip() if pattern: organisms[label] = pattern else: organisms = {} cfg = _apply_organism(cfg, organisms) if multispecies: - cfg['percolator']['shared_psm'] = typer.prompt('Shared PSM handling policy', default=config.get('percolator', {}).get('shared_psm', 'drop'), type=Literal['drop', 'include'], show_choices=True, show_default=True).strip() + cfg['percolator']['shared_psm'] = logMsg.input('Shared PSM handling policy', choices=['drop', 'include'], default=config.get('percolator', {}).get('shared_psm', 'drop'), show_choices=True, show_default=True).strip() report_meta = metadata.get('report', {}) organism_prefix = '' - include_report = typer.confirm('Create report?', default=report_meta.get('enabled', True)) + include_report = _confirm('Create report?', default=report_meta.get('enabled', True)) if include_report: - reference = typer.prompt('Reference protein annotation file (blank to skip)', default=report_meta.get('ref_info', ''), show_default=edit_mode).strip() - contaminants = typer.prompt('Contaminants list CSV path (blank to skip)', default=report_meta.get('cont_csv', ''), show_default=edit_mode).strip() + reference = logMsg.input('Reference protein annotation file (blank to skip)', default=report_meta.get('ref_info', ''), show_default=edit_mode).strip() + contaminants = logMsg.input('Contaminants list CSV path (blank to skip)', default=report_meta.get('cont_csv', ''), show_default=edit_mode).strip() if multispecies: - organism_prefix = typer.prompt('Primary organism ID prefix', default=report_meta.get('organism_prefix', ''), show_default=edit_mode).strip() - status = rdepsFuncs.check_r_dependencies() + organism_prefix = logMsg.input('Primary organism ID prefix', default=report_meta.get('organism_prefix', ''), show_default=edit_mode).strip() + status = check_r_dependencies() if status is not None and status['missing']: - if typer.confirm(f"Install missing R report dependencies now? ({', '.join(status['missing'])})", default=True): - rdepsFuncs.install_r_dependencies() + if _confirm(f"Install missing R report dependencies now? ({', '.join(status['missing'])})", default=True): + install_r_dependencies() # Write all three files out_dir = base_dir / 'comms' From 24b13ef397db4564115d672960f5769167566c1b Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 17:56:45 +0100 Subject: [PATCH 030/108] feat(experiment): update experiment gui launcher - Modified commands/experiment.py to update GUI launcher function to use passed experiment directory if present. --- src/comms/commands/experiment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index 7f45442..df50ace 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -34,7 +34,7 @@ def _existing_experiment(experiment_dir: Path): return root, comms_dir, metadata, config, rows # -- launch_experiment_gui: opens the PySide6 experiment setup window -def launch_experiment_gui() -> None: +def launch_experiment_gui(experiment_dir: Path | None = None) -> None: logMsg('experiment') try: from comms.gui.app import run_app @@ -42,7 +42,7 @@ def launch_experiment_gui() -> None: logMsg.error(f'Could not import GUI components: {e}') raise SystemExit(1) logMsg.info(f'Launching experiment setup GUI') - raise SystemExit(run_app()) + raise SystemExit(run_app(experiment_dir)) # -- _prompt_list: return a list of strings by repeated prompting, prepopulated with any existing items def _prompt_list(label: str, existing: list[str] | None = None) -> list[str]: @@ -61,7 +61,7 @@ def _prompt_list(label: str, existing: list[str] | None = None) -> list[str]: # -- _choose: prompt until the user picks one of the allowed options, prepopulated with any existing values def _choose(label: str, options: list[str], default: str | None = None) -> str: formatted_options = [f'{opt} (default)' if opt==default else opt for opt in options] - label = f'{label} [dim]\[{", ".join(formatted_options)}][/dim]' + label = f'{label} [dim]\\[{", ".join(formatted_options)}][/dim]' return logMsg.input(label, choices=options, default=default, show_default=False, show_choices=False) # -- _confirm: yes/no prompt via logMsg.input, returned as a bool From 8cd3f5bcc026eab6f34a4cc502297e23230f7216 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 17:59:27 +0100 Subject: [PATCH 031/108] feat(gui): update app launcher to pass arguments - Modified gui/app.py to pass a provided experiment directory (if present) to the main GUI window. --- src/comms/gui/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/comms/gui/app.py b/src/comms/gui/app.py index d4bf967..8bda882 100644 --- a/src/comms/gui/app.py +++ b/src/comms/gui/app.py @@ -4,14 +4,15 @@ # -- Import external dependencies import sys +from pathlib import Path from PySide6.QtWidgets import QApplication # -- Import internal functions from comms.gui.main_window import MainWindow # -- run_app: create the QApplication, show the main window, and run the event loop -def run_app() -> int: +def run_app(experiment_dir: Path | None = None) -> int: app = QApplication.instance() or QApplication(sys.argv) - window = MainWindow() + window = MainWindow(experiment_dir=experiment_dir) window.show() return app.exec() \ No newline at end of file From d25ce7678fe4d4192e11fb6e4d0861b46439d6d0 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 18:03:09 +0100 Subject: [PATCH 032/108] feat(gui): update main window - Modified gui/main_window.py to add function to load existing experiment directory as required and prepopulate relevant fields. --- src/comms/gui/main_window.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/comms/gui/main_window.py b/src/comms/gui/main_window.py index 3d3d97d..941e0a3 100644 --- a/src/comms/gui/main_window.py +++ b/src/comms/gui/main_window.py @@ -3,6 +3,7 @@ ''' # -- Import external dependencies +from pathlib import Path from PySide6.QtCore import QSize from PySide6.QtWidgets import QMainWindow, QTabWidget, QVBoxLayout, QWidget @@ -18,7 +19,7 @@ # -- MainWindow: four numbered tabs with per-tab status icons class MainWindow(QMainWindow): - def __init__(self, parent=None): + def __init__(self, experiment_dir: Path | None = None, parent=None): super().__init__(parent) self._log = logMsg('experiment') self.setWindowTitle('comms experiment setup') @@ -68,11 +69,31 @@ def __init__(self, parent=None): self.tabs.setTabIcon(self._config_index, status_icon(self.config.tracker.status)) self.tabs.setTabIcon(self._review_index, status_icon(self.experiment.tracker.status)) self.save.refresh() + + # if a directory provided, call _load_existing + if experiment_dir is not None: + self._load_existing(experiment_dir) def _on_tab_changed(self, index: int) -> None: if index == self._review_index: self.save.refresh() - + + def _load_existing(self, experiment_dir: Path) -> None: + from comms.commands.experiment import _existing_experiment + existing = _existing_experiment(experiment_dir) + if existing is None: + # If no experiment actually saved to directory, only pre-fill save to field + self.experiment._dir.setText(str(experiment_dir)) + return + root, comms_dir, metadata, config, rows = existing + self.experiment.load_from_metadata(root, metadata) + report_meta = metadata.get('report', {}) + self.config.load_from_config(config, report_meta) + treatments = sorted({r.treatment for r in rows if r.treatment}) + fractions = sorted({r.fraction for r in rows if r.fraction}) + self.sample.load(rows, treatments, fractions) + self._log.info(f'Loaded existing experiment from {comms_dir}') + def closeEvent(self, event) -> None: self._log.info('Closed experiment setup GUI') super().closeEvent(event) \ No newline at end of file From ed2d1137cfafa1ed26ca9a43b6eac145457f70a1 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 8 Jul 2026 18:04:16 +0100 Subject: [PATCH 033/108] feat(gui): add load existing functions to tabs - Modified gui/panels/config_panel.py, gui/panels/experiment_panel.py, and gui/panels/sample_panel.py to add function to each which loads existing experiment directory files and prepopulates fields as appropriate. --- src/comms/gui/panels/config_panel.py | 38 ++++++++++++++++++++++-- src/comms/gui/panels/experiment_panel.py | 12 ++++++++ src/comms/gui/panels/sample_panel.py | 10 ++++++- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/comms/gui/panels/config_panel.py b/src/comms/gui/panels/config_panel.py index 07f6327..5b371e5 100644 --- a/src/comms/gui/panels/config_panel.py +++ b/src/comms/gui/panels/config_panel.py @@ -3,18 +3,19 @@ ''' # -- Import external dependencies +import re from pathlib import Path from PySide6.QtCore import Qt, Signal from PySide6.QtWidgets import ( QWidget, QVBoxLayout, QHBoxLayout, QFileDialog, QFormLayout, QGroupBox, QCheckBox, - QComboBox, QLineEdit, QPushButton, QTableWidget, + QComboBox, QLineEdit, QPushButton, QTableWidget, QTableWidgetItem ) # -- Import internal functions from comms.gui.status import PanelStateTracker from comms.utils.settings import loadDefaultConfig from comms.commands.config import ( - _apply_protocol_flags, _apply_organism, _apply_custom_mod, _writeConfigTo, + _apply_protocol_flags, _apply_organism, _apply_custom_mod, _writeConfigTo, MZ_BIN_WIDTH_LOW_RES, ) # -- Define class ConfigPanel to define a structured form mirroring `comms config set` with an additional analysis type activating the organism table @@ -320,4 +321,35 @@ def sync_tracker(self) -> None: def write(self, out_dir: Path) -> Path: path = out_dir / 'config.toml' _writeConfigTo(self._build_config(), path) - return path \ No newline at end of file + return path + + # -- load_from_config: populate fields from a loaded config.toml + experiment.toml [report] section + def load_from_config(self, cfg: dict, report_meta: dict) -> None: + index_cfg = cfg.get('index', {}) + search_cfg = cfg.get('search', {}) + self._iodo.setChecked('C+0' not in index_cfg.get('fixed_mods', '')) + self._ox.setChecked(bool(re.search(r'M\+15\.9949', index_cfg.get('mods_spec', '')))) + self._phos.setChecked(bool(re.search(r'STY\+79\.966331', index_cfg.get('mods_spec', '')))) + self._n_cyc.setChecked(bool(index_cfg.get('nterm_peptide_mods_spec', ''))) + self._n_ace.setChecked(bool(index_cfg.get('nterm_protein_mods_spec', ''))) + self._clip_met.setChecked(index_cfg.get('clip_n_met', True)) + self._custom.setText(index_cfg.get('custom_mods', '')) + self._res.setCurrentIndex(1 if search_cfg.get('mz_bin_width') == MZ_BIN_WIDTH_LOW_RES else 0) + + organisms = cfg.get('organism', {}) + self._analysis.setCurrentIndex(1 if organisms else 0) + self._sharedpsm.setCurrentIndex(1 if cfg.get('percolator', {}).get('shared_psm') == 'include' else 0) + self._org_table.setRowCount(0) + for label, pattern in organisms.items(): + row = self._org_table.rowCount() + self._org_table.insertRow(row) + self._org_table.setItem(row, 0, QTableWidgetItem(label)) + self._org_table.setItem(row, 1, QTableWidgetItem(pattern)) + + self._report_enabled.setChecked(bool(report_meta.get('enabled', True))) + self._reference.setText(str(report_meta.get('ref_info', ''))) + self._contaminant.setText(str(report_meta.get('cont_csv', ''))) + self._organism_prefix.setText(str(report_meta.get('organism_prefix', ''))) + self._update_organism_enabled() + self._update_report_fields_enabled() + self._on_changed() \ No newline at end of file diff --git a/src/comms/gui/panels/experiment_panel.py b/src/comms/gui/panels/experiment_panel.py index 937cd76..7e14346 100644 --- a/src/comms/gui/panels/experiment_panel.py +++ b/src/comms/gui/panels/experiment_panel.py @@ -148,5 +148,17 @@ def write_metadata(self, out_dir: Path, files: dict | None = None, analysis=None tomli_w.dump(meta, f) return path + # -- load_from_metadata: populate fields from a loaded experiment.toml + resolved base_dir + def load_from_metadata(self, base_dir: Path, metadata: dict) -> None: + self._name.setText(metadata.get('experiment', {}).get('name', '')) + self._dir.setText(str(base_dir)) + database = metadata.get('files', {}).get('database', '') + if database: + self._database.setText(str(database)) + bin_dir = metadata.get('experiment', {}).get('bin_dir', '') + if bin_dir: + self._bin.setText(str(bin_dir)) + self.changed.emit() + def is_valid(self) -> bool: return bool(self.experiment_name()) and self.base_dir() is not None and self.database_path() is not None \ No newline at end of file diff --git a/src/comms/gui/panels/sample_panel.py b/src/comms/gui/panels/sample_panel.py index 957442e..2cb2d14 100644 --- a/src/comms/gui/panels/sample_panel.py +++ b/src/comms/gui/panels/sample_panel.py @@ -58,4 +58,12 @@ def sync_tracker(self) -> None: def write(self, out_dir: Path) -> Path: path = out_dir / 'sample_sheet.tsv' path.write_text(self.sample_sheet_text(), encoding='utf-8') - return path \ No newline at end of file + return path + + # -- load: populate the sample table and treatment/fraction groups from a loaded sample sheet + def load(self, rows: list, treatments: list[str], fractions: list[str]) -> None: + for t in treatments: + self._state.add_treatment(t) + for f in fractions: + self._state.add_fraction(f) + self._state.sample_model.set_rows(rows) \ No newline at end of file From c420a38d86998b673618d74688a0445fe5f31442 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 10:19:16 +0100 Subject: [PATCH 034/108] feat(validate): add non-raising binary checks - Modified utils/validate.py to add non-raising Crux/TRFP binary checks for the experiment GUI to use. --- src/comms/utils/validate.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/comms/utils/validate.py b/src/comms/utils/validate.py index a820b41..b4c050c 100644 --- a/src/comms/utils/validate.py +++ b/src/comms/utils/validate.py @@ -183,4 +183,14 @@ def _run(cmd: list[str]) -> Optional[str]: output = _run([mono, str(trfp_path), '--version']) if output is None: return None - return _parse_version(output) \ No newline at end of file + return _parse_version(output) + +# -- probe_crux: returns Path to Crux binary if found under bin_dir, else None (non-raising version for experiment GUI) +def probe_crux(bin_dir: Path) -> Optional[Path]: + result = _select_best(_find_all_crux(bin_dir), _get_crux_version) + return result[0] if result else None + +# -- probe_trfp: returns Path to ThermoRawFileParser binary if found under bin_dir, else None (non-raising version for experiment GUI) +def probe_trfp(bin_dir: Path) -> Optional[Path]: + result = _select_best(_find_all_trfp(bin_dir), _get_trfp_version) + return result[0] if result else None \ No newline at end of file From 370a079a60d6dee478de5842cd04a88ccc56b9ea Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 10:23:21 +0100 Subject: [PATCH 035/108] feat(readiness): update command readiness probes - Modified utils/readiness.py to update readiness probes to include Crux and ThermoRawFileParser for appropriate commands. --- src/comms/utils/readiness.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/comms/utils/readiness.py b/src/comms/utils/readiness.py index ba46841..ff1d1b9 100644 --- a/src/comms/utils/readiness.py +++ b/src/comms/utils/readiness.py @@ -12,17 +12,20 @@ def missing_requirements( has_sample_sheet, has_organism_prefix, multispecies, - has_organism_tags + has_organism_tags, + has_trfp, + has_crux, + has_r_deps, ) -> dict[str, list[str]]: '''Return, per command, the human-readable inputs still missing to run it.''' base = { - 'convert': [('data files', has_data)], - 'index': [('database', has_database)], - 'search': [('data files', has_data), ('database', has_database)], - 'rescore': [('database', has_database)] + [('organism patterns', has_organism_tags)], - 'lfq': [('sample sheet', has_sample_sheet), ('data files', has_data)], - 'quantify': [('database', has_database)], - 'report': [('sample sheet', has_sample_sheet), ('organism prefix', has_organism_prefix)], + 'convert': [('data files', has_data), ('ThermoRawFileParser', has_trfp)], + 'index': [('database', has_database), ('Crux', has_crux)], + 'search': [('data files', has_data), ('database', has_database), ('Crux', has_crux)], + 'rescore': [('database', has_database)] + [('organism patterns', has_organism_tags)] + [('Crux', has_crux)], + 'lfq': [('sample sheet', has_sample_sheet), ('data files', has_data), ('Crux', has_crux)], + 'quantify': [('database', has_database), ('Crux', has_crux)], + 'report': [('sample sheet', has_sample_sheet), ('organism prefix', has_organism_prefix), ('R dependencies', has_r_deps)], } base['pipeline'] = [item for items in base.values() for item in items] return {cmd: [name for name, ok in items if not ok] for cmd, items in base.items()} \ No newline at end of file From 1ae2e4c47aec9ce3197a932a59ebfcd8ae9811b1 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 10:26:38 +0100 Subject: [PATCH 036/108] feat(gui): update readiness_panel - Modified gui/panels/readiness_panel.py to add a new dependencies box which shows if R dependencies, Crux, ThermoRawFileParser are present, and includes these checks in the command readiness probes. --- src/comms/gui/panels/readiness_panel.py | 109 ++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/src/comms/gui/panels/readiness_panel.py b/src/comms/gui/panels/readiness_panel.py index 8c33c2c..390bf93 100644 --- a/src/comms/gui/panels/readiness_panel.py +++ b/src/comms/gui/panels/readiness_panel.py @@ -3,13 +3,17 @@ ''' # -- Import external dependencies +from pathlib import Path from PySide6.QtCore import Qt from PySide6.QtWidgets import ( - QWidget, QVBoxLayout, QGridLayout, QGroupBox, QLabel, + QApplication, QFileDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QMessageBox, QPushButton, QVBoxLayout, QWidget, ) # -- Import internal functions +from comms.utils.installrdeps import check_r_dependencies, install_r_dependencies +from comms.utils.paths import repoBinDir from comms.utils.readiness import COMMANDS, missing_requirements +from comms.utils.validate import probe_crux, probe_trfp from comms.gui.widgets.status_indicator import StatusIndicator from comms.gui.status import PanelStatus @@ -24,7 +28,10 @@ def __init__(self, experiment, sample, config, parent=None): self._experiment = experiment self._sample = sample self._config = config - + # -- cached dependency state, refreshed by _refresh_dependencies() rather than on every refresh() + self._r_deps_status: dict | None = None + self._crux_found = False + self._trfp_found = False layout = QVBoxLayout(self) box = QGroupBox('Command readiness') grid = QGridLayout(box) @@ -43,11 +50,34 @@ def __init__(self, experiment, sample, config, parent=None): self._indicators[command] = indicator self._details[command] = detail + deps_box = QGroupBox('Dependencies') + deps_layout = QVBoxLayout(deps_box) + summary_row = QHBoxLayout() + self._deps_indicator = StatusIndicator() + self._deps_label = QLabel() + summary_row.addWidget(self._deps_indicator, 0, Qt.AlignmentFlag.AlignVCenter) + summary_row.addWidget(self._deps_label, 1) + deps_layout.addLayout(summary_row) + + buttons_row = QHBoxLayout() + self._install_deps_btn = QPushButton('Install R dependencies') + self._install_deps_btn.clicked.connect(self._install_deps) + self._locate_trfp_btn = QPushButton('Locate TRFP…') + self._locate_trfp_btn.clicked.connect(self._set_bin_dir) + self._locate_crux_btn = QPushButton('Locate Crux…') + self._locate_crux_btn.clicked.connect(self._set_bin_dir) + buttons_row.addWidget(self._install_deps_btn) + buttons_row.addWidget(self._locate_trfp_btn) + buttons_row.addWidget(self._locate_crux_btn) + deps_layout.addLayout(buttons_row) + layout.addWidget(box) + layout.addWidget(deps_box) layout.addStretch(1) + self._refresh_dependencies() self.refresh() - # -- _state: gather the booleans the readiness model needs from the source panels + # -- _state: gather the booleans the readiness model needs from the source panels and cached dependency state def _state(self) -> dict: return dict( has_data=len(self._sample.data_files()) > 0, @@ -56,13 +86,16 @@ def _state(self) -> dict: has_organism_prefix=bool(self._config.organism_prefix()), multispecies=self._config.analysis_mode() == 'multi', has_organism_tags=self._config.has_organism_patterns(), + has_trfp=self._trfp_found, + has_crux=self._crux_found, + has_r_deps=self._r_deps_status is not None and not self._r_deps_status['missing'], ) - # -- refresh: recompute readiness and repaint each row + # -- refresh: recompute command-row readiness and repaint each row (reads cached dependency state; call refresh_dependencies() to re-probe) def refresh(self) -> None: missing = missing_requirements(**self._state()) for command in COMMANDS: - gaps = list(dict.fromkeys(missing[command])) # dedupe, keep order (pipeline repeats) + gaps = list(dict.fromkeys(missing[command])) ready = not gaps self._indicators[command].setStatus( PanelStatus.COMPLETE if ready else PanelStatus.UNEDITED @@ -71,4 +104,68 @@ def refresh(self) -> None: self._details[command].setText(text) tooltip = 'Ready to run' if ready else 'Missing: ' + ', '.join(gaps) self._indicators[command].setToolTip(tooltip) - self._details[command].setToolTip(tooltip) \ No newline at end of file + self._details[command].setToolTip(tooltip) + + # -- refresh_dependencies: public entry point for main_window.py to call after the bin_dir field changes + def refresh_dependencies(self) -> None: + self._refresh_dependencies() + self.refresh() + + # -- _refresh_dependencies: re-probe R deps / Crux / TRFP, repaint the Dependencies box, and cache results for _state() + def _refresh_dependencies(self) -> None: + self._r_deps_status = check_r_dependencies() + bin_dir = repoBinDir(experiment_bin_dir=self._experiment.bin_dir()) + self._crux_found = probe_crux(bin_dir) is not None + self._trfp_found = probe_trfp(bin_dir) is not None + parts = [] + if self._r_deps_status is None: + parts.append('Rscript not found on PATH') + elif self._r_deps_status['missing']: + parts.append(f"{len(self._r_deps_status['missing'])} R package(s) missing") + else: + parts.append('R packages OK') + parts.append('Crux OK' if self._crux_found else 'Crux not found') + parts.append('TRFP OK' if self._trfp_found else 'TRFP not found') + self._deps_label.setText(' · '.join(parts)) + tooltip_lines = [f'Searched for Crux/TRFP in: {bin_dir}'] + if self._r_deps_status and self._r_deps_status['missing']: + tooltip_lines.append(f"Missing R packages: {', '.join(self._r_deps_status['missing'])}") + tooltip = '\n'.join(tooltip_lines) + self._deps_label.setToolTip(tooltip) + r_ok = self._r_deps_status is not None and not self._r_deps_status['missing'] + all_ok = r_ok and self._crux_found and self._trfp_found + none_ok = not r_ok and not self._crux_found and not self._trfp_found + if all_ok: + self._deps_indicator.setStatus(PanelStatus.COMPLETE) + elif none_ok: + self._deps_indicator.setStatus(PanelStatus.UNEDITED) + else: + self._deps_indicator.setStatus(PanelStatus.INCOMPLETE) + self._deps_indicator.setToolTip(tooltip) + + can_install = self._r_deps_status is not None and bool(self._r_deps_status['missing']) + self._install_deps_btn.setEnabled(can_install) + + # -- _set_bin_dir: open a directory picker and write the chosen path into the experiment panel's bin_dir field + def _set_bin_dir(self) -> None: + current = self._experiment.bin_dir() + start_dir = str(current) if current else '' + chosen = QFileDialog.getExistingDirectory( + self, 'Select bin directory (containing Crux and/or ThermoRawFileParser)', start_dir, + ) + if not chosen: + return + self._experiment.set_bin_dir(Path(chosen)) + self.refresh_dependencies() + + def _install_deps(self) -> None: + QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) + try: + ok = install_r_dependencies() + finally: + QApplication.restoreOverrideCursor() + self.refresh_dependencies() + if ok: + QMessageBox.information(self, 'R dependencies', 'All comms report R dependencies are installed.') + else: + QMessageBox.warning(self, 'R dependencies', 'Some R dependencies could not be installed. Check the terminal log for details.') \ No newline at end of file From 65666add68ba0475d49a905598965da54ee65c13 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 10:29:09 +0100 Subject: [PATCH 037/108] feat(gui): add setter function for bin directory - Modified gui/panels/experiment_panel.py to add a setter function for the bin_dir field. --- src/comms/gui/panels/experiment_panel.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/comms/gui/panels/experiment_panel.py b/src/comms/gui/panels/experiment_panel.py index 7e14346..2664e7f 100644 --- a/src/comms/gui/panels/experiment_panel.py +++ b/src/comms/gui/panels/experiment_panel.py @@ -117,6 +117,11 @@ def bin_dir(self) -> Path | None: text = self._bin.text().strip() return Path(text) if text else None + # -- set_bin_dir: write a chosen bin directory into the field and mark the panel changed + def set_bin_dir(self, path: Path) -> None: + self._bin.setText(str(path)) + self.changed.emit() + def database_path(self) -> Path | None: text = self._database.text().strip() return Path(text) if text else None From 2e60d285148a57d0826de3e35bc3ac3522efd914 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 10:29:48 +0100 Subject: [PATCH 038/108] style(gui): reorder sections of experiment panel - Modified gui/panels/readiness_panel.py to place dependencies box above the command readiness box. --- src/comms/gui/panels/readiness_panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/gui/panels/readiness_panel.py b/src/comms/gui/panels/readiness_panel.py index 390bf93..22a1f50 100644 --- a/src/comms/gui/panels/readiness_panel.py +++ b/src/comms/gui/panels/readiness_panel.py @@ -71,8 +71,8 @@ def __init__(self, experiment, sample, config, parent=None): buttons_row.addWidget(self._locate_crux_btn) deps_layout.addLayout(buttons_row) - layout.addWidget(box) layout.addWidget(deps_box) + layout.addWidget(box) layout.addStretch(1) self._refresh_dependencies() self.refresh() From 73bc0bc505ba8ca6c036d674767802fe549528dc Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 11:37:41 +0100 Subject: [PATCH 039/108] fix(experiment): resolve root directory for paths - Modified commands/experiment.py to update _existing_experiment function to resolve root directory rather than using relative paths, so saved file paths are correct when editing an existing experiment. --- src/comms/commands/experiment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index df50ace..961598a 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -19,6 +19,7 @@ # -- _existing_experiment: returns (root, comms_dir, metadata, config, sample_rows) if experiment_dir already holds a saved experiment else None def _existing_experiment(experiment_dir: Path): root, comms_dir = _normalise_dirs(experiment_dir) + root = Path(root).resolve() meta_path = comms_dir / 'experiment.toml' config_path = comms_dir / 'config.toml' if not (meta_path.exists() and config_path.exists()): From 5433408494a8c3904e13f4e6c9f15ee3cbbcdfc3 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 14:50:44 +0100 Subject: [PATCH 040/108] fix(gui): fix existing data not persisting - Modified gui/panels/sample_panel.py to fix issue with existing data files/samples not persisting upon editing existing experiment. --- src/comms/gui/panels/sample_panel.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/comms/gui/panels/sample_panel.py b/src/comms/gui/panels/sample_panel.py index 2cb2d14..77b7573 100644 --- a/src/comms/gui/panels/sample_panel.py +++ b/src/comms/gui/panels/sample_panel.py @@ -61,9 +61,16 @@ def write(self, out_dir: Path) -> Path: return path # -- load: populate the sample table and treatment/fraction groups from a loaded sample sheet - def load(self, rows: list, treatments: list[str], fractions: list[str]) -> None: + def load(self, rows: list, treatments: list[str], fractions: list[str], data_files: list[str] | None = None) -> None: for t in treatments: self._state.add_treatment(t) for f in fractions: self._state.add_fraction(f) + if data_files: + by_name = {Path(p).name: str(p) for p in data_files} + for row in rows: + if not row.source_path: + match = by_name.get(row.raw_file) + if match: + row.source_path = match self._state.sample_model.set_rows(rows) \ No newline at end of file From a63598aa199c2b77b497bdb7f1b0a8a3cc36ca7b Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 14:53:21 +0100 Subject: [PATCH 041/108] fix(gui): fix existing data not persisting - Modified gui/main_window.py and commands/experiment.py to pass existing files/samples not persisting upon editing existing experiment. --- src/comms/commands/experiment.py | 5 +++-- src/comms/gui/main_window.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index 961598a..2f76308 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -88,7 +88,7 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: if edit_mode: base_dir = root else: - base_dir = Path(logMsg.input('Save experiment to (directory)', default=str(experiment_dir) if experiment_dir else None, show_default=experiment_dir is not None)).expanduser() + base_dir = Path(logMsg.input('Save experiment to directory (default: ".")', default=str(experiment_dir) if experiment_dir else '.', show_default=experiment_dir is not None)).expanduser() bin_dir = logMsg.input('Bin directory (blank to auto-resolve)', default=metadata.get('experiment', {}).get('bin_dir', ''), show_default=edit_mode).strip() database = logMsg.input('Combined database FASTA', default=metadata.get('files', {}).get('database', ''), show_default=edit_mode).strip() existing_treatments = sorted({r.treatment for r in existing_rows if r.treatment}) @@ -100,7 +100,8 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: raise SystemExit(1) input_dir = Path(logMsg.input('Directory of .RAW / .mzML files')).expanduser() - input_files = _prompt_list('data file') + existing_data_files = metadata.get('files', {}).get('data', []) if edit_mode else None + input_files = _prompt_list('data file', existing=existing_data_files) files = [] for f in input_files: f = Path(Path(f).expanduser()) diff --git a/src/comms/gui/main_window.py b/src/comms/gui/main_window.py index 941e0a3..fd592d0 100644 --- a/src/comms/gui/main_window.py +++ b/src/comms/gui/main_window.py @@ -91,7 +91,8 @@ def _load_existing(self, experiment_dir: Path) -> None: self.config.load_from_config(config, report_meta) treatments = sorted({r.treatment for r in rows if r.treatment}) fractions = sorted({r.fraction for r in rows if r.fraction}) - self.sample.load(rows, treatments, fractions) + data_files = metadata.get('files', {}).get('data', []) + self.sample.load(rows, treatments, fractions, data_files=data_files) self._log.info(f'Loaded existing experiment from {comms_dir}') def closeEvent(self, event) -> None: From 78743372192d1b0adff727c9baac4d0c3add8597 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 14:57:33 +0100 Subject: [PATCH 042/108] fix: fix clip_n_met config not storing as bool - Modified commands/config.py to ensure clip_n_met value is stored as a boolean, not a string. - Modified commands/experiment.py, gui/panels/config_panel.py and utils/crux.py to update parsing the clip_n_met key in config. --- src/comms/commands/config.py | 2 +- src/comms/commands/experiment.py | 2 +- src/comms/gui/panels/config_panel.py | 5 ++++- src/comms/utils/crux.py | 9 ++++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/comms/commands/config.py b/src/comms/commands/config.py index 4768185..34a4a6b 100644 --- a/src/comms/commands/config.py +++ b/src/comms/commands/config.py @@ -323,7 +323,7 @@ def _apply_protocol_flags( logMsg.debug(f'{'--low-res' if low_res else '--high-res'} applied: mz_bin_width: {cfg["search"]["mz_bin_width"]}, score_function: {cfg["search"]["score_function"]}') if clip_met is not None: cfg.setdefault('index', {}) - cfg['index']['clip_n_met'] = 'true' if clip_met else 'false' + cfg['index']['clip_n_met'] = bool(clip_met) logMsg.debug(f'{'--clip-met' if clip_met else '--no-clip-met'} applied: {cfg['index']['clip_n_met']}') return cfg diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index 2f76308..6df7d05 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -141,7 +141,7 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: phos=_confirm('STY phosphorylation (variable)?', default=bool(re.search(r'STY\+79\.966331', index_cfg.get('mods_spec', ''))) if edit_mode else False), n_cyc=_confirm('N-terminal Gln cyclisation?', default=bool(index_cfg.get('nterm_peptide_mods_spec', '')) if edit_mode else True), n_ace=_confirm('Protein N-terminal acetylation?', default=bool(index_cfg.get('nterm_protein_mods_spec', '')) if edit_mode else True), - clip_met=_confirm('Clip N-terminal methionine?', default=index_cfg.get('clip_n_met', True) if edit_mode else True), + clip_met=_confirm('Clip N-terminal methionine?', default=(str(index_cfg.get('clip_n_met', True)).strip().lower() == 'true') if edit_mode else True), low_res=_confirm('Low-resolution instrument (ion trap)?', default=(search_cfg.get('score_function') == 'combined-p-value') if edit_mode else False), ) cfg.setdefault('index', {})['custom_mods'] = index_cfg.get('custom_mods', '') diff --git a/src/comms/gui/panels/config_panel.py b/src/comms/gui/panels/config_panel.py index 5b371e5..fa04af9 100644 --- a/src/comms/gui/panels/config_panel.py +++ b/src/comms/gui/panels/config_panel.py @@ -332,7 +332,10 @@ def load_from_config(self, cfg: dict, report_meta: dict) -> None: self._phos.setChecked(bool(re.search(r'STY\+79\.966331', index_cfg.get('mods_spec', '')))) self._n_cyc.setChecked(bool(index_cfg.get('nterm_peptide_mods_spec', ''))) self._n_ace.setChecked(bool(index_cfg.get('nterm_protein_mods_spec', ''))) - self._clip_met.setChecked(index_cfg.get('clip_n_met', True)) + clip_met_value = index_cfg.get('clip_n_met', True) + if isinstance(clip_met_value, str): + clip_met_value = clip_met_value.strip().lower() == 'true' + self._clip_met.setChecked(bool(clip_met_value)) self._custom.setText(index_cfg.get('custom_mods', '')) self._res.setCurrentIndex(1 if search_cfg.get('mz_bin_width') == MZ_BIN_WIDTH_LOW_RES else 0) diff --git a/src/comms/utils/crux.py b/src/comms/utils/crux.py index 53ffcb2..1d4b342 100644 --- a/src/comms/utils/crux.py +++ b/src/comms/utils/crux.py @@ -230,8 +230,7 @@ def lfq(crux_bin, psm_files, mzml_files, out_dir, fileroot, config) -> bool: return ok # -- _tomlToCrux: helper function returning 'T' if True and 'F' if False -def _tomlToCrux(val: bool): - if val: - return 'T' - else: - return 'F' \ No newline at end of file +def _tomlToCrux(val) -> str: + if isinstance(val, str): + val = val.strip().lower() == 'true' + return 'T' if val else 'F' \ No newline at end of file From 3692aafc63d09eaac69a3ac08bb27b6de01e9340 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 15:02:34 +0100 Subject: [PATCH 043/108] fix(gui): fix dependencies panel not refreshing - Modified gui/panel/experiment_panel.py to add a new binDirChanged signal for triggering the dependency panel refresh, and wired this up. - Modified gui/main_window.py to connect the new signal. - Modified gui/readiness_panel.py to remove redundant refresh signal which is automatically triggered now. --- src/comms/gui/main_window.py | 1 + src/comms/gui/panels/experiment_panel.py | 5 +++++ src/comms/gui/panels/readiness_panel.py | 1 - 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/comms/gui/main_window.py b/src/comms/gui/main_window.py index fd592d0..d35bcf6 100644 --- a/src/comms/gui/main_window.py +++ b/src/comms/gui/main_window.py @@ -63,6 +63,7 @@ def __init__(self, experiment_dir: Path | None = None, parent=None): self.sample.contentChanged.connect(self.readiness.refresh) self.config.changed.connect(self.readiness.refresh) self.experiment.changed.connect(self.readiness.refresh) + self.experiment.binDirChanged.connect(self.readiness.refresh_dependencies) # paint the initial (unedited) icons self.tabs.setTabIcon(self._sample_index, status_icon(self.sample.tracker.status)) diff --git a/src/comms/gui/panels/experiment_panel.py b/src/comms/gui/panels/experiment_panel.py index 2664e7f..b01e054 100644 --- a/src/comms/gui/panels/experiment_panel.py +++ b/src/comms/gui/panels/experiment_panel.py @@ -17,6 +17,7 @@ # -- Define class ExperimentPanel to collect experiment name and base output directory class ExperimentPanel(QWidget): changed = Signal() + binDirChanged = Signal() def __init__(self, parent=None): super().__init__(parent) @@ -59,6 +60,7 @@ def __init__(self, parent=None): self._bin.setMinimumWidth(360) self._bin.setPlaceholderText('optional: directory containing Crux / ThermoRawFileParser') self._bin.textChanged.connect(self.changed) + self._bin.editingFinished.connect(self.binDirChanged) bin_browse = QPushButton('Select directory') bin_browse.clicked.connect(self._browse_bin) bin_row = QWidget() @@ -121,6 +123,7 @@ def bin_dir(self) -> Path | None: def set_bin_dir(self, path: Path) -> None: self._bin.setText(str(path)) self.changed.emit() + self.binDirChanged.emit() def database_path(self) -> Path | None: text = self._database.text().strip() @@ -164,6 +167,8 @@ def load_from_metadata(self, base_dir: Path, metadata: dict) -> None: if bin_dir: self._bin.setText(str(bin_dir)) self.changed.emit() + if bin_dir: + self.binDirChanged.emit() def is_valid(self) -> bool: return bool(self.experiment_name()) and self.base_dir() is not None and self.database_path() is not None \ No newline at end of file diff --git a/src/comms/gui/panels/readiness_panel.py b/src/comms/gui/panels/readiness_panel.py index 22a1f50..90be63e 100644 --- a/src/comms/gui/panels/readiness_panel.py +++ b/src/comms/gui/panels/readiness_panel.py @@ -156,7 +156,6 @@ def _set_bin_dir(self) -> None: if not chosen: return self._experiment.set_bin_dir(Path(chosen)) - self.refresh_dependencies() def _install_deps(self) -> None: QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) From ee7ae30297b87eb03a19fb1ab1c8317810193354 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 15:14:52 +0100 Subject: [PATCH 044/108] test(experiment): update experiment unit tests - Modified tests/unit/test_experiment.py to change patched functions to match current implementation. --- tests/unit/test_experiment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_experiment.py b/tests/unit/test_experiment.py index ba65d35..aae5246 100644 --- a/tests/unit/test_experiment.py +++ b/tests/unit/test_experiment.py @@ -67,8 +67,8 @@ def test_writes_three_files(self, tmp_path): 'MOCK', # treatment for sample_mock.mzML 'WCL', # fraction for sample_mock.mzML ]) - with patch('typer.prompt', side_effect=lambda *a, **k: next(prompts)), \ - patch('typer.confirm', return_value=False): + with patch('comms.utils.log.logMsg.input', side_effect=lambda *a, **k: next(prompts)), \ + patch('comms.commands.experiment._confirm', return_value=False): run_experiment_headless() out = base / 'comms' assert (out / 'sample_sheet.tsv').exists() From 7331e401fa0e28637c9aaaaad7f917aac481317e Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 16:10:01 +0100 Subject: [PATCH 045/108] feat(settings): add config value resolver utility - Modified utils/settings.py to add a config value resolution utility function. --- src/comms/utils/settings.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/comms/utils/settings.py b/src/comms/utils/settings.py index 113285f..82d8ca3 100644 --- a/src/comms/utils/settings.py +++ b/src/comms/utils/settings.py @@ -8,11 +8,14 @@ from pathlib import Path from platformdirs import user_config_dir from rich import print -from typing import Optional +from typing import Optional, TypeVar # Import internal classes/functions from comms.utils.log import logMsg +# -- Define TypeVar T +T = TypeVar('T') + # -- globalConfigPath: returns Path to OS-appropriate config file def globalConfigPath() -> Path: ''' @@ -68,6 +71,18 @@ def resolveConfig(comms_dir: Optional[Path] = None) -> tuple[dict, str]: logMsg.debug('Using bundled default config') return loadDefaultConfig(), 'bundled defaults' +# -- resolve_config_value: returns override if given, else the config.toml value at [section].key +def resolve_config_value(cfg: dict, section: str, key: str, override: Optional[T]) -> T: + ''' + Return override if given (not None), else cfg[section][key + ''' + if override is not None: + return override + try: + return cfg[section][key] + except KeyError: + raise KeyError(f'No value for [{section}].{key} in config, and no override given') from None + # -- initComms: returns None, but prints start-up message to terminal def initComms() -> None: ''' From d66e297c69d8ce069868d773ac35933dbe7c1fcd Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 16:30:06 +0100 Subject: [PATCH 046/108] feat(modspec): add modspec utility functions - Added utils/modspec.py to move modification config editing functions to a separate utility file for shared use. - Modified commands/config.py, commands/experiment.py and gui/panels/config_panel.py to all reference new utility functions and remove from commands/config.py. --- src/comms/commands/config.py | 174 ------------------------- src/comms/commands/experiment.py | 7 +- src/comms/gui/panels/config_panel.py | 11 +- src/comms/utils/modspec.py | 181 +++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 182 deletions(-) create mode 100644 src/comms/utils/modspec.py diff --git a/src/comms/commands/config.py b/src/comms/commands/config.py index 34a4a6b..0a1ed12 100644 --- a/src/comms/commands/config.py +++ b/src/comms/commands/config.py @@ -17,25 +17,6 @@ from comms.utils.log import logMsg from comms.utils.settings import loadDefaultConfig, globalConfigPath -# -- Define modification constants -CARBAMIDOMETHYL_MOD = 'C+57.0215' # static carbamidomethylation of Cys -MET_OX_MOD = '1M+15.9949' # variable Met oxidation -PHOSPHO_MOD = '1STY+79.966331' # variable STY phosphorylation -NCYC_MOD = '1Q-17.027' # N-terminal Gln cyclisation -NACE_MOD = '1X+42.011' # N-terminal protein acetylation -MANAGED_MOD_PATTERNS: dict[str, str] = { - r'^\d*C[+\-]': '--iodo / --no-iodo', - r'^\d*M\+15\.9949': '--ox / --no-ox', - r'^\d*STY\+79\.966331': '--phos / --no-phos', -} # mods that --custom is not allowed to duplicate (maps the residue/pattern that identifies each managed mod to its flag name) - -# -- Define resolution constants -MZ_BIN_WIDTH_HIGH_RES = 0.02 # high-resolution instruments (default) -MZ_BIN_WIDTH_LOW_RES = 1.0005079 # low-resolution instruments -SCORE_FUNC_HIGH_RES = 'xcorr' # high-resolution instruments (default) -SCORE_FUNC_LOW_RES = 'combined-p-value' # low-resolution instruments - - # ========================= # # DEFINE CONFIG SUBCOMMANDS # # ========================= # @@ -255,161 +236,6 @@ def _printTable(user_config: dict, default_config: dict): # ========================= # # DEFINE CONFIG SET HELPERS # # ========================= # -# -- _apply_protocol_flags: returns dictionary of config options -def _apply_protocol_flags( - cfg: dict, - *, - iodo: bool | None = None, - ox: bool | None = None, - phos: bool | None = None, - n_cyc: bool | None = None, - n_ace: bool | None = None, - clip_met: bool | None = None, - low_res: bool | None = None, -) -> dict: - ''' - Apply protocol flags to a config dictionary and return it - iodo — owns the Cys slot in index.fixed_mods exclusivel - ox — adds/removes 1M+15.9949 in index.mods_spec - phos — adds/removes 1STY+79.966331 in index.mods_spec - n_cyc — adds/removes 1Q-17.027 in index.nterm_peptide_mods_spec - n_ace — adds/removes 1X+42.011 in index.nterm_protein_mod_spec - low_res — sets search.mz_bin_width and index.score_function - ''' - cfg.setdefault('search', {}) - cfg['index'].setdefault('fixed_mods', '') - cfg['index'].setdefault('nterm_peptide_mods_spec', '') - cfg['index'].setdefault('nterm_protein_mods_spec', '') - if iodo is not None: - cfg['index']['fixed_mods'] = _apply_iodo(cfg['index'].get('fixed_mods', ''), iodo=iodo) - logMsg.debug(f'{'--iodo' if iodo else '--no-iodo'} applied: fixed_mods updated to {cfg['index']['fixed_mods']}') - if ox is not None: - spec = cfg['index'].get('mods_spec', '') - if ox: - cfg['index']['mods_spec'] = _apply_mod(spec, mod=MET_OX_MOD) - else: - cfg['index']['mods_spec'] = _apply_mod(spec, mod='', exclusive_pattern=r'^\d*M\+15\.9949') - logMsg.debug(f'{'--ox' if ox else '--no-ox'} applied: mods_spec updated to {cfg['index']['mods_spec']}') - if phos is not None: - spec = cfg['index'].get('mods_spec', '') - if phos: - cfg['index']['mods_spec'] = _apply_mod(spec, mod=PHOSPHO_MOD) - else: - cfg['index']['mods_spec'] = _apply_mod(spec, mod='', exclusive_pattern=r'^\d*STY\+79\.966331') - logMsg.debug(f'{'--phos' if phos else '--no-phos'} applied: mods_spec updated to {cfg['index']['mods_spec']}') - if n_cyc is not None: - spec = cfg['index'].get('nterm_peptide_mods_spec', '') - if n_cyc: - cfg['index']['nterm_peptide_mods_spec'] = _apply_mod(spec, mod=NCYC_MOD) - else: - cfg['index']['nterm_peptide_mods_spec'] = _apply_mod(spec, mod='', exclusive_pattern=r'^\d*Q\-17\.027') - logMsg.debug(f'{'--n-cyc' if n_cyc else '--no-n-cyc'} applied: nterm_peptide_mods_spec updated to {cfg['index']['nterm_peptide_mods_spec']}') - if n_ace is not None: - spec = cfg['index'].get('nterm_protein_mods_spec', '') - if n_ace: - cfg['index']['nterm_protein_mods_spec'] = _apply_mod(spec, mod=NACE_MOD) - else: - cfg['index']['nterm_protein_mods_spec'] = _apply_mod(spec, mod='', exclusive_pattern=r'^\d*X\+42\.011') - logMsg.debug(f'{'--n-ace' if n_ace else '--no-n-ace'} applied: nterm_protein_mods_spec updated to {cfg['index']['nterm_protein_mods_spec']}') - if low_res is not None: - if low_res: - cfg['search']['mz_bin_width'] = MZ_BIN_WIDTH_LOW_RES - cfg['search']['score_function'] = SCORE_FUNC_LOW_RES - logMsg.debug(f'--low-res applied: mz_bin_width: {cfg["search"]["mz_bin_width"]}, score_function: {cfg["search"]["score_function"]}') - else: - cfg['search']['mz_bin_width'] = MZ_BIN_WIDTH_HIGH_RES - cfg['search']['score_function'] = SCORE_FUNC_HIGH_RES - logMsg.debug(f'--high-res applied: mz_bin_width: {cfg["search"]["mz_bin_width"]}, score_function: {cfg["search"]["score_function"]}') - logMsg.debug(f'{'--low-res' if low_res else '--high-res'} applied: mz_bin_width: {cfg["search"]["mz_bin_width"]}, score_function: {cfg["search"]["score_function"]}') - if clip_met is not None: - cfg.setdefault('index', {}) - cfg['index']['clip_n_met'] = bool(clip_met) - logMsg.debug(f'{'--clip-met' if clip_met else '--no-clip-met'} applied: {cfg['index']['clip_n_met']}') - return cfg - -# -- _apply_mod: returns mod_spec string -def _apply_mod(mods_spec: str, mod: str, exclusive_pattern: str | None = None) -> str: - ''' - Add or remove a mod entry in a Tide mods_spec string. - ''' - # Split on commas, discard empty strings from a blank mods_spec - entries = [e.strip() for e in mods_spec.split(',') if e.strip()] - if exclusive_pattern: - pattern = re.compile(exclusive_pattern, re.IGNORECASE) - entries = [e for e in entries if not pattern.match(e)] - elif mod == '': - pass - else: - entries = [e for e in entries if e != mod] - if mod: - entries = [mod] + entries - return ','.join(entries) - -# -- _apply-iodo: returns fixed_mods string -def _apply_iodo(fixed_mods: str, iodo: bool) -> str: - ''' - Add or remove the carbamidomethylation Cys mod in a Tide fixed_mods string - ''' - # Split on commas, discard empty strings from a blank mods_spec - entries = [e.strip() for e in fixed_mods.split(',') if e.strip()] - entries = [e for e in entries if e != CARBAMIDOMETHYL_MOD and e != 'C+0'] - if iodo: - entries = [CARBAMIDOMETHYL_MOD] + entries - else: - entries = ['C+0'] + entries # Crux automatically adds cysteine carbamidomethylation unless this string present - result = ','.join(entries) - return result - -def _apply_custom_mod(custom_mods: str, new_entry: str) -> str: - ''' - Add a custom mod entry to the custom_mods string, or clear all custom mods if new_entry is an empty string - ''' - if new_entry == '': - return '' - # Check against managed mod patterns - for pattern, flag_name in MANAGED_MOD_PATTERNS.items(): - if re.match(pattern, new_entry, re.IGNORECASE): - logMsg.warn(f'{new_entry} is managed by the {flag_name} flag, ignoring') - return custom_mods - # Split on commas, discard empty strings from a blank mods_spec - entries = [e.strip() for e in custom_mods.split(',') if e.strip()] - if new_entry not in entries: - entries.append(new_entry) - out_str = ','.join(entries) - logMsg.debug(f'custom_mods updated: {out_str}') - return out_str - -# -- _apply_organism: returns config dict with organism section replaced -def _apply_organism(cfg: dict, organism: dict[str, str]) -> dict: - ''' - Replace the [organism] section of the user config with the supplied dictionary. - ''' - cfg['organism'] = organism - logMsg.debug(f'organism section set to {organism}') - return cfg - -# -- _parse_organism_arg: returns dict parsed from list of 'Key=Pattern' strings -def _parse_organism_arg(pairs: list[str]) -> dict[str, str]: - ''' - Parse a list of 'Label=Pattern' strings into a dict. - ''' - result = {} - for item in pairs: - if '=' not in item: - logMsg.error(f'Invalid organism argument {item} (expected format: Organism=Pattern)') - raise SystemExit(1) - key, _, pattern = item.partition('=') - key = ''.join(key.split()) - pattern = ''.join(pattern.split()) - if not key: - logMsg.error(f'Empty label in organism argument: {item}') - raise SystemExit(1) - if not pattern: - logMsg.error(f'Empty pattern in organism argument: {item}') - raise SystemExit(1) - result[key] = pattern - return result - # _mod_summary_line: prints a s def _mod_summary_line(flag: bool | None, mod: str, key: str): ''' diff --git a/src/comms/commands/experiment.py b/src/comms/commands/experiment.py index 6df7d05..e51b36b 100644 --- a/src/comms/commands/experiment.py +++ b/src/comms/commands/experiment.py @@ -9,10 +9,11 @@ from rich import print # -- Import internal functions -from comms.commands.config import _apply_protocol_flags, _apply_organism, _writeConfigTo +from comms.commands.config import _writeConfigTo from comms.utils.context import _normalise_dirs from comms.utils.installrdeps import check_r_dependencies, install_r_dependencies from comms.utils.log import logMsg +from comms.utils.modspec import apply_protocol_flags, apply_organism from comms.utils.settings import loadDefaultConfig from comms.utils.sheet import SampleRow, render_sample_sheet, parse_sample_sheet @@ -134,7 +135,7 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: index_cfg = config.get('index', {}) search_cfg = config.get('search', {}) cfg = loadDefaultConfig() - cfg = _apply_protocol_flags( + cfg = apply_protocol_flags( cfg, iodo=_confirm('Cysteine carbamidomethylation (static)?', default='C+0' not in index_cfg.get('fixed_mods', '')), ox=_confirm('Methionine oxidation (variable)?', default=bool(re.search(r'M\+15\.9949', index_cfg.get('mods_spec', ''))) if edit_mode else True), @@ -157,7 +158,7 @@ def run_experiment_headless(experiment_dir: Path | None = None) -> None: organisms[label] = pattern else: organisms = {} - cfg = _apply_organism(cfg, organisms) + cfg = apply_organism(cfg, organisms) if multispecies: cfg['percolator']['shared_psm'] = logMsg.input('Shared PSM handling policy', choices=['drop', 'include'], default=config.get('percolator', {}).get('shared_psm', 'drop'), show_choices=True, show_default=True).strip() report_meta = metadata.get('report', {}) diff --git a/src/comms/gui/panels/config_panel.py b/src/comms/gui/panels/config_panel.py index fa04af9..66e0996 100644 --- a/src/comms/gui/panels/config_panel.py +++ b/src/comms/gui/panels/config_panel.py @@ -12,10 +12,11 @@ ) # -- Import internal functions +from comms.commands.config import _writeConfigTo from comms.gui.status import PanelStateTracker from comms.utils.settings import loadDefaultConfig -from comms.commands.config import ( - _apply_protocol_flags, _apply_organism, _apply_custom_mod, _writeConfigTo, MZ_BIN_WIDTH_LOW_RES, +from comms.utils.modspec import ( + apply_protocol_flags, apply_organism, apply_custom_mod, MZ_BIN_WIDTH_LOW_RES, ) # -- Define class ConfigPanel to define a structured form mirroring `comms config set` with an additional analysis type activating the organism table @@ -288,7 +289,7 @@ def summary(self) -> str: # -- build config file and save -- def _build_config(self) -> dict: cfg = loadDefaultConfig() - cfg = _apply_protocol_flags( + cfg = apply_protocol_flags( cfg, iodo=self._iodo.isChecked(), ox=self._ox.isChecked(), @@ -304,13 +305,13 @@ def _build_config(self) -> dict: label: pattern for label, pattern in self._organism_rows() if label and pattern } cfg['percolator']['shared_psm'] = self.shared_policy() - cfg = _apply_organism(cfg, organisms) + cfg = apply_organism(cfg, organisms) cfg.setdefault('index', {}) cfg['index']['custom_mods'] = '' custom = self._custom.text().strip() if custom: for entry in [e.strip() for e in custom.split(',') if e.strip()]: - cfg['index']['custom_mods'] = _apply_custom_mod( + cfg['index']['custom_mods'] = apply_custom_mod( cfg['index']['custom_mods'], entry) return cfg diff --git a/src/comms/utils/modspec.py b/src/comms/utils/modspec.py new file mode 100644 index 0000000..65caa44 --- /dev/null +++ b/src/comms/utils/modspec.py @@ -0,0 +1,181 @@ +''' +comMS shared modification-spec and organism helpers + +Used by commands/config.py (persisting changes to a config file) and commands/index.py +(applying one-off, non-persisted overrides for a single `comms index` run). Every +function here is pure — it takes a value in and returns a new value out. +''' + +# -- Import external dependencies +import re + +# -- Import internal functions +from comms.utils.log import logMsg + +# -- Modification constants +CARBAMIDOMETHYL_MOD = 'C+57.0215' # static carbamidomethylation of Cys +MET_OX_MOD = '1M+15.9949' # variable Met oxidation +PHOSPHO_MOD = '1STY+79.966331' # variable STY phosphorylation +NCYC_MOD = '1Q-17.027' # N-terminal Gln cyclisation +NACE_MOD = '1X+42.011' # N-terminal protein acetylation +MANAGED_MOD_PATTERNS: dict[str, str] = { + r'^\d*C[+\-]': '--iodo / --no-iodo', + r'^\d*M\+15\.9949': '--ox / --no-ox', + r'^\d*STY\+79\.966331': '--phos / --no-phos', +} # mods that --custom is not allowed to duplicate (maps the residue/pattern that identifies each managed mod to its flag name) + +# -- Resolution constants +MZ_BIN_WIDTH_HIGH_RES = 0.02 # high-resolution instruments (default) +MZ_BIN_WIDTH_LOW_RES = 1.0005079 # low-resolution instruments +SCORE_FUNC_HIGH_RES = 'xcorr' # high-resolution instruments (default) +SCORE_FUNC_LOW_RES = 'combined-p-value' # low-resolution instruments + +# -- apply_mod: returns mods_spec string +def apply_mod(mods_spec: str, mod: str, exclusive_pattern: str | None = None) -> str: + ''' + Add or remove a mod entry in a Tide mods_spec string. + ''' + entries = [e.strip() for e in mods_spec.split(',') if e.strip()] + if exclusive_pattern: + pattern = re.compile(exclusive_pattern, re.IGNORECASE) + entries = [e for e in entries if not pattern.match(e)] + elif mod == '': + pass + else: + entries = [e for e in entries if e != mod] + if mod: + entries = [mod] + entries + return ','.join(entries) + +# -- apply_iodo: returns fixed_mods string +def apply_iodo(fixed_mods: str, iodo: bool) -> str: + ''' + Add or remove the carbamidomethylation Cys mod in a Tide fixed_mods string + ''' + entries = [e.strip() for e in fixed_mods.split(',') if e.strip()] + entries = [e for e in entries if e != CARBAMIDOMETHYL_MOD and e != 'C+0'] + if iodo: + entries = [CARBAMIDOMETHYL_MOD] + entries + else: + entries = ['C+0'] + entries # Crux automatically adds cysteine carbamidomethylation unless this string present + return ','.join(entries) + +# -- apply_custom_mod: returns custom_mods string +def apply_custom_mod(custom_mods: str, new_entry: str) -> str: + ''' + Add a custom mod entry to the custom_mods string, or clear all custom mods if new_entry is an empty string + ''' + if new_entry == '': + return '' + for pattern, flag_name in MANAGED_MOD_PATTERNS.items(): + if re.match(pattern, new_entry, re.IGNORECASE): + logMsg.warn(f'{new_entry} is managed by the {flag_name} flag, ignoring') + return custom_mods + entries = [e.strip() for e in custom_mods.split(',') if e.strip()] + if new_entry not in entries: + entries.append(new_entry) + out_str = ','.join(entries) + logMsg.debug(f'custom_mods updated: {out_str}') + return out_str + +# -- apply_organism: returns config dict with organism section replaced +def apply_organism(cfg: dict, organism: dict[str, str]) -> dict: + ''' + Replace the [organism] section of a config dict with the supplied dictionary. + ''' + cfg['organism'] = organism + logMsg.debug(f'organism section set to {organism}') + return cfg + +# -- parse_organism_arg: returns dict parsed from list of 'Key=Pattern' strings +def parse_organism_arg(pairs: list[str]) -> dict[str, str]: + ''' + Parse a list of 'Label=Pattern' strings into a dict. + ''' + result = {} + for item in pairs: + if '=' not in item: + logMsg.error(f'Invalid organism argument {item} (expected format: Organism=Pattern)') + raise SystemExit(1) + key, _, pattern = item.partition('=') + key = ''.join(key.split()) + pattern = ''.join(pattern.split()) + if not key: + logMsg.error(f'Empty label in organism argument: {item}') + raise SystemExit(1) + if not pattern: + logMsg.error(f'Empty pattern in organism argument: {item}') + raise SystemExit(1) + result[key] = pattern + return result + +# -- apply_protocol_flags: returns a config dict with protocol-level flags applied +def apply_protocol_flags( + cfg: dict, + *, + iodo: bool | None = None, + ox: bool | None = None, + phos: bool | None = None, + n_cyc: bool | None = None, + n_ace: bool | None = None, + clip_met: bool | None = None, + low_res: bool | None = None, + missed_cleavages: int | None = None, +) -> dict: + ''' + Apply protocol flags to a config dictionary and return it + iodo — owns the Cys slot in index.fixed_mods exclusively + ox — adds/removes 1M+15.9949 in index.mods_spec + phos — adds/removes 1STY+79.966331 in index.mods_spec + n_cyc — adds/removes 1Q-17.027 in index.nterm_peptide_mods_spec + n_ace — adds/removes 1X+42.011 in index.nterm_protein_mods_spec + low_res — sets search.mz_bin_width and search.score_function + missed_cleavages — sets index.missed_cleavages directly + ''' + cfg.setdefault('search', {}) + cfg.setdefault('index', {}) + cfg['index'].setdefault('fixed_mods', '') + cfg['index'].setdefault('nterm_peptide_mods_spec', '') + cfg['index'].setdefault('nterm_protein_mods_spec', '') + if iodo is not None: + cfg['index']['fixed_mods'] = apply_iodo(cfg['index'].get('fixed_mods', ''), iodo=iodo) + logMsg.debug(f'{"--iodo" if iodo else "--no-iodo"} applied: fixed_mods updated to {cfg["index"]["fixed_mods"]}') + if ox is not None: + spec = cfg['index'].get('mods_spec', '') + if ox: + cfg['index']['mods_spec'] = apply_mod(spec, mod=MET_OX_MOD) + else: + cfg['index']['mods_spec'] = apply_mod(spec, mod='', exclusive_pattern=r'^\d*M\+15\.9949') + logMsg.debug(f'{"--ox" if ox else "--no-ox"} applied: mods_spec updated to {cfg["index"]["mods_spec"]}') + if phos is not None: + spec = cfg['index'].get('mods_spec', '') + if phos: + cfg['index']['mods_spec'] = apply_mod(spec, mod=PHOSPHO_MOD) + else: + cfg['index']['mods_spec'] = apply_mod(spec, mod='', exclusive_pattern=r'^\d*STY\+79\.966331') + logMsg.debug(f'{"--phos" if phos else "--no-phos"} applied: mods_spec updated to {cfg["index"]["mods_spec"]}') + if n_cyc is not None: + spec = cfg['index'].get('nterm_peptide_mods_spec', '') + if n_cyc: + cfg['index']['nterm_peptide_mods_spec'] = apply_mod(spec, mod=NCYC_MOD) + else: + cfg['index']['nterm_peptide_mods_spec'] = apply_mod(spec, mod='', exclusive_pattern=r'^\d*Q\-17\.027') + logMsg.debug(f'{"--n-cyc" if n_cyc else "--no-n-cyc"} applied: nterm_peptide_mods_spec updated to {cfg["index"]["nterm_peptide_mods_spec"]}') + if n_ace is not None: + spec = cfg['index'].get('nterm_protein_mods_spec', '') + if n_ace: + cfg['index']['nterm_protein_mods_spec'] = apply_mod(spec, mod=NACE_MOD) + else: + cfg['index']['nterm_protein_mods_spec'] = apply_mod(spec, mod='', exclusive_pattern=r'^\d*X\+42\.011') + logMsg.debug(f'{"--n-ace" if n_ace else "--no-n-ace"} applied: nterm_protein_mods_spec updated to {cfg["index"]["nterm_protein_mods_spec"]}') + if low_res is not None: + cfg['search']['mz_bin_width'] = MZ_BIN_WIDTH_LOW_RES if low_res else MZ_BIN_WIDTH_HIGH_RES + cfg['search']['score_function'] = SCORE_FUNC_LOW_RES if low_res else SCORE_FUNC_HIGH_RES + logMsg.debug(f'{"--low-res" if low_res else "--high-res"} applied: mz_bin_width: {cfg["search"]["mz_bin_width"]}, score_function: {cfg["search"]["score_function"]}') + if clip_met is not None: + cfg['index']['clip_n_met'] = bool(clip_met) + logMsg.debug(f'{"--clip-met" if clip_met else "--no-clip-met"} applied: {cfg["index"]["clip_n_met"]}') + if missed_cleavages is not None: + cfg['index']['missed_cleavages'] = missed_cleavages + logMsg.debug(f'--missed-cleavages applied: {missed_cleavages}') + return cfg \ No newline at end of file From ebc45367a96672d030a09e303cc4a0f449093a8e Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Thu, 9 Jul 2026 16:50:54 +0100 Subject: [PATCH 047/108] chore: update default config.toml --- src/comms/config.toml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/comms/config.toml b/src/comms/config.toml index bff566f..c6a3ad1 100644 --- a/src/comms/config.toml +++ b/src/comms/config.toml @@ -1,9 +1,5 @@ # comMS default configuration -[global] -verbose = false -debug = false - [organism] @@ -22,30 +18,24 @@ nterm_protein_mods_spec = "1X+42.011" custom_mods = "" [search] -score_function = "xcorr" # xcorr for high-res MS (default); combined-p-value for low-res (run: comms config set --low-res) -mz_bin_width = 0.02 # 0.02 for high-res MS (default); 1.0005079 for low-res (run: comms config set --low-res) +score_function = "xcorr" # xcorr for high-res MS (default); combined-p-value for low-res (run: comms config --low-res) +mz_bin_width = 0.02 # 0.02 for high-res MS (default); 1.0005079 for low-res (run: comms config --low-res) min_peaks = 10 precursor_tolerance_ppm = 10.0 threads = 2 [percolator] protein_enzyme = "trypsin" -picked_protein = true # use picked-protein FDR (c.f. Savitski et al. 2015) +picked_protein = true # use picked-protein FDR (c.f. Savitski et al. 2015, doi:10.1074/mcp.M114.046995) shared_psm = "drop" -[lfq] -match_between_runs = true - [quantify] measure = "dNSAF" qvalue_threshold = 0.01 unique_mapping = true [report] -top_n_proteins = 50 -fdr_threshold = 0.01 -colour_palette = "Set2" -include_organism_panel = true -include_pca = true -include_volcano = true -include_heatmap = true \ No newline at end of file +min_reps = 3 +lfc_threshold = 1.0 +fdr_threshold = 0.05 +top_n_proteins = 20 \ No newline at end of file From 787456d9669c46e9da8d4cd52596c25a1f899d17 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 10:16:26 +0100 Subject: [PATCH 048/108] feat(config): update config resolution functions - Modified commands/config.py to remove file resolution functions and to remove subcommands as new command signature will use single config command with various flags. --- src/comms/commands/config.py | 355 ++++++++++++++--------------------- 1 file changed, 139 insertions(+), 216 deletions(-) diff --git a/src/comms/commands/config.py b/src/comms/commands/config.py index 0a1ed12..3b12191 100644 --- a/src/comms/commands/config.py +++ b/src/comms/commands/config.py @@ -11,93 +11,139 @@ from rich import print from rich.console import Console from rich.table import Table -from typing import Annotated # -- Import internal functions +from comms.utils.context import _normalise_dirs from comms.utils.log import logMsg +from comms.utils.modspec import apply_custom_mod, apply_organism, apply_protocol_flags, parse_organism_arg from comms.utils.settings import loadDefaultConfig, globalConfigPath -# ========================= # -# DEFINE CONFIG SUBCOMMANDS # -# ========================= # -# -- config_init: creates a config file with default settings in the OS config directory -def config_init(config_path: Path | None = None): - logMsg('config') - config_path = config_path or globalConfigPath() - logMsg.debug(f'Checking config path: {config_path}') - if not _configCheck(config_path, exists=False): - raise SystemExit(1) - try: - logMsg.progress(f'Writing default config to {config_path}') - _writeConfigTo(loadDefaultConfig(), config_path) - logMsg.info(f'Config file written to {config_path}') - except Exception as e: - logMsg.error(f'Failed to write config: {e}') - raise SystemExit(1) +# -- _confirm: yes/no prompt via logMsg.input, returned as a bool +def _confirm(msg: str, default: bool) -> bool: + msg = f'{msg} [dim]({"Y/n" if default else "y/N"})[/dim]' + answer = logMsg.input(msg, choices=['y', 'n'], default='y' if default else 'n', case_sensitive=False, show_choices=False, show_default=False) + return str(answer).strip().lower() == 'y' -# -- config_exists: reports whether a config file exists and prints its path -def config_exists(config_path: Path | None = None): - logMsg('config') - config_path = config_path or globalConfigPath() - logMsg.debug(f'Checking for config at {config_path}') - if config_path.exists(): - logMsg.info(f'Config file found at {config_path}') +# -- _loadConfigFile: returns the config as a dict +def _loadConfigFile(config_path: Path) -> dict: + with config_path.open('rb') as f: + return tomllib.load(f) + +# -- _writeConfigTo: writes a config dict to a given path +def _writeConfigTo(config: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('wb') as f: + tomli_w.dump(config, f) + +# -- _flatten: returns a flat dict from a nested dict, with dot-separated keys +def _flatten(d: dict, prefix: str = '') -> dict: + out = {} + for k, v in d.items(): + key = f'{prefix}.{k}' if prefix else k + if isinstance(v, dict): + out.update(_flatten(v, key)) + else: + out[key] = v + return out + +# -- _printTable: prints a Rich table comparing current and default config values +def _printTable(user_config: dict, default_config: dict) -> None: + console = Console() + table = Table(title='comMS configuration', show_header=True, header_style='bold', show_lines=False) + table.add_column('Key', style='cyan', no_wrap=True) + table.add_column('Current value', justify='right') + table.add_column('Default value', justify='right', style='dim') + table.add_column('', width=2) + for key in sorted(default_config.keys()): + default_val = default_config[key] + user_val = user_config.get(key, '[bold red]MISSING[/bold red]') + changed = str(user_val) != str(default_val) + status = '[yellow]≠[/yellow]' if changed else '[green]✓[/green]' + user_str = f'[yellow]{user_val}[/yellow]' if changed else str(user_val) + table.add_row(key, user_str, str(default_val), status) + console.print(table) + +# -- _print_diff_summary: prints only the keys that changed between two flattened config dicts +def _print_diff_summary(before: dict, after: dict) -> None: + changed = {k: (before.get(k), v) for k, v in after.items() if before.get(k) != v} + if not changed: + print('\n[dim]No changes made.[/dim]\n') + return + print() + for key, (old, new) in sorted(changed.items()): + print(f'[bold green]✓[/bold green] [dim]{key}[/dim]: [dim]{old}[/dim] → [cyan]{new}[/cyan]') + print() + +# -- resolve_or_create: returns the Path to edit, creating it from defaults first if needed +def resolve_or_create(path: Path | None, use_global: bool) -> Path: + ''' + Resolve the config.toml target and make sure it exists, creating it from bundled defaults if not + ''' + if use_global: + target = globalConfigPath() + elif path is not None: + _, comms_dir = _normalise_dirs(path) + target = comms_dir / 'config.toml' else: - logMsg.error(f'No config file at {config_path}') - raise SystemExit(1) + bare, nested = Path('config.toml'), Path('comms') / 'config.toml' + if bare.exists() and nested.exists(): + logMsg.error(f'Both {bare} and {nested} exist in the current directory. Remove one before running comms config here.') + raise SystemExit(1) + if bare.exists(): + target = bare + elif nested.exists(): + target = nested + else: + logMsg.warn(f'No local config found in the current directory. Did you mean to use [bold]--global[/bold]?') + create_answer = _confirm(msg=f'Create default config at {config}', default=True) + if not create_answer: + raise SystemExit(0) + target = nested + if not target.exists(): + logMsg.debug(f'Creating default config at {target}') + _writeConfigTo(loadDefaultConfig(), target) + return target # -- config_list: prints current config values, highlighting differences from bundled defaults -def config_list(config_path: Path | None = None): +def config_list(config_path: Path) -> None: logMsg('config') logMsg.debug(f'Listing config values') - config_path = config_path or globalConfigPath() default_config = _flatten(loadDefaultConfig()) - if _configCheck(config_path, exists=True): - print(f'[bold blue]Current config:[/bold blue] [cyan]{config_path}[/cyan]\n') - current_config = _flatten(_loadConfigFile(config_path)) - else: - print(f'[bold blue]Current config:[/bold blue] built-in defaults\n') - current_config = default_config + print(f'\n[bold blue]Current config:[/bold blue] [cyan]{config_path}[/cyan]\n') + current_config = _flatten(_loadConfigFile(config_path)) _printTable(current_config, default_config) print() # -- config_verify: checks that all expected keys are present in the config file -def config_verify(config_path: Path | None = None): +def config_verify(config_path: Path) -> None: logMsg('config') - config_path = config_path or globalConfigPath() logMsg.debug(f'Verifying config keys at {config_path}') - if not _configCheck(config_path, exists=True): - logMsg.error(f'No config to verify at {config_path}') - raise SystemExit(1) user_config = _flatten(_loadConfigFile(config_path)) default_config = _flatten(loadDefaultConfig()) missing = [k for k in default_config if k not in user_config] unexpected = [k for k in user_config if k not in default_config] if not missing and not unexpected: - logMsg.info(f'User config {config_path} is valid') + logMsg.info(f'Config {config_path} is valid') return - logMsg.error(f'User config invalid: {len(missing)} missing, {len(unexpected)} unexpected key(s)') + logMsg.error(f'Config invalid: {len(missing)} missing, {len(unexpected)} unexpected key(s)') if missing: - logMsg.warn(f'Missing keys in config: {missing}') print(f'[bold red]ERROR:[/bold red] {len(missing)} missing key(s):') for k in sorted(missing): print(f'\t[red]✗[/red] {k} [dim](expected: {default_config[k]})[/dim]') if unexpected: - logMsg.warn(f'Unexpected keys in config: {unexpected}') print(f'[bold red]ERROR:[/bold red] {len(unexpected)} unexpected key(s):') for k in sorted(unexpected): print(f'\t[red]?[/red] {k}: {user_config[k]}') - print(f'Run [bold]comms config reset[/bold] to restore defaults.\n') + print(f'Run [bold]comms config --reset[/bold] to restore defaults.\n') raise SystemExit(1) # -- config_reset: overwrites the config file with comMS built-in defaults -def config_reset(config_path: Path | None = None, force: bool = False): +def config_reset(config_path: Path, force: bool = False) -> None: logMsg('config') - config_path = config_path or globalConfigPath() if not force: logMsg.warn(f'This will overwrite {config_path} with comMS defaults.') - if not typer.confirm('All custom settings will be lost. Continue?'): - logMsg.debug(f'Reset cancelled') + if not _confirm('Continue with reset'): + logMsg.debug('Reset cancelled') raise SystemExit(0) try: _writeConfigTo(loadDefaultConfig(), config_path) @@ -106,184 +152,61 @@ def config_reset(config_path: Path | None = None, force: bool = False): logMsg.error(f'Failed to reset config: {e}') raise SystemExit(1) -# -- config_set: apply named flags to the config -def config_set( - config_path: Path | None = None, - iodo: bool | None = None, - low_res: bool | None = None, - organism: list[str] | None = None, - ox: bool | None = None, - phos: bool | None = None, - n_cyc: bool | None = None, - n_ace: bool | None = None, - custom: str | None = None, - clip_met: bool | None = None, -) -> None: - # Set up logger +# -- config_set: apply any given flags to the config file; returns True if anything changed +def config_set(config_path: Path, **flags) -> bool: logMsg('config') - logMsg.debug(f'Applying set flags: iodo={iodo}; ox={ox}; phos={phos}; n_cyc={n_cyc}; n_ace={n_ace}; low_res={low_res}; organism={organism}; custom={custom!r}; clip_met={clip_met}') - # Check at least one flag set - if all(v is None for v in (iodo, ox, phos, n_cyc, n_ace, low_res, organism, custom, clip_met)): - logMsg.error(f'No flags supplied to config set') - raise SystemExit(1) - # Check if config exists - config_path = config_path or globalConfigPath() - if not config_path.exists(): - logMsg.debug(f'No config found, creating from defaults at {config_path}') - _writeConfigTo(loadDefaultConfig(), path=config_path) - # Load config + logMsg.debug(f'Applying flags: {flags}') + if all(v is None for v in flags.values()): + return False try: cfg = _loadConfigFile(config_path) except Exception as e: logMsg.error(f'Failed to read config file: {e}') raise SystemExit(1) - # Apply any passed flags - cfg = _apply_protocol_flags( + before = _flatten(cfg).copy() + cfg = apply_protocol_flags( cfg, - iodo=iodo, - ox=ox, - phos=phos, - n_cyc=n_cyc, - n_ace=n_ace, - low_res=low_res, - clip_met=clip_met + iodo=flags.get('iodo'), + ox=flags.get('ox'), + phos=flags.get('phos'), + n_cyc=flags.get('n_cyc'), + n_ace=flags.get('n_ace'), + clip_met=flags.get('clip_met'), + low_res=flags.get('low_res'), + missed_cleavages=flags.get('missed_cleavages'), ) - if organism is not None: - cfg = _apply_organism(cfg, _parse_organism_arg(organism)) - if custom is not None: + if flags.get('organism') is not None: + cfg = apply_organism(cfg, parse_organism_arg(flags['organism'])) + if flags.get('custom') is not None: current = cfg.get('index', {}).get('custom_mods', '') - cfg.setdefault('index', {})['custom_mods'] = _apply_custom_mod(current, custom) - # Write updated config + cfg.setdefault('index', {})['custom_mods'] = apply_custom_mod(current, flags['custom']) + direct = { + ('convert', 'gzip'): flags.get('gzip'), + ('convert', 'format'): flags.get('format'), + ('convert', 'metadata'): flags.get('metadata'), + ('search', 'score_function'): flags.get('score_function'), + ('search', 'min_peaks'): flags.get('min_peaks'), + ('search', 'precursor_tolerance_ppm'): flags.get('precursor_tolerance_ppm'), + ('search', 'mz_bin_width'): flags.get('mz_bin_width'), + ('search', 'threads'): flags.get('threads'), + ('percolator', 'protein_enzyme'): flags.get('protein_enzyme'), + ('percolator', 'picked_protein'): flags.get('picked_protein'), + ('percolator', 'shared_psm'): flags.get('shared_psm'), + ('quantify', 'measure'): flags.get('measure'), + ('quantify', 'qvalue_threshold'): flags.get('qvalue_threshold'), + ('quantify', 'unique_mapping'): flags.get('unique_mapping'), + ('report', 'min_reps'): flags.get('min_reps'), + ('report', 'lfc_threshold'): flags.get('lfc_threshold'), + ('report', 'fdr_threshold'): flags.get('fdr_threshold'), + ('report', 'top_n_proteins'): flags.get('top_n'), + } + for (section, key), value in direct.items(): + if value is not None: + cfg.setdefault(section, {})[key] = value try: _writeConfigTo(cfg, config_path) except Exception as e: logMsg.error(f'Failed to write config file: {e}') raise SystemExit(1) - # Print summary - _printSetSummary(iodo=iodo, ox=ox, phos=phos, n_cyc=n_cyc, n_ace=n_ace, low_res=low_res, organism=organism, custom=custom, clip_met=clip_met) - print() - - -# ======================= # -# DEFINE INTERNAL HELPERS # -# ======================= # -# -- _resolveConfigTarget: returns the Path to edit (global user config or a local file) -def _resolveConfigTarget(target: str | None) -> Path: - if target is None or target.upper() == 'GLOBAL': - return globalConfigPath() - return Path(target) - -# -- _loadConfigFile: returns the config as a dict -def _loadConfigFile(config_path: Path | None = None) -> dict: - config_path = config_path or globalConfigPath() - if not config_path.exists(): - raise FileNotFoundError(f'No config found at {config_path}.') - with config_path.open('rb') as f: - return tomllib.load(f) - -# -- _writeConfigTo: writes a config dict to a given path -def _writeConfigTo(config: dict, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open('wb') as f: - tomli_w.dump(config, f) - -# -- _writeConfig: writes config dict to the global config path -def _writeConfig(config: dict): - _writeConfigTo(config, globalConfigPath()) - -# -- _flatten: returns a flat dict from a nested dict, with dot-separated keys -def _flatten(d: dict, prefix: str = '') -> dict: - out = {} - for k, v in d.items(): - key = f'{prefix}.{k}' if prefix else k - if isinstance(v, dict): - out.update(_flatten(v, key)) - else: - out[key] = v - return out - -# -- _configCheck: returns True if the config file existence matches the expected state -def _configCheck(config_path: Path, exists: bool) -> bool: - if exists: - if config_path.exists(): - return True - print(f'\n[bold yellow]WARNING:[/bold yellow] No config at [cyan]{config_path}[/cyan]\nRun [bold]comms config init[/bold] to create one.\n') - return False - else: - if config_path.exists(): - print(f'\n[bold yellow]WARNING:[/bold yellow] Config already exists at [cyan]{config_path}[/cyan]\nRun [bold]comms config reset[/bold] to reset to defaults.\n') - return False - return True - -# -- _printTable: prints a Rich table comparing current and default config values -def _printTable(user_config: dict, default_config: dict): - console = Console() - table = Table(title='comMS configuration', show_header=True, header_style='bold', show_lines=False) - table.add_column('Key', style='cyan', no_wrap=True) - table.add_column('Current value', justify='right') - table.add_column('Default value', justify='right', style='dim') - table.add_column('', width=2) - for key in sorted(default_config.keys()): - default_val = default_config[key] - user_val = user_config.get(key, '[bold red]MISSING[/bold red]') - changed = str(user_val) != str(default_val) - status = '[yellow]≠[/yellow]' if changed else '[green]✓[/green]' - user_str = f'[yellow]{user_val}[/yellow]' if changed else str(user_val) - table.add_row(key, user_str, str(default_val), status) - console.print(table) - - -# ========================= # -# DEFINE CONFIG SET HELPERS # -# ========================= # -# _mod_summary_line: prints a s -def _mod_summary_line(flag: bool | None, mod: str, key: str): - ''' - Print a single ✓ line for a boolean mod flag, or nothing if flag is None - ''' - if flag is None: - return - print(f'[bold green]✓[/bold green] [dim]{key}[/dim] → [cyan]{mod}[/cyan]') - -# _print_set_summary: prints a summary of changes made -def _printSetSummary( - *, - iodo: bool | None, - ox: bool | None, - phos: bool | None, - n_cyc: bool | None, - n_ace: bool | None, - low_res: bool | None, - organism: list[str] | None, - custom: str | None, - clip_met: bool | None, -) -> None: - ''' - Print a summary of what config_set changed - ''' - print() - _mod_summary_line(iodo, CARBAMIDOMETHYL_MOD, f'index.{'fixed_mods'}') - _mod_summary_line(ox, MET_OX_MOD, 'index.mods_spec') - _mod_summary_line(phos, PHOSPHO_MOD, 'index.mods_spec') - if custom is not None: - if custom == '': - print(f'[bold green]✓[/bold green] Custom mods cleared: [dim]index.custom_mods[/dim] → [cyan](empty)[/cyan]') - else: - print(f'[bold green]✓[/bold green] Custom mod added: [dim]index.custom_mods[/dim] → [cyan]{custom}[/cyan]') - _mod_summary_line(n_cyc, NCYC_MOD, f'index.{'nterm_peptide_mods_spec'}') - _mod_summary_line(n_ace, NACE_MOD, f'index.{'nterm_protein_mods_spec'}') - if clip_met is not None: - value = 'true' if clip_met else 'false' - print(f'[bold green]✓[/bold green] Clipped N-terminal methionine set: [dim]index.clip_n_met[/dim] → to [cyan]{value}[/cyan]') - if low_res is not None: - if low_res: - print(f'[bold green]✓[/bold green] Low-resolution mode set: [dim]search.mz_bin_width[/dim] → [cyan]{MZ_BIN_WIDTH_LOW_RES}[/cyan], [dim]search.score_function[/dim] → [cyan]{SCORE_FUNC_LOW_RES}[/cyan]') - else: - print(f'[bold green]✓[/bold green] High-resolution mode set: [dim]search.mz_bin_width[/dim] → [cyan]{MZ_BIN_WIDTH_HIGH_RES}[/cyan], [dim]search.score_function[/dim] → [cyan]{SCORE_FUNC_HIGH_RES}[/cyan]') - if organism is not None: - for item in organism: - key, _, pattern = item.partition('=') - key = ''.join(key.split()) - pattern = ''.join(pattern.split()) - print(f'[bold green]✓[/bold green] Organism pattern set: [dim]organism[/dim] → [cyan]{key}[/cyan]: [cyan]{pattern}[/cyan]') - print() \ No newline at end of file + _print_diff_summary(before, _flatten(cfg)) + return True \ No newline at end of file From 53309cbea6340f32a876d7b8b15774f0cf55b074 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 10:59:30 +0100 Subject: [PATCH 049/108] fix(config): resolve config file path per function - Modified commands/config.py to resolve the path to a config file as part of each function (instead of running independently before) to allow resolution function to use logMsg. --- src/comms/commands/config.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/comms/commands/config.py b/src/comms/commands/config.py index 3b12191..bf81dbc 100644 --- a/src/comms/commands/config.py +++ b/src/comms/commands/config.py @@ -74,8 +74,8 @@ def _print_diff_summary(before: dict, after: dict) -> None: print(f'[bold green]✓[/bold green] [dim]{key}[/dim]: [dim]{old}[/dim] → [cyan]{new}[/cyan]') print() -# -- resolve_or_create: returns the Path to edit, creating it from defaults first if needed -def resolve_or_create(path: Path | None, use_global: bool) -> Path: +# -- _resolve_or_create: returns the Path to edit, creating it from defaults first if needed +def _resolve_or_create(path: Path | None, use_global: bool) -> Path: ''' Resolve the config.toml target and make sure it exists, creating it from bundled defaults if not ''' @@ -95,7 +95,7 @@ def resolve_or_create(path: Path | None, use_global: bool) -> Path: target = nested else: logMsg.warn(f'No local config found in the current directory. Did you mean to use [bold]--global[/bold]?') - create_answer = _confirm(msg=f'Create default config at {config}', default=True) + create_answer = _confirm(msg=f'Create default config at {nested}', default=True) if not create_answer: raise SystemExit(0) target = nested @@ -105,8 +105,10 @@ def resolve_or_create(path: Path | None, use_global: bool) -> Path: return target # -- config_list: prints current config values, highlighting differences from bundled defaults -def config_list(config_path: Path) -> None: +def config_list(path, global_) -> None: logMsg('config') + logMsg.debug(f'Resolving configuration file') + config_path = _resolve_or_create(path, global_) logMsg.debug(f'Listing config values') default_config = _flatten(loadDefaultConfig()) print(f'\n[bold blue]Current config:[/bold blue] [cyan]{config_path}[/cyan]\n') @@ -115,8 +117,10 @@ def config_list(config_path: Path) -> None: print() # -- config_verify: checks that all expected keys are present in the config file -def config_verify(config_path: Path) -> None: +def config_verify(path, global_) -> None: logMsg('config') + logMsg.debug(f'Resolving configuration file') + config_path = _resolve_or_create(path, global_) logMsg.debug(f'Verifying config keys at {config_path}') user_config = _flatten(_loadConfigFile(config_path)) default_config = _flatten(loadDefaultConfig()) @@ -138,8 +142,10 @@ def config_verify(config_path: Path) -> None: raise SystemExit(1) # -- config_reset: overwrites the config file with comMS built-in defaults -def config_reset(config_path: Path, force: bool = False) -> None: +def config_reset(path, global_, force: bool = False) -> None: logMsg('config') + logMsg.debug(f'Resolving configuration file') + config_path = _resolve_or_create(path, global_) if not force: logMsg.warn(f'This will overwrite {config_path} with comMS defaults.') if not _confirm('Continue with reset'): @@ -153,8 +159,10 @@ def config_reset(config_path: Path, force: bool = False) -> None: raise SystemExit(1) # -- config_set: apply any given flags to the config file; returns True if anything changed -def config_set(config_path: Path, **flags) -> bool: +def config_set(path, global_, **flags) -> bool: logMsg('config') + logMsg.debug(f'Resolving configuration file') + config_path = _resolve_or_create(path, global_) logMsg.debug(f'Applying flags: {flags}') if all(v is None for v in flags.values()): return False From 8831fd601e24246d1ca6e629f7fe8b1873011139 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 11:01:23 +0100 Subject: [PATCH 050/108] feat(config): update config cli signature - Modified cli/cli.py and cli/config.py to update config command to use single command with flags for verification and resetting. Also updated command to use flags for all configuration options. --- src/comms/cli/cli.py | 2 +- src/comms/cli/config.py | 176 +++++++++++++++++++--------------------- 2 files changed, 83 insertions(+), 95 deletions(-) diff --git a/src/comms/cli/cli.py b/src/comms/cli/cli.py index 4075d27..bcb6dbe 100644 --- a/src/comms/cli/cli.py +++ b/src/comms/cli/cli.py @@ -51,7 +51,7 @@ comms.add_typer(commsLfq) comms.add_typer(commsQuantify) comms.add_typer(commsReport) -comms.add_typer(commsConfig, name='config', help='Manage comMS configuration', rich_help_panel='comMS Configuration') +comms.add_typer(commsConfig) comms.add_typer(commsLicense) comms.add_typer(commsRUtils, name='r-utils', help='Check or install required R dependencies', rich_help_panel='Utilities') comms.add_typer(commsUninstall) diff --git a/src/comms/cli/config.py b/src/comms/cli/config.py index 9aa941b..2f1630e 100644 --- a/src/comms/cli/config.py +++ b/src/comms/cli/config.py @@ -4,7 +4,8 @@ # -- Import external dependencies import typer -from typing import Annotated, List, Optional +from pathlib import Path +from typing import Annotated, List, Literal, Optional # -- Import internal functions from comms.commands import config as configFuncs @@ -12,100 +13,69 @@ # -- Initialise config Typer class commsConfig = typer.Typer(add_completion=False, invoke_without_command=True) -# Define config file option used in all commands -_CONFIG_OPT = typer.Option( - '-c', '--config', - help="Config file to edit; a path, or 'global' for the user config [default: global]", -) +# -- Define shared path/global parameters +_PATH_ARG = typer.Argument(help='Path to experiment directory [dim](default: ".")[/dim]') +_GLOBAL_OPT = typer.Option('--global', help='Use the global user config instead of a local config.toml') -# -- Define config callback -@commsConfig.callback(invoke_without_command=True) -def config_callback(ctx: typer.Context) -> None: - if ctx.invoked_subcommand is None: - configFuncs.config_exists() - -# -- Define config command: init -@commsConfig.command(rich_help_panel='Config Commands') -def init(config: Annotated[Optional[str], _CONFIG_OPT] = None): - '''Create a user config file with default settings in the OS config directory''' - configFuncs.config_init(config_path=configFuncs._resolveConfigTarget(config)) - -# -- Define config command: exists -@commsConfig.command(rich_help_panel='Config Commands') -def exists(config: Annotated[Optional[str], _CONFIG_OPT] = None): - '''Report whether a user config file exists and print its path''' - configFuncs.config_exists(config_path=configFuncs._resolveConfigTarget(config)) - -# -- Define config command: list -@commsConfig.command(rich_help_panel='Config Commands') -def list(config: Annotated[Optional[str], _CONFIG_OPT] = None): - '''Print current config values, highlighting differences from bundled defaults''' - configFuncs.config_list(config_path=configFuncs._resolveConfigTarget(config)) - -# -- Define config command: verify -@commsConfig.command(rich_help_panel='Config Commands') -def verify(config: Annotated[Optional[str], _CONFIG_OPT] = None): - '''Check that all expected keys are present in the user config file''' - configFuncs.config_verify(config_path=configFuncs._resolveConfigTarget(config)) - -# -- Define config command: reset -@commsConfig.command(rich_help_panel='Config Commands') -def reset( - config: Annotated[Optional[str], _CONFIG_OPT] = None, - force: Annotated[ - bool, - typer.Option('--force', help='Skip confirmation and immediately overwrite config.toml') - ] = False -): - '''Overwrite the user config file with comMS built-in defaults''' - configFuncs.config_reset(config_path=configFuncs._resolveConfigTarget(config), force=force) - -# -- Define config command: set -@commsConfig.command(rich_help_panel='Config Commands') -def set( - config: Annotated[Optional[str], _CONFIG_OPT] = None, - iodo: Annotated[ - Optional[bool], - typer.Option('--iodo/--no-iodo', help='Add (--iodo) or remove (--no-iodo) carbamidomethylation of cysteine as a static modification'), - ] = None, - ox: Annotated[ - Optional[bool], - typer.Option('--ox/--no-ox', help='Add (--ox) or remove (--no-ox) oxidation of methionine as a variable modification'), - ] = None, - phos: Annotated[ - Optional[bool], - typer.Option('--phos/--no-phos', help='Add (--phos) or remove (--no-phos) phosphorylation of serine/threonine/tyrosine as a variable modification'), - ] = None, - n_cyc: Annotated[ - Optional[bool], - typer.Option('--n-cyc/--no-n-cyc', help='Add (--n-cyc) or remove (--no-n-cyc) cyclisation of peptide N-terminal glutamine to pyro-glutamic acid as a variable modification'), - ] = None, - n_ace: Annotated[ - Optional[bool], - typer.Option('--n-ace/--no-n-ace', help='Add (--n-ace) or remove (--no-n-nace) acetylation of protein N-terminal residue as a variable modification'), - ] = None, - custom: Annotated[ - Optional[str], - typer.Option('--custom', help='Add a custom variable modification following Tide mods_spec format; can be passed multiple times; pass empty string "" to remove all custom modifications') - ] = None, - clip_met: Annotated[ - Optional[bool], - typer.Option('--clip-met/--no-clip-met', help="Include (--clip-met) or don't include (--no-clip-met) duplicate N-terminal peptides with clipped N-terminal methionine") - ] = None, - low_res: Annotated[ - Optional[bool], - typer.Option('--low-res/--high-res', help='Set search parameters for low-resolution (--low-res) or high-resolution (--high-res) instruments'), - ] = None, - organism: Annotated[ - Optional[List[str]], - typer.Option('--organism', help='Set organism header patterns for per-organism picked protein FDR [dim](format: OrganismLabel=Pattern)[/dim]'), - ] = None, -): +# -- Define config command (single command, no subcommands) +@commsConfig.command(rich_help_panel='comMS Configuration') +def config( + # -- Global options -- + path: Annotated[Optional[Path], _PATH_ARG] = None, + global_: Annotated[bool, _GLOBAL_OPT] = False, + verify: Annotated[bool, typer.Option('--verify', help='Check that all expected keys are present')] = False, + reset: Annotated[bool, typer.Option('--reset', help='Overwrite with comMS built-in defaults')] = False, + force: Annotated[bool, typer.Option('--force', help='Skip confirmation when using --reset')] = False, + # -- convert command options -- + gzip: Annotated[Optional[bool], typer.Option('--gzip/--no-gzip', help='Compress mzML output using gzip')] = None, + format: Annotated[Optional[int], typer.Option('--format', help='ThermoRawFileParser output format code', min=0, max=4)] = None, + metadata: Annotated[Optional[int], typer.Option('--metadata', help='ThermoRawFileParser metadata capture code', min=0, max=2)] = None, + # -- index command options -- + iodo: Annotated[Optional[bool], typer.Option('--iodo/--no-iodo', help='Add or remove carbamidomethylation of cysteine as a static modification')] = None, + ox: Annotated[Optional[bool], typer.Option('--ox/--no-ox', help='Add or remove oxidation of methionine as a variable modification')] = None, + phos: Annotated[Optional[bool], typer.Option('--phos/--no-phos', help='Add or remove phosphorylation of serine/threonine/tyrosine as a variable modification')] = None, + n_cyc: Annotated[Optional[bool], typer.Option('--n-cyc/--no-n-cyc', help='Add or remove cyclisation of peptide N-terminal glutamine to pyro-glutamic acid')] = None, + n_ace: Annotated[Optional[bool], typer.Option('--n-ace/--no-n-ace', help='Add or remove acetylation of protein N-terminal residue')] = None, + custom: Annotated[Optional[str], typer.Option('--custom', help='Add a custom variable modification (Tide mods_spec format); use "" to clear all custom mods')] = None, + clip_met: Annotated[Optional[bool], typer.Option('--clip-met/--no-clip-met', help='Include or exclude duplicate N-terminal peptides with clipped N-terminal methionine')] = None, + missed_cleavages: Annotated[Optional[int], typer.Option('--missed-cleavages', help='Number of missed enzymatic cleavages allowed', min=0)] = None, + organism: Annotated[Optional[List[str]], typer.Option('--organism', help='Organism header pattern for per-organism picked protein FDR [dim](format: Label=Pattern)[/dim]')] = None, + low_res: Annotated[Optional[bool], typer.Option('--low-res/--high-res', help='Set score_function/mz_bin_width for low- or high-resolution instruments')] = None, + # -- search command options -- + score_function: Annotated[Optional[str], typer.Option('--score-function', help='Tide-search score function')] = None, + min_peaks: Annotated[Optional[int], typer.Option('--min-peaks', help='Minimum peaks required per spectrum', min=1)] = None, + precursor_tolerance_ppm: Annotated[Optional[float], typer.Option('--precursor-tolerance-ppm', help='Precursor mass tolerance in ppm')] = None, + mz_bin_width: Annotated[Optional[float], typer.Option('--mz-bin-width', help='Fragment m/z bin width in Da')] = None, + threads: Annotated[Optional[int], typer.Option('--threads', help='Default number of threads', min=1)] = None, + # -- percolator command options -- + protein_enzyme: Annotated[Optional[str], typer.Option('--protein-enzyme', help='Enzyme used for protein-level picked-FDR grouping')] = None, + picked_protein: Annotated[Optional[bool], typer.Option('--picked-protein/--no-picked-protein', help='Use picked-protein FDR')] = None, + shared_psm: Annotated[Optional[Literal['drop', 'include']], typer.Option('--shared-psm', help='Policy for PSMs shared between organisms')] = None, + # -- quantify command options -- + measure: Annotated[Optional[Literal['NSAF', 'dNSAF', 'SIN', 'EMPAI']], typer.Option('--measure', help='Spectral-counting measure')] = None, + qvalue_threshold: Annotated[Optional[float], typer.Option('--qvalue-threshold', help='PSM q-value threshold for quantification', min=0.0, max=1.0)] = None, + unique_mapping: Annotated[Optional[bool], typer.Option('--unique-mapping/--no-unique-mapping', help='Require unique peptide-to-protein mapping')] = None, + # -- report command options -- + min_reps: Annotated[Optional[int], typer.Option('--min-reps', help='Minimum replicates per fraction-treatment group', min=1)] = None, + lfc_threshold: Annotated[Optional[float], typer.Option('--lfc-threshold', help='|log2FC| threshold for DA', min=0.0)] = None, + fdr_threshold: Annotated[Optional[float], typer.Option('--fdr-threshold', help='BH-FDR threshold for DA', min=0.0, max=1.0)] = None, + top_n: Annotated[Optional[int], typer.Option('--top-n', help='Number of top DA proteins labelled per volcano plot', min=1)] = None, +) -> None: ''' - Set values in user configuration file + View or edit comMS configurations ''' - configFuncs.config_set( - config_path=configFuncs._resolveConfigTarget(config), + if reset: + configFuncs.config_reset(path, global_, force=force) + return + if verify: + configFuncs.config_verify(path, global_) + return + changed = configFuncs.config_set( + path, + global_, + gzip=gzip, + format=format, + metadata=metadata, iodo=iodo, ox=ox, phos=phos, @@ -113,6 +83,24 @@ def set( n_ace=n_ace, custom=custom, clip_met=clip_met, - low_res=low_res, + missed_cleavages=missed_cleavages, organism=organism, - ) \ No newline at end of file + low_res=low_res, + score_function=score_function, + min_peaks=min_peaks, + precursor_tolerance_ppm=precursor_tolerance_ppm, + mz_bin_width=mz_bin_width, + threads=threads, + protein_enzyme=protein_enzyme, + picked_protein=picked_protein, + shared_psm=shared_psm, + measure=measure, + qvalue_threshold=qvalue_threshold, + unique_mapping=unique_mapping, + min_reps=min_reps, + lfc_threshold=lfc_threshold, + fdr_threshold=fdr_threshold, + top_n=top_n, + ) + if not changed: + configFuncs.config_list(path, global_) \ No newline at end of file From f80ab9ac730aac4cbdb3873bc21c1e5409b87e2e Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 11:20:53 +0100 Subject: [PATCH 051/108] refactor(config): move config writer to utilities - Modified commands/config.py and utils/settings.py to move config TOML writer function to utilities so run-specific parameters can be saved to a file. --- src/comms/commands/config.py | 8 +------- src/comms/utils/settings.py | 6 ++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/comms/commands/config.py b/src/comms/commands/config.py index bf81dbc..d82c446 100644 --- a/src/comms/commands/config.py +++ b/src/comms/commands/config.py @@ -16,7 +16,7 @@ from comms.utils.context import _normalise_dirs from comms.utils.log import logMsg from comms.utils.modspec import apply_custom_mod, apply_organism, apply_protocol_flags, parse_organism_arg -from comms.utils.settings import loadDefaultConfig, globalConfigPath +from comms.utils.settings import loadDefaultConfig, globalConfigPath, _writeConfigTo # -- _confirm: yes/no prompt via logMsg.input, returned as a bool def _confirm(msg: str, default: bool) -> bool: @@ -29,12 +29,6 @@ def _loadConfigFile(config_path: Path) -> dict: with config_path.open('rb') as f: return tomllib.load(f) -# -- _writeConfigTo: writes a config dict to a given path -def _writeConfigTo(config: dict, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open('wb') as f: - tomli_w.dump(config, f) - # -- _flatten: returns a flat dict from a nested dict, with dot-separated keys def _flatten(d: dict, prefix: str = '') -> dict: out = {} diff --git a/src/comms/utils/settings.py b/src/comms/utils/settings.py index 82d8ca3..7ee5e51 100644 --- a/src/comms/utils/settings.py +++ b/src/comms/utils/settings.py @@ -54,6 +54,12 @@ def _loadTomlFile(path: Path) -> dict: with path.open('rb') as f: return tomllib.load(f) +# -- _writeConfigTo: writes a config dict to a given path +def _writeConfigTo(config: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('wb') as f: + tomli_w.dump(config, f) + # -- resolveConfig: returns (config, source) def resolveConfig(comms_dir: Optional[Path] = None) -> tuple[dict, str]: ''' From 1c4a93fd806e897d118614f50a5a420d77155feb Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 11:41:09 +0100 Subject: [PATCH 052/108] feat(convert): align command cli with config - Modified cli/convert.py and commands/convert.py to align command logic and orchestration with configuration options and CLI overrides. --- src/comms/cli/convert.py | 12 ++++++++++-- src/comms/commands/convert.py | 30 +++++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/comms/cli/convert.py b/src/comms/cli/convert.py index 1f2df9f..0ccff82 100644 --- a/src/comms/cli/convert.py +++ b/src/comms/cli/convert.py @@ -27,8 +27,16 @@ def convert( ] = Path('.'), gzip: Annotated[ Optional[bool], - typer.Option('--gzip/--no-gzip', help='Gzip-compress mzML output file(s)') + typer.Option('--gzip/--no-gzip', help='Gzip-compress mzML output file(s) [dim][default: config convert.gzip][/dim]') + ] = None, + format: Annotated[ + Optional[int], + typer.Option('--format', help='ThermoRawFileParser output format code [dim][default: config convert.format][/dim]', min=0, max=4) + ] = None, + metadata: Annotated[ + Optional[int], + typer.Option('--metadata', help='ThermoRawFileParser metadata capture code [dim][default: config convert.metadata][/dim]', min=0, max=2) ] = None, ): ctx = ExperimentContext.resolve(experiment_dir) - convertFuncs.run_convert(data, ctx, gzip) \ No newline at end of file + convertFuncs.run_convert(data, ctx, gzip, format, metadata) \ No newline at end of file diff --git a/src/comms/commands/convert.py b/src/comms/commands/convert.py index fd32adf..bb800a1 100644 --- a/src/comms/commands/convert.py +++ b/src/comms/commands/convert.py @@ -9,16 +9,23 @@ # -- Import internal functions from comms.utils.log import configureFileLogging, logMsg from comms.utils.context import ExperimentContext, resolve_data_files +from comms.utils.settings import resolve_config_value, _writeConfigTo from comms.utils.validate import validate from comms.utils import trfp as trfputil from comms.utils import paths as pathutil # -- run_convert: converts all .RAW files in input_dir to indexed mzML and writes them to output -def run_convert(data_files, ctx: ExperimentContext, gzip: bool | None = None, in_pipeline: bool = False): +def run_convert( + data_files, + ctx: ExperimentContext, + gzip: bool | None = None, + format: int | None = None, + metadata: int | None = None, + in_pipeline: bool = False, +): if not in_pipeline: logMsg('convert') logMsg.debug('Started command: convert') - gzip = ctx.config['convert']['gzip'] if gzip is None else gzip _, trfp_path = validate(check_trfp=True, bin_dir=ctx.bin_dir) data_files = resolve_data_files(ctx, data_files) raw_files = [f for f in data_files if f.suffix.lower() == '.raw'] @@ -38,6 +45,19 @@ def run_convert(data_files, ctx: ExperimentContext, gzip: bool | None = None, in log_path = out_dir / 'convert.log' configureFileLogging(log_path) logMsg.debug(f'Output log file: {log_path}') + # Build config for this run only if any override was given + overrides_given = any(v is not None for v in (gzip, format, metadata)) + if overrides_given: + logMsg.debug('Using run-specific configuration parameters') + run_config = {**ctx.config, 'convert': dict(ctx.config.get('convert', {}))} + run_config['convert']['gzip'] = resolve_config_value(ctx.config, 'convert', 'gzip', gzip) + run_config['convert']['format'] = resolve_config_value(ctx.config, 'convert', 'format', format) + run_config['convert']['metadata'] = resolve_config_value(ctx.config, 'convert', 'metadata', metadata) + logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "convert.config.toml"') + _writeConfigTo(run_config, path=Path(out_dir, 'convert.config.toml')) + else: + logMsg.debug('Using contextual configuration parameters') + run_config = ctx.config n_ok, n_fail = 0, 0 for raw_file in raw_files: logMsg.progress(f'Converting {raw_file.name}') @@ -45,8 +65,8 @@ def run_convert(data_files, ctx: ExperimentContext, gzip: bool | None = None, in trfp_path=trfp_path, raw_file=raw_file, out_dir=out_dir, - output_format=ctx.config['convert']['format'], - metadata=ctx.config['convert']['metadata'], + output_format=run_config['convert']['format'], + metadata=run_config['convert']['metadata'], ) if ok: n_ok += 1 @@ -71,7 +91,7 @@ def run_convert(data_files, ctx: ExperimentContext, gzip: bool | None = None, in except: continue # If --gzip was provided, gzip TRFP output - if gzip: + if run_config['convert']['gzip']: import gzip, os, shutil # Loop through each mzML file in directory for file in out_dir.glob('[!.]*.mzML'): From 9ce52284e4834c95a76cc85cf0115ace53635d80 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 11:55:52 +0100 Subject: [PATCH 053/108] feat(index): align command cli with config - Modified cli/index.py and commands/index.py to align command logic and orchestration with configuration options and CLI overrides. --- src/comms/cli/index.py | 21 ++++++++++++++++++- src/comms/commands/index.py | 42 ++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/comms/cli/index.py b/src/comms/cli/index.py index 97c2700..c568a40 100644 --- a/src/comms/cli/index.py +++ b/src/comms/cli/index.py @@ -25,6 +25,25 @@ def index( Optional[Path], typer.Option('-e', '--experiment-dir', help='Experiment directory', exists=True, file_okay=False, dir_okay=True, writable=True) ] = Path('.'), + iodo: Annotated[Optional[bool], typer.Option('--iodo/--no-iodo', help='Override carbamidomethylation of cysteine for this run only [dim][default: config][/dim]')] = None, + ox: Annotated[Optional[bool], typer.Option('--ox/--no-ox', help='Override oxidation of methionine for this run only [dim][default: config][/dim]')] = None, + phos: Annotated[Optional[bool], typer.Option('--phos/--no-phos', help='Override phosphorylation of S/T/Y for this run only [dim][default: config][/dim]')] = None, + n_cyc: Annotated[Optional[bool], typer.Option('--n-cyc/--no-n-cyc', help='Override N-terminal Gln cyclisation for this run only [dim][default: config][/dim]')] = None, + n_ace: Annotated[Optional[bool], typer.Option('--n-ace/--no-n-ace', help='Override N-terminal protein acetylation for this run only [dim][default: config][/dim]')] = None, + custom: Annotated[Optional[str], typer.Option('--custom', help='Add a custom variable modification for this run only [dim][default: config][/dim]')] = None, + clip_met: Annotated[Optional[bool], typer.Option('--clip-met/--no-clip-met', help='Override clipped N-terminal methionine handling for this run only [dim][default: config][/dim]')] = None, + missed_cleavages: Annotated[Optional[int], typer.Option('--missed-cleavages', help='Missed cleavages for this run only [dim][default: config index.missed_cleavages][/dim]', min=0)] = None, ): ctx = ExperimentContext.resolve(experiment_dir) - indexFuncs.run_index(database, ctx) \ No newline at end of file + indexFuncs.run_index( + database, + ctx, + iodo=iodo, + ox=ox, + phos=phos, + n_cyc=n_cyc, + n_ace=n_ace, + custom=custom, + clip_met=clip_met, + missed_cleavages=missed_cleavages, + ) \ No newline at end of file diff --git a/src/comms/commands/index.py b/src/comms/commands/index.py index 6ce5446..2ea200e 100644 --- a/src/comms/commands/index.py +++ b/src/comms/commands/index.py @@ -9,12 +9,25 @@ # -- Import internal functions from comms.utils.log import configureFileLogging, logMsg from comms.utils.context import ExperimentContext, resolve_database +from comms.utils.modspec import apply_protocol_flags, apply_custom_mod +from comms.utils.settings import _writeConfigTo from comms.utils.validate import validate from comms.utils import crux as cruxutil from comms.utils import paths as pathutil -# -- run_index: builds a Tide peptide index from database and writes it to output -def run_index(database, ctx: ExperimentContext, in_pipeline: bool = False): +def run_index( + database, + ctx: ExperimentContext, + in_pipeline: bool = False, + iodo=None, + ox=None, + phos=None, + n_cyc=None, + n_ace=None, + custom=None, + clip_met=None, + missed_cleavages=None, +): if not in_pipeline: logMsg('index') logMsg.debug('Started command: index') @@ -26,12 +39,35 @@ def run_index(database, ctx: ExperimentContext, in_pipeline: bool = False): log_path = out_dir / 'index.log' configureFileLogging(log_path) logMsg.debug(f'Output log file: {log_path}') + # Build a config for this run only if any override was given + overrides_given = any(v is not None for v in (iodo, ox, phos, n_cyc, n_ace, custom, clip_met, missed_cleavages)) + if overrides_given: + logMsg.debug('Using run-specific configuration parameters') + run_config = {**ctx.config, 'index': dict(ctx.config.get('index', {}))} + run_config = apply_protocol_flags( + run_config, + iodo=iodo, + ox=ox, + phos=phos, + n_cyc=n_cyc, + n_ace=n_ace, + clip_met=clip_met, + missed_cleavages=missed_cleavages, + ) + if custom is not None: + current = run_config['index'].get('custom_mods', '') + run_config['index']['custom_mods'] = apply_custom_mod(current, custom) + logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "index.config.toml"') + _writeConfigTo(run_config, path=Path(out_dir, 'index.config.toml')) + else: + logMsg.debug('Using contextual configuration parameters') + run_config = ctx.config logMsg.progress(f'Building Tide peptide index') ok = cruxutil.tideIndex( crux_bin=crux_bin, database=database, index_dir=out_dir, - config=ctx.config, + config=run_config, ) if not ok: logMsg.error(f'tide-index failed, see {log_path}') From 1c066d628afdf78916c86e6a44694eb9ab99e86c Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 12:11:12 +0100 Subject: [PATCH 054/108] feat(search): align command cli with config - Modified cli/search.py and commands/search.py to align command logic and orchestration with configuration options and CLI overrides. - Modified utils/crux.py to update tideSearch function arguments to accept overrides. --- src/comms/cli/search.py | 28 ++++++++++++++++++- src/comms/commands/search.py | 54 ++++++++++++++++++++++++++++-------- src/comms/utils/crux.py | 12 ++++---- 3 files changed, 76 insertions(+), 18 deletions(-) diff --git a/src/comms/cli/search.py b/src/comms/cli/search.py index d30a5c7..585e67f 100644 --- a/src/comms/cli/search.py +++ b/src/comms/cli/search.py @@ -37,6 +37,32 @@ def search( int, typer.Option('--threads', help='Number of threads', min=1) ] = None, + score_function: Annotated[ + Optional[str], + typer.Option('--score-function', help='Tide-search score function [dim][default: config search.score_function][/dim]') + ] = None, + min_peaks: Annotated[ + Optional[int], + typer.Option('--min-peaks', help='Minimum peaks required per spectrum [dim][default: config search.min_peaks][/dim]', min=1) + ] = None, + precursor_tolerance_ppm: Annotated[ + Optional[float], + typer.Option('--precursor-tolerance-ppm', help='Precursor mass tolerance in ppm; takes priority over --param-medic if both given [dim][default: config search.precursor_tolerance_ppm][/dim]') + ] = None, + mz_bin_width: Annotated[ + Optional[float], + typer.Option('--mz-bin-width', help='Fragment m/z bin width in Da; takes priority over --param-medic if both given [dim][default: config search.mz_bin_width][/dim]') + ] = None, ): ctx = ExperimentContext.resolve(experiment_dir) - searchFuncs.run_search(data, index, ctx, param_medic, threads) \ No newline at end of file + searchFuncs.run_search( + data, + index, + ctx, + param_medic, + threads, + score_function=score_function, + min_peaks=min_peaks, + precursor_tolerance_ppm=precursor_tolerance_ppm, + mz_bin_width=mz_bin_width, + ) \ No newline at end of file diff --git a/src/comms/commands/search.py b/src/comms/commands/search.py index 6cb634e..2ca9eb3 100644 --- a/src/comms/commands/search.py +++ b/src/comms/commands/search.py @@ -11,34 +11,62 @@ # -- Import internal functions from comms.utils.log import configureFileLogging, logMsg from comms.utils.context import ExperimentContext, resolve_results_input, resolve_mzml_files +from comms.utils.settings import resolve_config_value, _writeConfigTo from comms.utils.validate import validate from comms.utils import crux as cruxutil from comms.utils import paths as pathutil # -- run_search: runs tide-search on all mzML files in input_dir and writes results to output -def run_search(data_files, index_dir, ctx: ExperimentContext, param_medic: bool, threads: int, in_pipeline: bool = False): +def run_search( + data_files, + index_dir, + ctx: ExperimentContext, + param_medic: bool, + threads: int | None, + score_function: str | None = None, + min_peaks: int | None = None, + precursor_tolerance_ppm: float | None = None, + mz_bin_width: float | None = None, + in_pipeline: bool = False, +): if not in_pipeline: logMsg('search') logMsg.debug('Started command: search') crux_bin, _ = validate(check_crux=True, bin_dir=ctx.bin_dir) index_dir = resolve_results_input(ctx, 'index', index_dir) mzml_files = resolve_mzml_files(ctx, data_files) - threads = threads or ctx.config['search']['threads'] logMsg.info(f'Searching {len(mzml_files)} mzML file(s)') out_dir = pathutil.generateOutputFileStructure(ctx.root, 'search') logMsg.debug(f'Output directory: {out_dir}') log_path = out_dir / 'search.log' configureFileLogging(log_path) logMsg.debug(f'Output log file: {log_path}') + # -- Optional: param-medic tolerance estimation - precursor_tol = None - mz_bin_width = None + pm_precursor, pm_bin_width = None, None if param_medic: logMsg.progress(f'Estimating tolerances with param-medic') - precursor_tol, mz_bin_width = _runParamMedic(crux_bin=crux_bin, mzml_files=mzml_files, out_dir=out_dir) - prec_display = precursor_tol or ctx.config['search']['precursor_tolerance_ppm'] - bin_width_display = mz_bin_width or ctx.config['search']['mz_bin_width'] - logMsg.debug(f'Precursor tolerance {prec_display} ppm, m/z bin width {bin_width_display} Da') + pm_precursor, pm_bin_width = _runParamMedic(crux_bin=crux_bin, mzml_files=mzml_files, out_dir=out_dir) + + # Build config for this run only if any override was given + overrides_given = any(v is not None for v in (threads, score_function, min_peaks, precursor_tolerance_ppm, mz_bin_width)) + if overrides_given: + logMsg.debug('Using run-specific configuration parameters') + run_config = {**ctx.config, 'search': dict(ctx.config.get('search', {}))} + run_config['search']['threads'] = resolve_config_value(ctx.config, 'search', 'threads', threads) + run_config['search']['score_function'] = resolve_config_value(ctx.config, 'search', 'score_function', score_function) + run_config['search']['min_peaks'] = resolve_config_value(ctx.config, 'search', 'min_peaks', min_peaks) + run_config['search']['precursor_tolerance_ppm'] = resolve_config_value(ctx.config, 'search', 'precursor_tolerance_ppm', precursor_tolerance_ppm if precursor_tolerance_ppm is not None else pm_precursor) + run_config['search']['mz_bin_width'] = resolve_config_value(ctx.config, 'search', 'mz_bin_width', mz_bin_width if mz_bin_width is not None else pm_bin_width) + logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "search.config.toml"') + _writeConfigTo(run_config, path=Path(out_dir, 'search.config.toml')) + else: + logMsg.debug('Using contextual configuration parameters') + run_config = ctx.config + + logMsg.debug(f'Precursor tolerance {run_config["search"]["precursor_tolerance_ppm"]} ppm, m/z bin width {run_config["search"]["mz_bin_width"]} Da') + + # -- PSM search n_ok, n_fail = 0, 0 with logging_redirect_tqdm(): for mzml_file in tqdm(mzml_files, desc='Files searched'): @@ -50,10 +78,12 @@ def run_search(data_files, index_dir, ctx: ExperimentContext, param_medic: bool, index_dir=index_dir, out_dir=out_dir, fileroot=fileroot, - config=ctx.config, - threads=threads, - precursor_tol=prec_display, - mz_bin_width=bin_width_display, + config=run_config, + threads=run_config['search']['threads'], + score_function=run_config['search']['score_function'], + min_peaks=run_config['search']['min_peaks'], + precursor_tol=run_config['search']['precursor_tolerance_ppm'], + mz_bin_width=run_config['search']['mz_bin_width'], ) if ok: n_ok += 1 diff --git a/src/comms/utils/crux.py b/src/comms/utils/crux.py index 1d4b342..ecbc059 100644 --- a/src/comms/utils/crux.py +++ b/src/comms/utils/crux.py @@ -114,10 +114,12 @@ def paramMedic(crux_bin: Path, mzml_file: Path, out_dir: Path) -> bool: return runCrux(crux_bin, 'param-medic', args) # -- tideSearch: returns True if Tide-search completed successfully for the given mzML file, False on failure -def tideSearch(crux_bin: Path, mzml_file: Path, index_dir: Path, out_dir: Path, fileroot: str, config: dict, threads, precursor_tol=None, mz_bin_width=None) -> bool: +def tideSearch(crux_bin: Path, mzml_file: Path, index_dir: Path, out_dir: Path, fileroot: str, config: dict, threads, precursor_tol=None, mz_bin_width=None, score_function=None, min_peaks=None) -> bool: logMsg.debug(f'tide-search: {mzml_file.name}') - prec = precursor_tol or config['search']['precursor_tolerance_ppm'] - bin_width = mz_bin_width or config['search']['mz_bin_width'] + prec = precursor_tol if precursor_tol is not None else config['search']['precursor_tolerance_ppm'] + bin_width = mz_bin_width if mz_bin_width is not None else config['search']['mz_bin_width'] + score_fn = score_function or config['search']['score_function'] + peaks = min_peaks or config['search']['min_peaks'] logMsg.debug(f'Precursor tolerance {prec} ppm, m/z bin width {bin_width}') args = [ '--verbosity', '40', @@ -126,8 +128,8 @@ def tideSearch(crux_bin: Path, mzml_file: Path, index_dir: Path, out_dir: Path, '--precursor-window', str(prec), '--precursor-window-type', 'ppm', '--mz-bin-width', str(bin_width), - '--score-function', config['search']['score_function'], - '--min-peaks', str(config['search']['min_peaks']), + '--score-function', score_fn, + '--min-peaks', str(peaks), '--missed-cleavages', str(config['index']['missed_cleavages']), '--output-dir', str(out_dir), '--fileroot', fileroot, From 8ada764e515350fe7a7392afc380c0d4e96b3488 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 15:02:19 +0100 Subject: [PATCH 055/108] feat(rescore): align command cli with config - Modified cli/rescore.py and commands/rescore.py to align command logic and orchestration with configuration options and CLI overrides. --- src/comms/cli/rescore.py | 24 +++++++++++++++-- src/comms/commands/rescore.py | 51 ++++++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/src/comms/cli/rescore.py b/src/comms/cli/rescore.py index d4793ca..667df16 100644 --- a/src/comms/cli/rescore.py +++ b/src/comms/cli/rescore.py @@ -5,7 +5,7 @@ # -- Import external dependencies import typer from pathlib import Path -from typing import Annotated, Optional +from typing import Annotated, Literal, Optional # -- Import internal functions from comms.commands import rescore as rescoreFuncs @@ -33,6 +33,26 @@ def rescore( Optional[Path], typer.Option('-e', '--experiment-dir', help='Experiment directory', exists=True, file_okay=False, dir_okay=True, writable=True) ] = Path('.'), + protein_enzyme: Annotated[ + Optional[str], + typer.Option('--protein-enzyme', help='Enzyme used for protein-level picked-FDR grouping [dim][default: config percolator.protein_enzyme][/dim]') + ] = None, + picked_protein: Annotated[ + Optional[bool], + typer.Option('--picked-protein/--no-picked-protein', help='Use picked-protein FDR [dim][default: config percolator.picked_protein][/dim]') + ] = None, + shared_psm: Annotated[ + Optional[Literal['drop', 'include']], + typer.Option('--shared-psm', help='Policy for PSMs shared between organisms [dim][default: config percolator.shared_psm][/dim]') + ] = None, ): ctx = ExperimentContext.resolve(experiment_dir) - rescoreFuncs.run_rescore(psm_dir, database, ctx, organism_tags) \ No newline at end of file + rescoreFuncs.run_rescore( + psm_dir, + database, + ctx, + organism_tags, + protein_enzyme=protein_enzyme, + picked_protein=picked_protein, + shared_psm=shared_psm, + ) \ No newline at end of file diff --git a/src/comms/commands/rescore.py b/src/comms/commands/rescore.py index aa7fd54..ea6922d 100644 --- a/src/comms/commands/rescore.py +++ b/src/comms/commands/rescore.py @@ -13,6 +13,7 @@ from comms.utils.fasta import splitFastaByOrganism from comms.utils.log import configureFileLogging, logMsg from comms.utils.context import ExperimentContext, resolve_database, resolve_results_input +from comms.utils.settings import resolve_config_value, _writeConfigTo from comms.utils.validate import validate from comms.utils import crux as cruxutil from comms.utils import paths as pathutil @@ -25,6 +26,9 @@ def run_rescore( database, ctx: ExperimentContext, organism_tags: Optional[str] = None, + protein_enzyme: Optional[str] = None, + picked_protein: Optional[bool] = None, + shared_psm: Optional[str] = None, in_pipeline: bool = False, ): if not in_pipeline: @@ -51,20 +55,36 @@ def run_rescore( log_path = out_dir / 'rescore.log' configureFileLogging(log_path) logMsg.debug(f'Output log file: {log_path}') + + # Build config for this run only if any override was given + overrides_given = any(v is not None for v in (protein_enzyme, picked_protein, shared_psm)) + if overrides_given: + logMsg.debug('Using run-specific configuration parameters') + run_config = {**ctx.config, 'rescore': dict(ctx.config.get('rescore', {}))} + run_config['rescore']['protein_enzyme'] = resolve_config_value(ctx.config, 'rescore', 'protein_enzyme', protein_enzyme) + run_config['rescore']['picked_protein'] = resolve_config_value(ctx.config, 'rescore', 'picked_protein', picked_protein) + run_config['rescore']['shared_psm'] = resolve_config_value(ctx.config, 'rescore', 'shared_psm', shared_psm) + logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "rescore.config.toml"') + _writeConfigTo(run_config, path=Path(out_dir, 'rescore.config.toml')) + else: + logMsg.debug('Using contextual configuration parameters') + run_config = ctx.config + # Round 1: run Percolator on the full combined database, with one call per sample file - combined_target_files = _run_combined_percolator_round( - crux_bin, target_files, database, out_dir, ctx, + _run_combined_percolator_round( + crux_bin, + target_files, + database, + out_dir, + run_config, ) if not combined_target_files: logMsg.error('No combined Percolator output found, cannot continue') raise SystemExit(1) + # Round 2: run Percolator on each organism sub-FASTA if multispecies analysis if multispecies: - organism_tags = ( - _parseOrganismTags(organism_tags) - if organism_tags - else ctx.config.get('organism') - ) + organism_tags = (_parseOrganismTags(organism_tags) if organism_tags else ctx.config.get('organism')) if not organism_tags: logMsg.error('No organism tags supplied or configured for multi-species analysis') raise SystemExit(1) @@ -72,13 +92,18 @@ def run_rescore( sub_fastas = splitFastaByOrganism(database, out_dir, organism_tags) logMsg.debug(f'Built {len(sub_fastas)} per-organism sub-FASTA(s)') _run_per_organism_percolator_round( - crux_bin, target_files, sub_fastas, organism_tags, out_dir, ctx + crux_bin, + target_files, + sub_fastas, + organism_tags, + out_dir, + run_config, ) # Log command as complete logMsg.debug('Finished command: rescore') # -- _run_percolator_round: returns list of PSM files after runn Percolator (via Crux) on database, with one call per sample file -def _run_combined_percolator_round(crux_bin, target_files, database, out_dir, ctx) -> list: +def _run_combined_percolator_round(crux_bin, target_files, database, out_dir, run_config) -> list: logMsg.progress(f'Rescoring {len(target_files)} file(s) using combined database') n_ok, n_fail = 0, 0 with logging_redirect_tqdm(): @@ -91,7 +116,7 @@ def _run_combined_percolator_round(crux_bin, target_files, database, out_dir, ct database=database, out_dir=out_dir, fileroot=fileroot, - config=ctx.config, + config=run_config, ) if ok: n_ok += 1 @@ -103,10 +128,10 @@ def _run_combined_percolator_round(crux_bin, target_files, database, out_dir, ct return sorted(out_dir.glob('[!.]*.percolator.target.psms.txt')) # -- _run_per_organism_percolator_round: returns None but splits combined Tide search outputs by organism and runs Percolator (via Crux) -def _run_per_organism_percolator_round(crux_bin, combined_target_files, sub_fastas, organism_tags, out_dir, ctx): +def _run_per_organism_percolator_round(crux_bin, combined_target_files, sub_fastas, organism_tags, out_dir, run_config): logMsg.progress(f'Rescoring {len(combined_target_files)} file(s) using per-organism sub-FASTAs') n_ok, n_fail = 0, 0 - shared_policy = ctx.config['percolator']['shared_psm'] + shared_policy = run_config['percolator']['shared_psm'] with logging_redirect_tqdm(): for combined_file in tqdm(combined_target_files, desc='Files rescored'): logMsg.progress(f'Rescoring {combined_file.name}') @@ -140,7 +165,7 @@ def _run_per_organism_percolator_round(crux_bin, combined_target_files, sub_fast database=sub_fastas[label], out_dir=org_out_dir, fileroot=org_fileroot, - config=ctx.config, + config=run_config, ) if ok: n_ok += 1 From 78658cfd0a2d445c5bfed1a151b34fcf0084e18d Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 15:10:32 +0100 Subject: [PATCH 056/108] feat(quantify): align command cli with config - Modified cli/quantify.py and commands/quantify.py to align command logic and orchestration with configuration options and CLI overrides. --- src/comms/cli/quantify.py | 23 +++++++++++++++++++++-- src/comms/commands/quantify.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/comms/cli/quantify.py b/src/comms/cli/quantify.py index 343c1ee..1a8bf76 100644 --- a/src/comms/cli/quantify.py +++ b/src/comms/cli/quantify.py @@ -5,7 +5,7 @@ # -- Import external dependencies import typer from pathlib import Path -from typing import Annotated, Optional +from typing import Annotated, Literal, Optional # -- Import internal functions from comms.commands import quantify as quantifyFuncs @@ -29,6 +29,25 @@ def quantify( Optional[Path], typer.Option('-e', '--experiment-dir', help='Experiment directory', exists=True, file_okay=False, dir_okay=True, writable=True) ] = Path('.'), + measure: Annotated[ + Optional[Literal['NSAF', 'dNSAF', 'EMPAI', 'SIN']], + typer.Option('--measure', help='Spectral-counting measure [dim][default: config quantify.measure][/dim]') + ] = None, + qvalue_threshold: Annotated[ + Optional[float], + typer.Option('--qvalue-threshold', help='PSM q-value threshold for inclusion [dim][default: config quantify.qvalue_threshold][/dim]', min=0.0, max=1.0) + ] = None, + unique_mapping: Annotated[ + Optional[bool], + typer.Option('--unique-mapping/--no-unique-mapping', help='Require unique peptide-to-protein mapping [dim][default: config quantify.unique_mapping][/dim]') + ] = None, ): ctx = ExperimentContext.resolve(experiment_dir) - quantifyFuncs.run_quantify(psm_dir, database, ctx) \ No newline at end of file + quantifyFuncs.run_quantify( + psm_dir, + database, + ctx, + measure=measure, + qvalue_threshold=qvalue_threshold, + unique_mapping=unique_mapping, + ) \ No newline at end of file diff --git a/src/comms/commands/quantify.py b/src/comms/commands/quantify.py index 75cb4c3..68202f1 100644 --- a/src/comms/commands/quantify.py +++ b/src/comms/commands/quantify.py @@ -11,12 +11,21 @@ # -- Import internal functions from comms.utils.log import configureFileLogging, logMsg from comms.utils.context import ExperimentContext, resolve_database, resolve_results_input +from comms.utils.settings import resolve_config_value, _writeConfigTo from comms.utils.validate import validate from comms.utils import crux as cruxutil from comms.utils import paths as pathutil -# -- run_quantify: runs dNSAF spectral counting on rescored PSM files (discovering single/multi-species results) and writes results to output -def run_quantify(input_dir, database, ctx: ExperimentContext, in_pipeline: bool = False): +# -- run_quantify: run spectral counting on rescored PSM files (discovering single/multi-species results) and writes results to output +def run_quantify( + input_dir, + database, + ctx: ExperimentContext, + measure=None, + qvalue_threshold=None, + unique_mapping=None, + in_pipeline: bool = False, +): if not in_pipeline: logMsg('quantify') logMsg.debug('Started command: quantify') @@ -40,6 +49,21 @@ def run_quantify(input_dir, database, ctx: ExperimentContext, in_pipeline: bool log_path = out_dir / 'quantify.log' configureFileLogging(log_path) logMsg.debug(f'Output log file: {log_path}') + + # Build config for this run only if any override was given + overrides_given = any(v is not None for v in (measure, qvalue_threshold, unique_mapping)) + if overrides_given: + logMsg.debug('Using run-specific configuration parameters') + run_config = {**ctx.config, 'quantify': dict(ctx.config.get('quantify', {}))} + run_config['quantify']['measure'] = resolve_config_value(ctx.config, 'quantify', 'measure', measure) + run_config['quantify']['qvalue_threshold'] = resolve_config_value(ctx.config, 'quantify', 'qvalue_threshold', qvalue_threshold) + run_config['quantify']['unique_mapping'] = resolve_config_value(ctx.config, 'quantify', 'unique_mapping', unique_mapping) + logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "quantify.config.toml"') + _writeConfigTo(run_config, path=Path(out_dir, 'quantify.config.toml')) + else: + logMsg.debug('Using contextual configuration parameters') + run_config = ctx.config + n_ok, n_fail = 0, 0 with logging_redirect_tqdm(): for psm_file in tqdm(psm_files, desc='Files quantified'): @@ -51,7 +75,7 @@ def run_quantify(input_dir, database, ctx: ExperimentContext, in_pipeline: bool database=database, out_dir=out_dir, fileroot=fileroot, - config=ctx.config, + config=run_config, ) if ok: n_ok += 1 From 799e87e777e547696004fe7672e5980e72d760d4 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 15:43:04 +0100 Subject: [PATCH 057/108] feat(report): align command cli with config - Modified cli/report.py and commands/report.py to align command logic and orchestration with configuration options and CLI overrides. - Modified r/sections/da.R to update argument parsing and use set config value for labelling proteins in volcano plots. --- src/comms/cli/report.py | 35 +++++++++++++++++++--------------- src/comms/commands/report.py | 37 +++++++++++++++++++++++++++--------- src/comms/r/sections/da.R | 3 ++- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/comms/cli/report.py b/src/comms/cli/report.py index a48f738..04a99b8 100644 --- a/src/comms/cli/report.py +++ b/src/comms/cli/report.py @@ -23,15 +23,15 @@ def report( organism_prefix: Annotated[ Optional[str], - typer.Option('-o', '--organism-prefix', help='ID prefix for the primary organism [dim][default: experiment organism prefix][/dim]') + typer.Option('-o', '--organism-prefix', help='ID prefix for the primary organism [dim]\\[default: experiment organism prefix][/dim]') ] = None, quantify_dir: Annotated[ Optional[Path], - typer.Option('-q', '--quantify-dir', help='Path to quantification results [dim][default: quantify output][/dim]') + typer.Option('-q', '--quantify-dir', help='Path to quantification results [dim]\\[default: quantify output][/dim]') ] = None, sample_sheet: Annotated[ Optional[Path], - typer.Option('-s', '--sample-sheet', help='Path to sample sheet [dim][default: experiment sample sheet][/dim]') + typer.Option('-s', '--sample-sheet', help='Path to sample sheet [dim]\\[default: experiment sample sheet][/dim]') ] = None, experiment_dir: Annotated[ Optional[Path], @@ -39,28 +39,32 @@ def report( ] = Path('.'), lfq_dir: Annotated[ Optional[Path], - typer.Option('-l', '--lfq-dir', help='Path to LFQ results [dim][default: lfq output][/dim]') + typer.Option('-l', '--lfq-dir', help='Path to LFQ results [dim]\\[default: lfq output][/dim]') ] = None, ref_info: Annotated[ Optional[Path], - typer.Option('-r', '--ref-info', help='Protein metadata TSV [dim][default: experiment ref_info][/dim]') + typer.Option('-r', '--ref-info', help='Protein metadata TSV [dim]\\[default: experiment ref_info][/dim]') ] = None, cont_csv: Annotated[ Optional[Path], - typer.Option('-c', '--cont-csv', help='Contaminant annotations CSV [dim][default: experiment cont_csv][/dim]') + typer.Option('-c', '--cont-csv', help='Contaminant annotations CSV [dim]\\[default: experiment cont_csv][/dim]') ] = None, min_reps: Annotated[ - int, - typer.Option('--min-reps', help='Minimum replicates per fraction-treatment group', min=1) - ] = 3, + Optional[int], + typer.Option('--min-reps', help='Minimum replicates per fraction-treatment group [dim]\\[default: config report.min_reps][/dim]', min=1) + ] = None, lfc_threshold: Annotated[ - float, - typer.Option('--lfc-threshold', help='|log2FC| threshold for DA', min=0.0) - ] = 1.0, + Optional[float], + typer.Option('--lfc-threshold', help='|log2FC| threshold for DA [dim]\\[default: config report.lfc_threshold][/dim]', min=0.0) + ] = None, fdr_threshold: Annotated[ - float, - typer.Option('--fdr-threshold', help='BH-FDR threshold for DA', min=0.0, max=1.0) - ] = 0.05, + Optional[float], + typer.Option('--fdr-threshold', help='BH-FDR threshold for DA [dim]\\[default: config report.fdr_threshold][/dim]', min=0.0, max=1.0) + ] = None, + top_n: Annotated[ + Optional[int], + typer.Option('--top-n', help='Number of top DA proteins labelled per volcano plot [dim]\\[default: config report.top_n_proteins][/dim]', min=1) + ] = None, section: Annotated[ Optional[list[str]], typer.Option('--section', help='Section(s) to run (repeatable)') @@ -91,6 +95,7 @@ def report( min_reps=min_reps, lfc_threshold=lfc_threshold, fdr_threshold=fdr_threshold, + top_n=top_n, sections=sections, overwrite=overwrite, rscript=rscript, diff --git a/src/comms/commands/report.py b/src/comms/commands/report.py index 0b4d0de..3808140 100644 --- a/src/comms/commands/report.py +++ b/src/comms/commands/report.py @@ -13,6 +13,7 @@ # -- Import internal functions from comms.utils.log import logMsg from comms.utils.samples import loadSampleSheet +from comms.utils.settings import resolve_config_value, _writeConfigTo from comms.utils.context import ExperimentContext, resolve_organism_prefix, resolve_sample_sheet, resolve_results_input, results_dir # -- Initialise Rich console @@ -131,9 +132,10 @@ def run_report( ref_info: Path | None, cont_csv: Path | None, organism_prefix: str | None, - min_reps: int, - lfc_threshold: float, - fdr_threshold: float, + min_reps: int | None, + lfc_threshold: float | None, + fdr_threshold: float | None, + top_n: int | None, sections: list, overwrite: bool, rscript: str, @@ -188,6 +190,22 @@ def run_report( if shutil.which(rscript) is None: logMsg.error(f'Rscript not callable: {rscript}') raise SystemExit(1) + + # Build config for this run only if any override was given + overrides_given = any(v is not None for v in (min_reps, lfc_threshold, fdr_threshold, top_n)) + if overrides_given: + logMsg.debug('Using run-specific configuration parameters') + run_config = {**ctx.config, 'report': dict(ctx.config.get('report', {}))} + run_config['report']['min_reps'] = resolve_config_value(ctx.config, 'report', 'min_reps', min_reps) + run_config['report']['lfc_threshold'] = resolve_config_value(ctx.config, 'report', 'lfc_threshold', lfc_threshold) + run_config['report']['fdr_threshold'] = resolve_config_value(ctx.config, 'report', 'fdr_threshold', fdr_threshold) + run_config['report']['top_n_proteins'] = resolve_config_value(ctx.config, 'report', 'top_n_proteins', top_n) + logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "report.config.toml"') + _writeConfigTo(run_config, path=Path(out_dir, 'report.config.toml')) + else: + logMsg.debug('Using contextual configuration parameters') + run_config = ctx.config + # Run command logMsg.info(f'Generating report: {len(sections)} section(s)') # Define arguments passed to every R script @@ -197,7 +215,7 @@ def run_report( str(ref_info) if ref_info else '', str(cont_csv) if cont_csv else '', organism_prefix, - str(min_reps), + str(run_config['report']['min_reps']), ] section_status: dict[str, str] = {} organism_results: dict[str, dict[str, str]] = {} @@ -206,9 +224,9 @@ def run_report( script, needs_lfq, per_organism = _SECTIONS[sec] extra: list[str] = [] if sec == 'da': - extra = [str(lfc_threshold), str(fdr_threshold)] + extra = [str(run_config['report']['lfc_threshold']), str(run_config['report']['fdr_threshold']), str(run_config['report']['top_n_proteins'])] elif sec == 'concordance': - extra = [str(lfq_dir), str(lfc_threshold), str(fdr_threshold)] + extra = [str(lfq_dir), str(run_config['report']['lfc_threshold']), str(run_config['report']['fdr_threshold'])] output_subdir = output_dir / sec.replace('-', '_') proc_ok = _run_r_section( section = sec, @@ -230,9 +248,10 @@ def run_report( 'sample_sheet': sample_sheet, 'lqf_dir': lfq_dir or 'not provided', 'organism_prefix': organism_prefix, - 'min_reps': min_reps, - 'lfc_threshold': lfc_threshold, - 'fdr_threshold': fdr_threshold, + 'min_reps': run_config['report']['min_reps'], + 'lfc_threshold': run_config['report']['lfc_threshold'], + 'fdr_threshold': run_config['report']['fdr_threshold'], + 'top_n_proteins': run_config['report']['top_n_proteins'], }, section_status, organism_results, diff --git a/src/comms/r/sections/da.R b/src/comms/r/sections/da.R index aec234e..6dffbdc 100644 --- a/src/comms/r/sections/da.R +++ b/src/comms/r/sections/da.R @@ -12,6 +12,7 @@ organism_prefix <- args[6] min_reps <- as.integer(args[7]) lfc_threshold <- as.numeric(args[8]) fdr_threshold <- as.numeric(args[9]) +top_n <- as.integer(args[10]) # Get script directory for path traversal script_dir <- local({ @@ -100,7 +101,7 @@ for (org in organisms) { da_results_all[[key]] <- da_res top_labels <- filter(da_res, Abundance != "Unchanged") %>% - slice_min(adj_pval, n=20) + slice_min(adj_pval, n=top_n) volcano <- ggplot(da_res, aes(x=log2FC, y=-log10(adj_pval), colour=Abundance)) + geom_point(alpha=0.7, size=1.5) + geom_hline(yintercept=-log10(fdr_threshold), linetype="dashed", colour="grey50") + From e3fbf63290270033d2056295e7426f34f2008fab Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 15:48:22 +0100 Subject: [PATCH 058/108] fix(pipeline): fix pipeline overriding config - Modified commands/pipeline.py to fix issue with hard-coded values overriding config-read values when using pipeline command. --- src/comms/commands/pipeline.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/comms/commands/pipeline.py b/src/comms/commands/pipeline.py index 965ad06..15f531d 100644 --- a/src/comms/commands/pipeline.py +++ b/src/comms/commands/pipeline.py @@ -47,19 +47,27 @@ def run_pipeline( raise SystemExit(1) logMsg.debug(f"Sample sheet loaded: {len(samples)} sample(s); {samples['treatment'].nunique()} treatment(s)") logMsg.info(f"Running comMS pipeline: {len(samples)} sample(s), {samples['treatment'].nunique()} treatment(s)") + # -- Step 1: Convert (optional) if not skip_convert: current_step += 1 logMsg.progress(f'Step {current_step}/{num_steps}: converting .RAW files') - convert.run_convert(data_files, ctx=ctx, gzip=True, in_pipeline=True) + convert.run_convert( + data_files, + ctx=ctx, + gzip=None, + in_pipeline=True + ) mzml_override = None # search/lfq glob the convert results else: logMsg.progress(f'Skipped .RAW -> .mzML conversion') mzml_override = [f for f in data_files if f.suffix.lower() == '.mzml' or f.name.endswith('.mzML.gz')] + # -- Step 2: Build index current_step += 1 logMsg.progress(f'Step {current_step}/{num_steps}: building peptide index') index.run_index(database=database, ctx=ctx, in_pipeline=True) + # -- Step 3: Search current_step += 1 logMsg.progress(f'Step {current_step}/{num_steps}: searching spectra') @@ -71,6 +79,7 @@ def run_pipeline( threads=threads, in_pipeline=True ) + # -- Step 4: Rescore current_step += 1 logMsg.progress(f'Step {current_step}/{num_steps}: rescoring PSMs') @@ -80,7 +89,8 @@ def run_pipeline( ctx=ctx, organism_tags=org_tags, in_pipeline=True -) + ) + # -- Steps 5 & 6: Quantify if skip_lfq and skip_quantify: logMsg.progress(f'Skipped LFQ and dNSAF quantification') @@ -113,9 +123,10 @@ def run_pipeline( cont_csv=None, organism_prefix=None, # ! TODO: make below configurable via CLI or config? - min_reps=3, - fdr_threshold=0.05, - lfc_threshold=1.0, + min_reps=None, + fdr_threshold=None, + lfc_threshold=None, + top_n=None, sections=VALID_SECTIONS, overwrite=False, rscript='Rscript', From 83025626c88301b5b95824437633e9ea0f347b41 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 15:50:15 +0100 Subject: [PATCH 059/108] fix(pipeline): remove outdated comment --- src/comms/commands/pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/comms/commands/pipeline.py b/src/comms/commands/pipeline.py index 15f531d..34c8bbc 100644 --- a/src/comms/commands/pipeline.py +++ b/src/comms/commands/pipeline.py @@ -122,7 +122,6 @@ def run_pipeline( ref_info=None, cont_csv=None, organism_prefix=None, - # ! TODO: make below configurable via CLI or config? min_reps=None, fdr_threshold=None, lfc_threshold=None, From a4a83de6a08b3e75ebc92967f0d4012be6657869 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 15:56:31 +0100 Subject: [PATCH 060/108] refactor(search): de-duplicate search configs - Modified commands/search.py and utils/crux.py to de-duplicate search config parsing. --- src/comms/commands/search.py | 5 ----- src/comms/utils/crux.py | 17 ++++++----------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/comms/commands/search.py b/src/comms/commands/search.py index 2ca9eb3..0a4cc18 100644 --- a/src/comms/commands/search.py +++ b/src/comms/commands/search.py @@ -79,11 +79,6 @@ def run_search( out_dir=out_dir, fileroot=fileroot, config=run_config, - threads=run_config['search']['threads'], - score_function=run_config['search']['score_function'], - min_peaks=run_config['search']['min_peaks'], - precursor_tol=run_config['search']['precursor_tolerance_ppm'], - mz_bin_width=run_config['search']['mz_bin_width'], ) if ok: n_ok += 1 diff --git a/src/comms/utils/crux.py b/src/comms/utils/crux.py index ecbc059..2b7d8da 100644 --- a/src/comms/utils/crux.py +++ b/src/comms/utils/crux.py @@ -114,22 +114,17 @@ def paramMedic(crux_bin: Path, mzml_file: Path, out_dir: Path) -> bool: return runCrux(crux_bin, 'param-medic', args) # -- tideSearch: returns True if Tide-search completed successfully for the given mzML file, False on failure -def tideSearch(crux_bin: Path, mzml_file: Path, index_dir: Path, out_dir: Path, fileroot: str, config: dict, threads, precursor_tol=None, mz_bin_width=None, score_function=None, min_peaks=None) -> bool: +def tideSearch(crux_bin: Path, mzml_file: Path, index_dir: Path, out_dir: Path, fileroot: str, config: dict) -> bool: logMsg.debug(f'tide-search: {mzml_file.name}') - prec = precursor_tol if precursor_tol is not None else config['search']['precursor_tolerance_ppm'] - bin_width = mz_bin_width if mz_bin_width is not None else config['search']['mz_bin_width'] - score_fn = score_function or config['search']['score_function'] - peaks = min_peaks or config['search']['min_peaks'] - logMsg.debug(f'Precursor tolerance {prec} ppm, m/z bin width {bin_width}') args = [ '--verbosity', '40', - '--num-threads', threads, + '--num-threads', config['search']['threads'], '--spectrum-parser', 'pwiz', - '--precursor-window', str(prec), + '--precursor-window', str(config['search']['precursor_tolerance_ppm']), '--precursor-window-type', 'ppm', - '--mz-bin-width', str(bin_width), - '--score-function', score_fn, - '--min-peaks', str(peaks), + '--mz-bin-width', str(config['search']['mz_bin_width']), + '--score-function', config['search']['score_function'], + '--min-peaks', str(config['search']['min_peaks'],), '--missed-cleavages', str(config['index']['missed_cleavages']), '--output-dir', str(out_dir), '--fileroot', fileroot, From 4140444e5d801862e971bc6eaed1f23e36b0232e Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 16:12:09 +0100 Subject: [PATCH 061/108] fix(report): fix variable name in report command - Modified commands/report.py to use correct output directory variable. --- src/comms/commands/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/commands/report.py b/src/comms/commands/report.py index 3808140..5905775 100644 --- a/src/comms/commands/report.py +++ b/src/comms/commands/report.py @@ -201,7 +201,7 @@ def run_report( run_config['report']['fdr_threshold'] = resolve_config_value(ctx.config, 'report', 'fdr_threshold', fdr_threshold) run_config['report']['top_n_proteins'] = resolve_config_value(ctx.config, 'report', 'top_n_proteins', top_n) logMsg.info('Command-line overrides detected - run configuration file will be saved to output folder as "report.config.toml"') - _writeConfigTo(run_config, path=Path(out_dir, 'report.config.toml')) + _writeConfigTo(run_config, path=Path(output_dir, 'report.config.toml')) else: logMsg.debug('Using contextual configuration parameters') run_config = ctx.config From a97cb4ede81781075c64e6e1c505e41cdceb4fd2 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 16:15:23 +0100 Subject: [PATCH 062/108] fix(settings): add missing library import --- src/comms/utils/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/utils/settings.py b/src/comms/utils/settings.py index 7ee5e51..ad45545 100644 --- a/src/comms/utils/settings.py +++ b/src/comms/utils/settings.py @@ -3,7 +3,7 @@ ''' # -- Import external dependencies -import tomllib +import tomllib, tomli_w from importlib.resources import files as pkg_files from pathlib import Path from platformdirs import user_config_dir From fbb1c8f6c99817f1858da78d3c846e7b2b0ad56a Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Fri, 10 Jul 2026 16:47:33 +0100 Subject: [PATCH 063/108] test(conftest): update binary directory - Modified tests/conftest.py to update expected directory for Crux and ThermoRawFileParser binaries to tests/bin/. --- tests/conftest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 824d2e6..0470fcc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,10 +12,9 @@ from PySide6.QtCore import qInstallMessageHandler, QtMsgType from typing import Optional -# -- Define root directories external dependencies -TESTS_DIR = Path(__file__).parent -REPO_ROOT = TESTS_DIR.parent -BIN_DIR = REPO_ROOT / 'bin' + +# -- Define bin directory for tests requiring Crux/ThermoRawFileParser +BIN_DIR = Path(__file__).parent / 'bin' # -- Import internal dependencies from tests.fixtures.generate_fixtures import generate_all, write_fasta, write_mzml From 664a6297e1bce189a8d82535ce72bbbd572e5f1f Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Mon, 27 Jul 2026 12:03:28 +0100 Subject: [PATCH 064/108] test(conftest): update test suite fixtures --- tests/conftest.py | 156 +++++++++++++++++++++++----------------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0470fcc..9394ef7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,51 +86,29 @@ def synthetic_fixtures(tmp_path: Path) -> tuple[Path, Path]: # -- Create sample sheet fixtures @pytest.fixture() -def valid_sample_sheet(tmp_path: Path) -> Path: +def sample_sheet_factory(tmp_path: Path): ''' - Write a minimal valid comMS sample sheet (TSV) and return its path + Returns a function that writes a minimal comMS sample sheet (TSV) with the + given fractions (two treatments, one replicate each, optional batch column) + and returns its path. ''' - content = ( - 'sample_id\traw_file\ttreatment\tfraction\treplicate\tbatch\n' - 'S1\tsynthetic.mzML\tCONTROL\tWCL\t1\tA\n' - 'S2\tsynthetic.mzML\tTREATMENT\tWCL\t1\tA\n' - ) - p = tmp_path / 'sample_sheet.tsv' - p.write_text(content) - return p - -@pytest.fixture() -def valid_sample_sheet_single_fraction(tmp_path: Path) -> Path: - ''' - Write a minimal sample sheet with a single fraction (WCL) and return its path - ''' - content = ( - 'sample_id\traw_file\ttreatment\tfraction\treplicate\n' - 'S1\tsample_mock_wcl_1.RAW\tMOCK\tWCL\t1\n' - 'S2\tsample_treat_wcl_1.RAW\tTREAT\tWCL\t1\n' - ) - p = tmp_path / 'sample_sheet_single_fraction.tsv' - p.write_text(content) - return p - -@pytest.fixture() -def valid_sample_sheet_multiple_fractions(tmp_path: Path) -> Path: - ''' - Write a minimal sample sheet with three fractions (WCL, ECF, PUR), two treatments, and one - replicate each and return its path - ''' - content = ( - 'sample_id\traw_file\ttreatment\tfraction\treplicate\tbatch\n' - 'S1\tsample_mock_wcl_1.RAW\tMOCK\tWCL\t1\tA\n' - 'S2\tsample_treat_wcl_1.RAW\tTREAT\tWCL\t1\tA\n' - 'S3\tsample_mock_ecf_1.RAW\tMOCK\tECF\t1\tA\n' - 'S4\tsample_treat_ecf_1.RAW\tTREAT\tECF\t1\tA\n' - 'S5\tsample_mock_pur_1.RAW\tMOCK\tPUR\t1\tA\n' - 'S6\tsample_treat_pur_1.RAW\tTREAT\tPUR\t1\tA\n' - ) - p = tmp_path / 'sample_sheet_multiple_fractions.tsv' - p.write_text(content) - return p + def _make(fractions: list[str] = ('WCL',), batch: bool = True) -> Path: + columns = ['sample_id', 'raw_file', 'treatment', 'fraction', 'replicate'] + if batch: + columns.append('batch') + lines = ['\t'.join(columns)] + for i, fraction in enumerate(fractions): + for j, treatment in enumerate(('MOCK', 'TREAT')): + sample_id = f'S{i * 2 + j + 1}' + raw_file = f'sample_{treatment.lower()}_{fraction.lower()}_1.RAW' + row = [sample_id, raw_file, treatment, fraction, '1'] + if batch: + row.append('A') + lines.append('\t'.join(row)) + p = tmp_path / f'sample_sheet_{"_".join(fractions).lower()}.tsv' + p.write_text('\n'.join(lines) + '\n') + return p + return _make @pytest.fixture() def sample_sheet_missing_col(tmp_path: Path) -> Path: @@ -192,40 +170,21 @@ def synthetic_percolator_results(tmp_path): return rescore_dir @pytest.fixture() -def multi_fraction_psm_dir(tmp_path: Path) -> Path: +def psm_dir_factory(tmp_path: Path): ''' - Write synthetic Percolator PSM files for three fractions (WCL, AWF, EV), two samples per fraction, matching the filenames in valid_multi_fraction_sample_sheet. and return the directory path + Returns a function that writes one synthetic Percolator PSM file per given + stem under comms/results/rescore/, and returns that directory. ''' - rescore_dir = tmp_path / 'comms' / 'results' / 'rescore' - rescore_dir.mkdir(parents=True) psm_header = 'PSMId\tscore\tq-value\tposterior_error_prob\tpeptide\tproteinIds\n' psm_row = 'synthetic_1\t1.5\t0.01\t0.001\tK.ACDEFGHIK.L\tSP|PROT1|GENE1\n' - stems = [ - 'sample_mock_wcl_1', - 'sample_treat_wcl_1', - 'sample_mock_ecf_1', - 'sample_treat_ecf_1', - 'sample_mock_pur_1', - 'sample_treat_pur_1', - ] - for stem in stems: - psm_file = rescore_dir / f'{stem}.percolator.target.psms.txt' - psm_file.write_text(psm_header + psm_row) - return rescore_dir -@pytest.fixture() -def single_fraction_psm_dir(tmp_path: Path) -> Path: - ''' - Write synthetic Percolator PSM files for a single fraction (WCL), matching the filenames in valid_sample_sheet_single_fraction - ''' - rescore_dir = tmp_path / 'comms' / 'results' / 'rescore' - rescore_dir.mkdir(parents=True) - psm_header = 'PSMId\tscore\tq-value\tposterior_error_prob\tpeptide\tproteinIds\n' - psm_row = 'synthetic_1\t1.5\t0.01\t0.001\tK.ACDEFGHIK.L\tSP|PROT1|GENE1\n' - for stem in ('sample_mock_wcl_1', 'sample_treat_wcl_1'): - psm_file = rescore_dir / f'{stem}.percolator.target.psms.txt' - psm_file.write_text(psm_header + psm_row) - return rescore_dir + def _make(stems: list[str]) -> Path: + rescore_dir = tmp_path / 'comms' / 'results' / 'rescore' + rescore_dir.mkdir(parents=True, exist_ok=True) + for stem in stems: + (rescore_dir / f'{stem}.percolator.target.psms.txt').write_text(psm_header + psm_row) + return rescore_dir + return _make # -- Define session-scoped QApplication for GUI tests @pytest.fixture(scope='session') @@ -255,11 +214,52 @@ def _qt_message_handler(mode: QtMsgType, context, message: str) -> None: # -- Add a function-scoped experiment context for most tests @pytest.fixture() -def experiment_ctx(tmp_path, monkeypatch): +def experiment_ctx(tmp_path, isolated_config_dir): '''A bare ExperimentContext rooted at tmp_path (no experiment.toml)''' - monkeypatch.setattr( - 'comms.utils.settings.globalConfigPath', - lambda: tmp_path / '_no_global_config.toml', - ) from comms.utils.context import ExperimentContext - return ExperimentContext.resolve(tmp_path) \ No newline at end of file + return ExperimentContext.resolve(tmp_path) + +# -- Add a fixture for integration-layer tests +@pytest.fixture() +def experiment_builder(tmp_path: Path, isolated_config_dir, sample_sheet_factory, psm_dir_factory): + ''' + Compose a comms/ directory from only the pieces a test asks for. + Usage: root, ctx = experiment_builder.with_sample_sheet().with_stage_output('rescore').build() + ''' + import tomli_w + from comms.utils.context import ExperimentContext + + class _Builder: + def __init__(self): + self._metadata: dict = {'experiment': {'name': 'exp', 'updated': '2026-01-01T00:00:00+00:00'}} + self._sample_sheet_path: Path | None = None + + def with_sample_sheet(self, fractions=('WCL',), batch=True): + self._sample_sheet_path = sample_sheet_factory(list(fractions), batch=batch) + self._metadata.setdefault('files', {})['sample_sheet'] = str(self._sample_sheet_path) + return self + + def with_stage_output(self, stage: str, files: list[str] | None = None): + if stage in ('rescore',): + psm_dir_factory(files or ['sample_mock_wcl_1', 'sample_treat_wcl_1']) + else: + out_dir = tmp_path / 'comms' / 'results' / stage + out_dir.mkdir(parents=True, exist_ok=True) + for name in (files or [f'placeholder.{stage}.txt']): + (out_dir / name).write_text('') + return self + + def with_metadata(self, **kwargs): + for section, values in kwargs.items(): + self._metadata.setdefault(section, {}).update(values) + return self + + def build(self): + comms_dir = tmp_path / 'comms' + comms_dir.mkdir(parents=True, exist_ok=True) + with (comms_dir / 'experiment.toml').open('wb') as f: + tomli_w.dump(self._metadata, f) + ctx = ExperimentContext.resolve(tmp_path) + return tmp_path, ctx + + return _Builder() \ No newline at end of file From 79465297feaf1d5d5d1735d457d67599e6093f5f Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Mon, 10 Aug 2026 19:04:35 +0100 Subject: [PATCH 065/108] test: update unit tests - Modified tests/unit/gui/test_gui_panels.py to add new unit tests for config and sample panels. - Added tests/unit/gui/test_gui_readiness.py to define unit tests for readiness panel. - Modified tests/unit/test_context.py to add unit tests for resolving report command. - Added tests/unit/test_license.py to define unit tests for license subcommand. - Added tests/unit/test_modspec.py to define unit tests for all modifications. - Modified tests/unit/test_rescore.py to add unit tests for combined Percolator round and for finding protein ID index. - Added tests/unit/test_sheet.py to define unit tests for sample sheet parsing and rendering. - Modified tests/unit/test_validate.py to add unit tests for probing binaries. --- tests/unit/gui/test_gui_panels.py | 65 +++++++++- tests/unit/gui/test_gui_readiness.py | 75 ++++++++++++ tests/unit/test_context.py | 76 +++++++++++- tests/unit/test_license.py | 21 ++++ tests/unit/test_modspec.py | 173 +++++++++++++++++++++++++++ tests/unit/test_rescore.py | 57 ++++++++- tests/unit/test_sheet.py | 63 ++++++++++ tests/unit/test_validate.py | 27 ++++- 8 files changed, 550 insertions(+), 7 deletions(-) create mode 100644 tests/unit/gui/test_gui_readiness.py create mode 100644 tests/unit/test_license.py create mode 100644 tests/unit/test_modspec.py create mode 100644 tests/unit/test_sheet.py diff --git a/tests/unit/gui/test_gui_panels.py b/tests/unit/gui/test_gui_panels.py index da54e14..77e6e7b 100644 --- a/tests/unit/gui/test_gui_panels.py +++ b/tests/unit/gui/test_gui_panels.py @@ -10,7 +10,6 @@ from unittest.mock import patch # -- Import functions under test -from comms.commands.config import MET_OX_MOD from comms.gui.status import PanelStatus from comms.gui.models.experiment_state import ExperimentState from comms.gui.models.sample_table import COL_TREATMENT, COL_FRACTION @@ -18,6 +17,7 @@ from comms.gui.panels.experiment_panel import ExperimentPanel from comms.gui.panels.sample_panel import SamplePanel from comms.gui.panels.save_panel import SavePanel +from comms.utils.modspec import MET_OX_MOD # -- Helper: bring a sample panel's state to completeness def _complete_sample(state): @@ -107,6 +107,27 @@ def test_changed_signal_and_tracker_update(self): def test_summary_reports_single_species(self): assert 'single species' in ConfigPanel().summary() + def test_loads_string_clip_n_met_as_checked(self, qapp): + panel = ConfigPanel() + panel.load_from_config({'index': {'clip_n_met': 'true'}}, {}) + assert panel._clip_met.isChecked() is True + + def test_loads_string_clip_n_met_false_as_unchecked(self, qapp): + panel = ConfigPanel() + panel.load_from_config({'index': {'clip_n_met': 'false'}}, {}) + assert panel._clip_met.isChecked() is False + + def test_loads_bool_clip_n_met(self, qapp): + panel = ConfigPanel() + panel.load_from_config({'index': {'clip_n_met': True}}, {}) + assert panel._clip_met.isChecked() is True + + def test_build_config_always_writes_bool(self, qapp): + panel = ConfigPanel() + panel.load_from_config({'index': {'clip_n_met': 'true'}}, {}) + cfg = panel._build_config() + assert isinstance(cfg['index']['clip_n_met'], bool) + # -- Define tests for experiment_panel class TestExperimentPanel: def test_experiment_name_strips_whitespace(self): @@ -244,6 +265,48 @@ def test_data_files_skips_empty_paths(self, tmp_path): assert path in result assert '' not in result + def test_load_populates_treatments_and_fractions(self, qapp): + state = ExperimentState() + panel = SamplePanel(state) + panel.load(rows=[], treatments=['MOCK', 'MYC'], fractions=['WCL', 'AWF']) + assert 'MOCK' in state.treatments and 'MYC' in state.treatments + assert 'WCL' in state.fractions and 'AWF' in state.fractions + + def test_load_matches_source_path_by_filename(self, tmp_path, qapp): + from comms.utils.sheet import SampleRow + state = ExperimentState() + panel = SamplePanel(state) + row = SampleRow(sample_id='S1', raw_file='s1.RAW', source_path='') + data_file = tmp_path / 's1.RAW' + panel.load(rows=[row], treatments=[], fractions=[], data_files=[str(data_file)]) + assert state.sample_model.rows()[0].source_path == str(data_file) + + def test_load_does_not_overwrite_existing_source_path(self, tmp_path, qapp): + from comms.utils.sheet import SampleRow + state = ExperimentState() + panel = SamplePanel(state) + existing = str(tmp_path / 'already_set.RAW') + row = SampleRow(sample_id='S1', raw_file='s1.RAW', source_path=existing) + new_match = tmp_path / 's1.RAW' + panel.load(rows=[row], treatments=[], fractions=[], data_files=[str(new_match)]) + assert state.sample_model.rows()[0].source_path == existing + + def test_load_leaves_unmatched_row_source_path_empty(self, tmp_path, qapp): + from comms.utils.sheet import SampleRow + state = ExperimentState() + panel = SamplePanel(state) + row = SampleRow(sample_id='S1', raw_file='no_match.RAW', source_path='') + panel.load(rows=[row], treatments=[], fractions=[], data_files=[str(tmp_path / 'other.RAW')]) + assert state.sample_model.rows()[0].source_path == '' + + def test_load_with_no_data_files_still_sets_rows(self, qapp): + from comms.utils.sheet import SampleRow + state = ExperimentState() + panel = SamplePanel(state) + row = SampleRow(sample_id='S1', raw_file='s1.RAW') + panel.load(rows=[row], treatments=[], fractions=[], data_files=None) + assert len(state.sample_model.rows()) == 1 + class TestSavePanel: def _build(self, tmp_path): header = ExperimentPanel() diff --git a/tests/unit/gui/test_gui_readiness.py b/tests/unit/gui/test_gui_readiness.py new file mode 100644 index 0000000..4cd5e95 --- /dev/null +++ b/tests/unit/gui/test_gui_readiness.py @@ -0,0 +1,75 @@ +''' +Unit tests for src/comms/gui/panels/readiness_panel.py +''' +from unittest.mock import MagicMock, patch +import pytest + +from comms.gui.panels.readiness_panel import CommandReadinessPanel + + +def _mock_panels(**overrides): + experiment = MagicMock() + experiment.bin_dir.return_value = overrides.get('bin_dir') + experiment.database_path.return_value = overrides.get('database_path', '/db.fasta') + sample = MagicMock() + sample.data_files.return_value = overrides.get('data_files', ['/f1.RAW']) + sample.is_complete.return_value = overrides.get('sample_complete', True) + config = MagicMock() + config.organism_prefix.return_value = overrides.get('organism_prefix', 'Mtrun') + config.analysis_mode.return_value = overrides.get('analysis_mode', 'single') + config.has_organism_patterns.return_value = overrides.get('has_organism_patterns', False) + return experiment, sample, config + + +class TestReadinessPanel: + def test_refresh_dependencies_probes_with_resolved_bin_dir(self, qapp, tmp_path): + experiment, sample, config = _mock_panels(bin_dir=tmp_path) + with patch('comms.gui.panels.readiness_panel.check_r_dependencies', return_value={'installed': [], 'missing': []}), \ + patch('comms.gui.panels.readiness_panel.repoBinDir', return_value=tmp_path) as mock_repo, \ + patch('comms.gui.panels.readiness_panel.probe_crux', return_value=None) as mock_crux, \ + patch('comms.gui.panels.readiness_panel.probe_trfp', return_value=None) as mock_trfp: + panel = CommandReadinessPanel(experiment, sample, config) + panel.refresh_dependencies() + mock_repo.assert_called_with(experiment_bin_dir=tmp_path) + mock_crux.assert_called_with(tmp_path) + mock_trfp.assert_called_with(tmp_path) + + def test_bin_dir_change_is_reflected_on_next_refresh(self, qapp, tmp_path): + '''Regression test: changing bin_dir and calling refresh_dependencies() must re-probe against the new path, not a stale one.''' + experiment, sample, config = _mock_panels(bin_dir=tmp_path / 'old') + with patch('comms.gui.panels.readiness_panel.check_r_dependencies', return_value={'installed': [], 'missing': []}), \ + patch('comms.gui.panels.readiness_panel.repoBinDir', side_effect=lambda experiment_bin_dir: experiment_bin_dir), \ + patch('comms.gui.panels.readiness_panel.probe_crux', return_value=None) as mock_crux, \ + patch('comms.gui.panels.readiness_panel.probe_trfp', return_value=None): + panel = CommandReadinessPanel(experiment, sample, config) + experiment.bin_dir.return_value = tmp_path / 'new' + panel.refresh_dependencies() + mock_crux.assert_called_with(tmp_path / 'new') + + def test_crux_and_trfp_found_states_reflected(self, qapp, tmp_path): + experiment, sample, config = _mock_panels() + with patch('comms.gui.panels.readiness_panel.check_r_dependencies', return_value={'installed': ['limma'], 'missing': []}), \ + patch('comms.gui.panels.readiness_panel.repoBinDir', return_value=tmp_path), \ + patch('comms.gui.panels.readiness_panel.probe_crux', return_value=tmp_path / 'crux'), \ + patch('comms.gui.panels.readiness_panel.probe_trfp', return_value=None): + panel = CommandReadinessPanel(experiment, sample, config) + assert panel._crux_found is True + assert panel._trfp_found is False + + def test_tooltip_names_searched_directory(self, qapp, tmp_path): + experiment, sample, config = _mock_panels() + with patch('comms.gui.panels.readiness_panel.check_r_dependencies', return_value={'installed': [], 'missing': []}), \ + patch('comms.gui.panels.readiness_panel.repoBinDir', return_value=tmp_path), \ + patch('comms.gui.panels.readiness_panel.probe_crux', return_value=None), \ + patch('comms.gui.panels.readiness_panel.probe_trfp', return_value=None): + panel = CommandReadinessPanel(experiment, sample, config) + assert str(tmp_path) in panel._deps_label.toolTip() + + def test_missing_requirements_reflected_per_command(self, qapp, tmp_path): + experiment, sample, config = _mock_panels(data_files=[]) + with patch('comms.gui.panels.readiness_panel.check_r_dependencies', return_value={'installed': [], 'missing': []}), \ + patch('comms.gui.panels.readiness_panel.repoBinDir', return_value=tmp_path), \ + patch('comms.gui.panels.readiness_panel.probe_crux', return_value=tmp_path / 'crux'), \ + patch('comms.gui.panels.readiness_panel.probe_trfp', return_value=tmp_path / 'trfp'): + panel = CommandReadinessPanel(experiment, sample, config) + assert 'data files' in panel._details['convert'].text() \ No newline at end of file diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index a1664b2..9652af1 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -10,9 +10,16 @@ # -- Import functions under test from comms.utils.context import ( ExperimentContext, - _check_files, _choose, _normalise_dirs, - resolve_data_files, resolve_database, resolve_mzml_files, - resolve_organism_prefix, resolve_sample_sheet, + _check_files, + _choose, + _normalise_dirs, + resolve_data_files, + resolve_database, + resolve_mzml_files, + resolve_organism_prefix, + resolve_report, + resolve_results_input, + resolve_sample_sheet, results_dir, ) from comms.utils.settings import loadDefaultConfig @@ -97,6 +104,57 @@ def test_bin_dir_read_from_metadata(self, tmp_path, isolated_config_dir): ctx = ExperimentContext.resolve(tmp_path) assert ctx.bin_dir == Path('/opt/comms/bin') +class TestResolveResultsInput: + def test_delegates_to_results_dir(self, tmp_path): + ctx = MagicMock(spec=ExperimentContext, root=tmp_path) + result = resolve_results_input(ctx, 'search', override=None, must_exist=False) + assert result == tmp_path / 'comms' / 'results' / 'search' + + def test_must_exist_false_suppresses_existence_check(self, tmp_path): + ctx = MagicMock(spec=ExperimentContext, root=tmp_path) + # should not raise even though the directory doesn't exist + resolve_results_input(ctx, 'search', override=None, must_exist=False) + + def test_override_wins(self, tmp_path): + override_dir = tmp_path / 'custom_search' + override_dir.mkdir() + ctx = MagicMock(spec=ExperimentContext, root=tmp_path) + result = resolve_results_input(ctx, 'search', override=override_dir) + assert result == override_dir + +class TestResolveReport: + def test_no_override_enabled_true_means_do_not_skip(self): + ctx = MagicMock(spec=ExperimentContext, report_enabled=True) + assert resolve_report(ctx, override=None) is False + + def test_no_override_enabled_false_means_skip(self): + ctx = MagicMock(spec=ExperimentContext, report_enabled=False) + assert resolve_report(ctx, override=None) is True + + def test_no_override_unset_defaults_to_do_not_skip(self): + ctx = MagicMock(spec=ExperimentContext, report_enabled=None) + assert resolve_report(ctx, override=None) is False + + def test_override_agreeing_with_context_no_warning(self, caplog): + ctx = MagicMock(spec=ExperimentContext, report_enabled=True) # ctx_skip = False + caplog.clear() + resolve_report(ctx, override=False) + assert 'overrides' not in caplog.text.lower() + + def test_override_disagreeing_with_context_warns_and_wins(self, caplog): + ctx = MagicMock(spec=ExperimentContext, report_enabled=True) # ctx_skip = False + caplog.clear() + result = resolve_report(ctx, override=True) # override disagrees + assert result is True + assert 'overrides' in caplog.text.lower() + + def test_return_value_is_always_skip_polarity(self): + '''resolve_report must return "skip", never "enabled" — pin the polarity explicitly.''' + ctx = MagicMock(spec=ExperimentContext, report_enabled=True) + assert resolve_report(ctx, override=None) is False # enabled=True -> skip=False + ctx2 = MagicMock(spec=ExperimentContext, report_enabled=False) + assert resolve_report(ctx2, override=None) is True # enabled=False -> skip=True + # =========================================================================== # ExperimentContext stored-input properties (data_files, database, sample_sheet, organism_prefix, ref_info, cont_csv) # =========================================================================== @@ -186,6 +244,18 @@ def test_cont_csv_returns_none_when_absent(self, tmp_path): ctx = _make_ctx(tmp_path) assert ctx.cont_csv is None + def test_report_enabled_true(self, tmp_path): + ctx = _make_ctx(tmp_path, metadata={'report': {'enabled': True}}) + assert ctx.report_enabled is True + + def test_report_enabled_false(self, tmp_path): + ctx = _make_ctx(tmp_path, metadata={'report': {'enabled': False}}) + assert ctx.report_enabled is False + + def test_report_enabled_none_when_absent(self, tmp_path): + ctx = _make_ctx(tmp_path) + assert ctx.report_enabled is None + # =========================================================================== # results_dir # =========================================================================== diff --git a/tests/unit/test_license.py b/tests/unit/test_license.py new file mode 100644 index 0000000..ea8bb04 --- /dev/null +++ b/tests/unit/test_license.py @@ -0,0 +1,21 @@ +''' +Unit tests for src/comms/commands/license.py +''' +from unittest.mock import patch +import pytest + +from comms.commands.license import printLicense + + +class TestPrintLicense: + def test_raises_systemexit_zero(self): + with patch('pydoc.pager'): + with pytest.raises(SystemExit) as exc: + printLicense() + assert exc.value.code == 0 + + def test_reads_license_file_without_raising(self): + with patch('pydoc.pager') as mock_pager: + with pytest.raises(SystemExit): + printLicense() + mock_pager.assert_called_once() \ No newline at end of file diff --git a/tests/unit/test_modspec.py b/tests/unit/test_modspec.py new file mode 100644 index 0000000..55eb31e --- /dev/null +++ b/tests/unit/test_modspec.py @@ -0,0 +1,173 @@ +''' +Unit tests for src/comms/utils/modspec.py +''' +import pytest + +from comms.utils.modspec import ( + CARBAMIDOMETHYL_MOD, MET_OX_MOD, PHOSPHO_MOD, NCYC_MOD, NACE_MOD, + MANAGED_MOD_PATTERNS, MZ_BIN_WIDTH_HIGH_RES, MZ_BIN_WIDTH_LOW_RES, + SCORE_FUNC_HIGH_RES, SCORE_FUNC_LOW_RES, + apply_mod, apply_iodo, apply_custom_mod, apply_organism, + parse_organism_arg, apply_protocol_flags, +) + + +class TestApplyMod: + def test_adds_to_empty_spec(self): + assert apply_mod('', MET_OX_MOD) == MET_OX_MOD + + def test_adds_to_existing_spec(self): + assert MET_OX_MOD in apply_mod('1K+28.0313', MET_OX_MOD) + + def test_prepends_new_mod(self): + result = apply_mod('1K+28.0313', MET_OX_MOD) + assert result.startswith(MET_OX_MOD) + + def test_duplicate_not_added_twice(self): + result = apply_mod(MET_OX_MOD, MET_OX_MOD) + assert result.count(MET_OX_MOD) == 1 + + def test_removal_with_exclusive_pattern(self): + result = apply_mod(MET_OX_MOD, mod='', exclusive_pattern=r'^\d*M\+15\.9949') + assert MET_OX_MOD not in result + + def test_exclusive_pattern_replaces_on_add(self): + result = apply_mod('1M+999.0', MET_OX_MOD, exclusive_pattern=r'^\d*M\+') + assert '1M+999.0' not in result + assert MET_OX_MOD in result + + def test_no_leading_or_trailing_commas(self): + result = apply_mod('', MET_OX_MOD) + assert not result.startswith(',') and not result.endswith(',') + + def test_removal_of_absent_mod_is_noop(self): + assert apply_mod('1K+28.0313', mod='', exclusive_pattern=r'^\d*M\+15\.9949') == '1K+28.0313' + + +class TestApplyIodo: + def test_adds_carbamidomethyl(self): + assert CARBAMIDOMETHYL_MOD in apply_iodo('', iodo=True) + + def test_removes_carbamidomethyl(self): + assert CARBAMIDOMETHYL_MOD not in apply_iodo(CARBAMIDOMETHYL_MOD, iodo=False) + + def test_no_iodo_adds_c_plus_0(self): + assert 'C+0' in apply_iodo('', iodo=False) + + def test_idempotent(self): + result = apply_iodo(apply_iodo('', iodo=True), iodo=True) + assert result.count(CARBAMIDOMETHYL_MOD) == 1 + + +class TestApplyCustomMod: + def test_adds_entry_to_empty(self): + assert apply_custom_mod('', '1K+28.0313') == '1K+28.0313' + + def test_empty_string_clears_all(self): + assert apply_custom_mod('1K+28.0313', '') == '' + + def test_duplicate_not_added(self): + once = apply_custom_mod('', '1K+28.0313') + twice = apply_custom_mod(once, '1K+28.0313') + assert twice.count('1K+28.0313') == 1 + + @pytest.mark.parametrize('managed_entry', ['1M+15.9949', 'C+57.0215', '1STY+79.966331']) + def test_managed_mod_rejected(self, managed_entry): + assert apply_custom_mod('', managed_entry) == '' + + def test_managed_mod_patterns_is_dict_of_pattern_to_flag(self): + assert isinstance(MANAGED_MOD_PATTERNS, dict) + assert all(isinstance(k, str) and isinstance(v, str) for k, v in MANAGED_MOD_PATTERNS.items()) + + +class TestApplyOrganism: + def test_sets_organism_section(self): + cfg = apply_organism({}, {'Mt': 'MEDTR'}) + assert cfg['organism'] == {'Mt': 'MEDTR'} + + def test_replaces_existing(self): + cfg = apply_organism({'organism': {'Old': 'X'}}, {'New': 'Y'}) + assert cfg['organism'] == {'New': 'Y'} + + def test_does_not_touch_other_sections(self): + cfg = apply_organism({'search': {'threads': 4}}, {'Mt': 'MEDTR'}) + assert cfg['search'] == {'threads': 4} + + +class TestParseOrganismArg: + def test_single_pair(self): + assert parse_organism_arg(['Mt=MEDTR']) == {'Mt': 'MEDTR'} + + def test_multiple_pairs(self): + assert parse_organism_arg(['Mt=MEDTR', 'Ri=RHIIR']) == {'Mt': 'MEDTR', 'Ri': 'RHIIR'} + + def test_strips_whitespace(self): + assert parse_organism_arg([' Mt = MEDTR ']) == {'Mt': 'MEDTR'} + + def test_raises_on_missing_equals(self): + with pytest.raises(SystemExit): + parse_organism_arg(['MtMEDTR']) + + def test_raises_on_empty_key(self): + with pytest.raises(SystemExit): + parse_organism_arg(['=MEDTR']) + + def test_raises_on_empty_pattern(self): + with pytest.raises(SystemExit): + parse_organism_arg(['Mt=']) + + def test_empty_list_returns_empty_dict(self): + assert parse_organism_arg([]) == {} + + +class TestApplyProtocolFlags: + def test_low_res_sets_bin_width_and_score(self): + cfg = apply_protocol_flags({}, low_res=True) + assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_LOW_RES + assert cfg['search']['score_function'] == SCORE_FUNC_LOW_RES + + def test_high_res_sets_bin_width_and_score(self): + cfg = apply_protocol_flags({}, low_res=False) + assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_HIGH_RES + assert cfg['search']['score_function'] == SCORE_FUNC_HIGH_RES + + def test_none_flags_are_noop(self): + cfg = apply_protocol_flags({'search': {'threads': 4}}) + assert cfg['search']['threads'] == 4 + + # -- new: clip_met -- + def test_clip_met_true_sets_bool_true(self): + cfg = apply_protocol_flags({}, clip_met=True) + assert cfg['index']['clip_n_met'] is True + + def test_clip_met_false_sets_bool_false(self): + cfg = apply_protocol_flags({}, clip_met=False) + assert cfg['index']['clip_n_met'] is False + + def test_clip_met_stored_as_bool_not_string(self): + '''Regression test: clip_n_met must never be persisted as a string.''' + cfg = apply_protocol_flags({}, clip_met=True) + assert isinstance(cfg['index']['clip_n_met'], bool) + + def test_clip_met_none_is_noop(self): + cfg = apply_protocol_flags({'index': {}}, clip_met=None) + assert 'clip_n_met' not in cfg['index'] + + # -- new: missed_cleavages -- + def test_missed_cleavages_sets_value(self): + cfg = apply_protocol_flags({}, missed_cleavages=3) + assert cfg['index']['missed_cleavages'] == 3 + + def test_missed_cleavages_none_is_noop(self): + cfg = apply_protocol_flags({'index': {}}, missed_cleavages=None) + assert 'missed_cleavages' not in cfg['index'] + + def test_missed_cleavages_and_clip_met_coexist(self): + cfg = apply_protocol_flags({}, clip_met=True, missed_cleavages=2) + assert cfg['index']['clip_n_met'] is True + assert cfg['index']['missed_cleavages'] == 2 + + def test_mods_flags_unaffected_by_new_flags(self): + cfg = apply_protocol_flags({}, ox=True, clip_met=True, missed_cleavages=1) + assert MET_OX_MOD in cfg['index']['mods_spec'] + assert cfg['index']['clip_n_met'] is True \ No newline at end of file diff --git a/tests/unit/test_rescore.py b/tests/unit/test_rescore.py index ef8b165..ec2885d 100644 --- a/tests/unit/test_rescore.py +++ b/tests/unit/test_rescore.py @@ -5,9 +5,17 @@ # -- Import external dependencies import pytest from pathlib import Path +from unittest.mock import patch # -- Import functions under test -from comms.commands.rescore import _parseOrganismTags, _classifyPsmRow, _splitPsmsByOrganism +from comms.commands.rescore import ( + _classifyPsmRow, + _findProteinIdsIndex, + _parseOrganismTags, + _run_combined_percolator_round, + _run_per_organism_percolator_round, + _splitPsmsByOrganism, +) # -- Define constants PSM_HEADER = 'scan\tcharge\tspectrum precursor m/z\tpeptide\tflanking aa\tprotein id\n' @@ -95,6 +103,15 @@ def test_returns_multiple_labels_for_shared_psm(self): assert len(result) == 2 assert 'EUK' in result and 'ALSO' in result +class TestFindProteinIdsIndex: + def test_returns_correct_index(self): + header = 'scan\tcharge\tprotein id\tpeptide\n' + assert _findProteinIdsIndex(header, 'protein id') == 2 + + def test_first_column_returns_zero(self): + header = 'protein id\tpeptide\n' + assert _findProteinIdsIndex(header, 'protein id') == 0 + # -- Define tests for _splitPsmsByOrganism helper function class TestSplitPsmsByOrganism: @pytest.fixture() @@ -201,4 +218,40 @@ def test_shared_policy_include_duplicates_shared_psms(self, tmp_path): for label in ('EUK', 'PRO'): f = tmp_path / label / f'synthetic.{label}.tide-search.target.txt' if f.exists(): - assert 'TESTEUK_TESTPRO' in f.read_text() \ No newline at end of file + assert 'TESTEUK_TESTPRO' in f.read_text() + +class TestRunCombinedPercolatorRound: + def test_returns_sorted_output_files(self, tmp_path): + target_files = [tmp_path / 'a.tide-search.target.txt', tmp_path / 'b.tide-search.target.txt'] + for f in target_files: + f.write_text('header\n') + + def _mock_percolator(**kwargs): + fileroot = kwargs['fileroot'] + (kwargs['out_dir'] / f'{fileroot}.percolator.target.psms.txt').write_text('x') + return True + + with patch('comms.commands.rescore.cruxutil.percolator', side_effect=_mock_percolator): + result = _run_combined_percolator_round('crux_bin', target_files, 'db.fasta', tmp_path, {}) + assert result == sorted(tmp_path.glob('*.percolator.target.psms.txt')) + + def test_empty_input_returns_empty_list(self, tmp_path): + result = _run_combined_percolator_round('crux_bin', [], 'db.fasta', tmp_path, {}) + assert result == [] + + def test_failed_file_does_not_stop_the_rest(self, tmp_path): + target_files = [tmp_path / 'a.tide-search.target.txt', tmp_path / 'b.tide-search.target.txt'] + for f in target_files: + f.write_text('header\n') + call_count = {'n': 0} + + def _mock_percolator(**kwargs): + call_count['n'] += 1 + if call_count['n'] == 1: + return False # first file fails + (kwargs['out_dir'] / f"{kwargs['fileroot']}.percolator.target.psms.txt").write_text('x') + return True + + with patch('comms.commands.rescore.cruxutil.percolator', side_effect=_mock_percolator): + result = _run_combined_percolator_round('crux_bin', target_files, 'db.fasta', tmp_path, {}) + assert len(result) == 1 \ No newline at end of file diff --git a/tests/unit/test_sheet.py b/tests/unit/test_sheet.py new file mode 100644 index 0000000..8f441f2 --- /dev/null +++ b/tests/unit/test_sheet.py @@ -0,0 +1,63 @@ +''' +Unit tests for src/comms/utils/sheet.py +''' +from comms.utils.sheet import SampleRow, render_sample_sheet, parse_sample_sheet + + +class TestRenderSampleSheet: + def test_header_uses_canonical_column_order(self): + text = render_sample_sheet([]) + assert text.splitlines()[0] == 'sample_id\traw_file\ttreatment\tfraction\treplicate\tbatch' + + def test_row_renders_tab_joined(self): + row = SampleRow(sample_id='S1', raw_file='s1.RAW', treatment='MOCK', fraction='WCL', replicate=1, batch='A') + text = render_sample_sheet([row]) + assert 'S1\ts1.RAW\tMOCK\tWCL\t1\tA' in text + + def test_none_replicate_renders_empty(self): + row = SampleRow(sample_id='S1', raw_file='s1.RAW', replicate=None) + text = render_sample_sheet([row]) + assert '\t\t' in text.splitlines()[1] or text.splitlines()[1].endswith('\t') + + def test_trailing_newline(self): + row = SampleRow(sample_id='S1', raw_file='s1.RAW') + assert render_sample_sheet([row]).endswith('\n') + + def test_empty_rows_still_writes_header(self): + text = render_sample_sheet([]) + assert len(text.splitlines()) == 1 + + +class TestParseSampleSheet: + def test_round_trips_with_render(self): + row = SampleRow(sample_id='S1', raw_file='s1.RAW', treatment='MOCK', fraction='WCL', replicate=2, batch='A') + text = render_sample_sheet([row]) + parsed = parse_sample_sheet(text) + assert len(parsed) == 1 + assert parsed[0].sample_id == 'S1' + assert parsed[0].replicate == 2 + + def test_empty_text_returns_empty_list(self): + assert parse_sample_sheet('') == [] + + def test_numeric_replicate_sets_overridden_true(self): + text = 'sample_id\traw_file\ttreatment\tfraction\treplicate\tbatch\nS1\ts1.RAW\tMOCK\tWCL\t3\tA\n' + parsed = parse_sample_sheet(text) + assert parsed[0].replicate == 3 + assert parsed[0].replicate_overridden is True + + def test_empty_replicate_parses_to_none(self): + text = 'sample_id\traw_file\ttreatment\tfraction\treplicate\tbatch\nS1\ts1.RAW\tMOCK\tWCL\t\tA\n' + parsed = parse_sample_sheet(text) + assert parsed[0].replicate is None + assert parsed[0].replicate_overridden is False + + def test_whitespace_stripped_from_fields(self): + text = 'sample_id\traw_file\ttreatment\tfraction\treplicate\tbatch\n S1 \t s1.RAW \tMOCK\tWCL\t1\tA\n' + parsed = parse_sample_sheet(text) + assert parsed[0].sample_id == 'S1' + + def test_missing_batch_column_defaults_to_empty(self): + text = 'sample_id\traw_file\ttreatment\tfraction\treplicate\nS1\ts1.RAW\tMOCK\tWCL\t1\n' + parsed = parse_sample_sheet(text) + assert parsed[0].batch == '' \ No newline at end of file diff --git a/tests/unit/test_validate.py b/tests/unit/test_validate.py index 17ffee1..7afe2e2 100644 --- a/tests/unit/test_validate.py +++ b/tests/unit/test_validate.py @@ -9,6 +9,8 @@ # -- Import functions under test from comms.utils.validate import ( + probe_crux, + probe_trfp, validate, _parse_version, _find_all_crux, @@ -645,4 +647,27 @@ def test_bin_dir_reaches_check_trfp(self, tmp_path): patch('comms.utils.validate.shutil.which', return_value='/usr/bin/mono'): validate(check_trfp=True, bin_dir=exp_bin) - mock_find.assert_called_once_with(exp_bin) \ No newline at end of file + mock_find.assert_called_once_with(exp_bin) + +class TestProbeCrux: + def test_returns_path_when_found(self, tmp_path, monkeypatch): + crux_path = tmp_path / 'crux-4.1.linux' / 'bin' / 'crux' + crux_path.parent.mkdir(parents=True) + crux_path.touch() + monkeypatch.setattr('comms.utils.validate._get_crux_version', lambda p: (4, 1, 0)) + assert probe_crux(tmp_path) == crux_path + + def test_returns_none_without_raising_when_not_found(self, tmp_path): + assert probe_crux(tmp_path) is None + + +class TestProbeTrfp: + def test_returns_path_when_found(self, tmp_path, monkeypatch): + trfp_path = tmp_path / 'trfp-1.4' / 'ThermoRawFileParser' + trfp_path.parent.mkdir(parents=True) + trfp_path.touch() + monkeypatch.setattr('comms.utils.validate._get_trfp_version', lambda p: (1, 4, 0)) + assert probe_trfp(tmp_path) == trfp_path + + def test_returns_none_without_raising_when_not_found(self, tmp_path): + assert probe_trfp(tmp_path) is None \ No newline at end of file From 51aa1dd7436945186f97dd50615ad04a9bd9595f Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Tue, 11 Aug 2026 08:09:36 +0100 Subject: [PATCH 066/108] fix(rescore): fix bug in rescoring - Modified commands/rescore.py to capture the return value of the combined Percolator run. --- src/comms/commands/rescore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/commands/rescore.py b/src/comms/commands/rescore.py index ea6922d..ad280c0 100644 --- a/src/comms/commands/rescore.py +++ b/src/comms/commands/rescore.py @@ -71,7 +71,7 @@ def run_rescore( run_config = ctx.config # Round 1: run Percolator on the full combined database, with one call per sample file - _run_combined_percolator_round( + combined_target_files = _run_combined_percolator_round( crux_bin, target_files, database, From 5fb9d9dd6fbf92410202e2fbfa8e3db65e1e5cbf Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 10:05:03 +0100 Subject: [PATCH 067/108] test(conftest): fix COMMS_BIN_DIR wiring - Modified tests/conftest.py to add autoused fixture that points COMMS_BIN_DIR at the same directory as other fixtures. --- tests/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 9394ef7..5bb0bf0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,11 @@ # -- Define bin directory for tests requiring Crux/ThermoRawFileParser BIN_DIR = Path(__file__).parent / 'bin' +# -- Point comMS at the test bin/ directory for the whole session +@pytest.fixture(autouse=True) +def _comms_bin_dir_env(monkeypatch): + monkeypatch.setenv('COMMS_BIN_DIR', str(BIN_DIR)) + # -- Import internal dependencies from tests.fixtures.generate_fixtures import generate_all, write_fasta, write_mzml From 9ab7483e88b11824235a83f3398adc412998d373 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 10:07:37 +0100 Subject: [PATCH 068/108] test: fix crux integration tests - Modified tests/integration/test_crux.py to remove extraneous threads arguments, and to correct expected output filenames of spectral counts. --- tests/integration/test_crux.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_crux.py b/tests/integration/test_crux.py index 980ed29..78a3622 100644 --- a/tests/integration/test_crux.py +++ b/tests/integration/test_crux.py @@ -84,7 +84,6 @@ def test_creates_target_psm_file(self, crux_bin, built_index, synthetic_mzml, tm out_dir=out_dir, fileroot='synthetic', config=cfg, - threads=cfg['search']['threads'] ) assert ok, 'tideSearch returned False — check search.log' target_file = out_dir / 'synthetic.tide-search.target.txt' @@ -100,7 +99,6 @@ def test_target_file_has_header_and_data(self, crux_bin, built_index, synthetic_ out_dir=out_dir, fileroot='synthetic', config=cfg, - threads=cfg['search']['threads'] ) target_file = out_dir / 'synthetic.tide-search.target.txt' lines = target_file.read_text().splitlines() @@ -116,7 +114,6 @@ def test_log_file_is_written(self, crux_bin, built_index, synthetic_mzml, tmp_pa out_dir=out_dir, fileroot='synthetic', config=cfg, - threads=cfg['search']['threads'] ) log = out_dir / 'synthetic.tide-search.log.txt' assert log.exists() @@ -138,7 +135,6 @@ def search_results(crux_bin, built_index, tmp_path_factory): out_dir=out_dir, fileroot='synthetic', config=cfg, - threads=cfg['search']['threads'] ) if not ok: pytest.skip('tideSearch failed — cannot run percolator tests') @@ -218,7 +214,7 @@ def test_creates_spectral_counts_file(self, crux_bin, synthetic_percolator_resul config=cfg, ) assert ok, 'spectralCounts returned False — check quantify.log' - assert (out_dir / 'synthetic.spectral-counts.target.txt').exists() + assert (out_dir / 'synthetic_dNSAF.spectral-counts.target.txt').exists() def test_counts_file_has_content(self, crux_bin, synthetic_percolator_results, synthetic_fasta, tmp_path): psm_file = synthetic_percolator_results / 'EUK' / 'synthetic.EUK.percolator.target.psms.txt' @@ -232,4 +228,4 @@ def test_counts_file_has_content(self, crux_bin, synthetic_percolator_results, s fileroot='synthetic', config=cfg, ) - assert (out_dir / 'synthetic.spectral-counts.target.txt').stat().st_size > 0 \ No newline at end of file + assert (out_dir / 'synthetic_dNSAF.spectral-counts.target.txt').stat().st_size > 0 \ No newline at end of file From f93f46f1e235024d6957a94485f4a6d784f741d9 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:18:19 +0100 Subject: [PATCH 069/108] fix(config): use absolute not relative paths - Modified commands/config.py to use absolute paths for bare and nested instead of original relative paths. --- src/comms/commands/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/commands/config.py b/src/comms/commands/config.py index d82c446..4f6257b 100644 --- a/src/comms/commands/config.py +++ b/src/comms/commands/config.py @@ -79,7 +79,7 @@ def _resolve_or_create(path: Path | None, use_global: bool) -> Path: _, comms_dir = _normalise_dirs(path) target = comms_dir / 'config.toml' else: - bare, nested = Path('config.toml'), Path('comms') / 'config.toml' + bare, nested = Path.cwd() / 'config.toml', Path.cwd() / 'comms' / 'config.toml' if bare.exists() and nested.exists(): logMsg.error(f'Both {bare} and {nested} exist in the current directory. Remove one before running comms config here.') raise SystemExit(1) From 48bbe091bf965bdd0252075282d403383bb3e53f Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:26:26 +0100 Subject: [PATCH 070/108] refactor(report): update logging levels - Modified commands/report.py to update logging levels used for success/ skip/fail states. --- src/comms/commands/report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/comms/commands/report.py b/src/comms/commands/report.py index 5905775..8013829 100644 --- a/src/comms/commands/report.py +++ b/src/comms/commands/report.py @@ -70,11 +70,11 @@ def _log_organism_outcomes(section: str, organisms: dict[str, str], reasons: dic reason = reasons.get(org) suffix = f' ({reason})' if reason else '' if status == 'ok': - logMsg.progress(f'{section} — {org}: succeeded') + logMsg.info(f'{section} — {org}: succeeded') elif status == 'skipped': - logMsg.progress(f'{section} — {org}: skipped{suffix}') + logMsg.info(f'{section} — {org}: skipped{suffix}') else: - logMsg.progress(f'{section} — {org}: failed{suffix}') + logMsg.warn(f'{section} — {org}: failed{suffix}') # -- _run_r_section: returns boolean indicating if the R process itself exited cleanly def _run_r_section( From dca83c46a36f56da747685223139e2b3d03c4547 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:51:08 +0100 Subject: [PATCH 071/108] refactor(utils): update R installs logging level - Modified utils/installrdeps.py to use info log level rather than progress. --- src/comms/utils/installrdeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/utils/installrdeps.py b/src/comms/utils/installrdeps.py index dc909d6..fdbc50c 100644 --- a/src/comms/utils/installrdeps.py +++ b/src/comms/utils/installrdeps.py @@ -60,7 +60,7 @@ def install_r_dependencies(rscript: str = 'Rscript') -> bool: for line in process.stdout: line = line.rstrip() if line: - logMsg.progress(line) + logMsg.info(line) process.wait() if process.returncode != 0: logMsg.error('R dependency installation failed; see above output') From 2781b9664c272a133f6f6d53aec7cb9754835b88 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:52:07 +0100 Subject: [PATCH 072/108] fix(lfq): fix file name parsing - Modified commands/lfq.py to fix filenames not being parsed from raw filename in sample sheet. --- src/comms/commands/lfq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/comms/commands/lfq.py b/src/comms/commands/lfq.py index 31e4013..8b299da 100644 --- a/src/comms/commands/lfq.py +++ b/src/comms/commands/lfq.py @@ -79,4 +79,4 @@ def _groupPsmsByFraction(psm_files: list[Path], samples: pd.DataFrame) -> dict[s # -- _get_stem: return str corresponding to stem of file from raw_file in sample sheet def _get_stem(row): - return str(row['sample_id']) \ No newline at end of file + return Path(str(row['raw_file'])).stem \ No newline at end of file From 618e6c67df4d5c3b9b5474b31bcef406cfd7e3d0 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:55:00 +0100 Subject: [PATCH 073/108] fix(utils): update readiness checks - Modified utils/readiness.py to exclude convert from pipeline readiness check, and to only require organism tags if multispecies analysis running. --- src/comms/utils/readiness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/comms/utils/readiness.py b/src/comms/utils/readiness.py index ff1d1b9..9646612 100644 --- a/src/comms/utils/readiness.py +++ b/src/comms/utils/readiness.py @@ -22,10 +22,10 @@ def missing_requirements( 'convert': [('data files', has_data), ('ThermoRawFileParser', has_trfp)], 'index': [('database', has_database), ('Crux', has_crux)], 'search': [('data files', has_data), ('database', has_database), ('Crux', has_crux)], - 'rescore': [('database', has_database)] + [('organism patterns', has_organism_tags)] + [('Crux', has_crux)], + 'rescore': ([('database', has_database), ('Crux', has_crux)] + ([('organism patterns', has_organism_tags)] if multispecies else [])), 'lfq': [('sample sheet', has_sample_sheet), ('data files', has_data), ('Crux', has_crux)], 'quantify': [('database', has_database), ('Crux', has_crux)], 'report': [('sample sheet', has_sample_sheet), ('organism prefix', has_organism_prefix), ('R dependencies', has_r_deps)], } - base['pipeline'] = [item for items in base.values() for item in items] + base['pipeline'] = [item for cmd, items in base.items() if cmd != 'convert' for item in items] return {cmd: [name for name, ok in items if not ok] for cmd, items in base.items()} \ No newline at end of file From cbd76730a6d0a47fe3a68a8f0d9282a5b27b412a Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:56:57 +0100 Subject: [PATCH 074/108] test(conftest): change fixture scope to session - Modified test/conftest.py to change _comms_bin_dir_env fixture scope from function to session. --- tests/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5bb0bf0..b39ed96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,9 +17,12 @@ BIN_DIR = Path(__file__).parent / 'bin' # -- Point comMS at the test bin/ directory for the whole session -@pytest.fixture(autouse=True) -def _comms_bin_dir_env(monkeypatch): - monkeypatch.setenv('COMMS_BIN_DIR', str(BIN_DIR)) +@pytest.fixture(scope='session', autouse=True) +def _comms_bin_dir_env(): + mp = pytest.MonkeyPatch() + mp.setenv('COMMS_BIN_DIR', str(BIN_DIR)) + yield + mp.undo() # -- Import internal dependencies from tests.fixtures.generate_fixtures import generate_all, write_fasta, write_mzml From a9485ed56ea3d2387c24f3e565111354b941e900 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 12:58:23 +0100 Subject: [PATCH 075/108] test: update unit tests - Modified tests/unit/test_config.py to refactor unit tests to reflect current functionality. - Modified tests/unit/test_report.py to refactor unit tests to reflect current functionality. - Modified tests/unit/test_samples.py to refactor unit tests to reflect current functionality. - Modified tests/unit/test_settings.py to refactor unit tests to reflect current functionality. --- tests/unit/test_config.py | 924 ++++++++---------------------------- tests/unit/test_report.py | 143 +++++- tests/unit/test_samples.py | 80 ++-- tests/unit/test_settings.py | 88 ++-- 4 files changed, 428 insertions(+), 807 deletions(-) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 4f9ba25..6cd4673 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,759 +1,219 @@ ''' -Unit tests for src/comms/commands/config.py and src/comms/utils/settings.py +Unit tests for src/comms/commands/config.py ''' - -# -- Import external dependencies -import click, pytest, tomllib +import tomllib +import pytest from pathlib import Path +from unittest.mock import patch + +from comms.commands.config import ( + _confirm, _loadConfigFile, _flatten, _printTable, _print_diff_summary, + _resolve_or_create, config_list, config_verify, config_reset, config_set, +) +from comms.utils.settings import loadDefaultConfig, globalConfigPath +from comms.utils.modspec import CARBAMIDOMETHYL_MOD, MET_OX_MOD -# Import internal constants -from comms.commands.config import CARBAMIDOMETHYL_MOD, MET_OX_MOD, PHOSPHO_MOD, NCYC_MOD, NACE_MOD, MANAGED_MOD_PATTERNS, MZ_BIN_WIDTH_HIGH_RES, MZ_BIN_WIDTH_LOW_RES, SCORE_FUNC_HIGH_RES, SCORE_FUNC_LOW_RES - -# -- Import internal functions -import comms.utils.settings as settings -from comms.commands.config import _apply_custom_mod, _apply_iodo, _apply_mod, _apply_organism, _apply_protocol_flags, _flatten, _configCheck, _loadConfigFile, _parse_organism_arg, _resolveConfigTarget, _writeConfig, config_init, config_exists, config_list, config_verify, config_reset, config_set - -# -- Define fixture for initialised user config file -@pytest.fixture() -def initialised_config(isolated_config_dir): - ''' - Builds on isolated_config_dir: also calls config_init() so a valid user - config file exists before the test runs. - ''' - config_init() - return isolated_config_dir - -# -- Define tests for resolving targeted config file -class TestResolveConfigTarget: - def test_none_is_global(self): - from comms.utils.settings import globalConfigPath - assert _resolveConfigTarget(None) == globalConfigPath() - - def test_global_keyword_case_insensitive(self): - from comms.utils.settings import globalConfigPath - assert _resolveConfigTarget('GLOBAL') == globalConfigPath() - assert _resolveConfigTarget('global') == globalConfigPath() - - def test_path_is_returned_verbatim(self, tmp_path): - target = tmp_path / 'comms' / 'config.toml' - assert _resolveConfigTarget(str(target)) == target -# -- Define tests for setting local target config file -class TestConfigSetLocalTarget: - def test_set_writes_to_local_path(self, tmp_path): - local = tmp_path / 'comms' / 'config.toml' - config_set(iodo=True, config_path=local) - assert local.exists() - import tomllib - with local.open('rb') as f: - cfg = tomllib.load(f) - assert 'index' in cfg - -# -- Define tests for default config file structure -class TestLoadDefaultConfig: - def test_returns_dict(self): - cfg = settings.loadDefaultConfig() - assert isinstance(cfg, dict) - - def test_contains_expected_top_level_sections(self): - cfg = settings.loadDefaultConfig() - for section in ('global', 'convert', 'search', 'percolator', 'quantify'): - assert section in cfg, f'Missing top-level section: {section}' - - def test_search_section_has_required_keys(self): - search = settings.loadDefaultConfig()['search'] - for key in ('threads', 'precursor_tolerance_ppm', 'mz_bin_width', 'score_function'): - assert key in search, f'Missing search key: {key}' - - def test_no_fragment_tolerance_da_key(self): - '''fragment_tolerance_da has been replaced by mz_bin_width.''' - assert 'fragment_tolerance_da' not in settings.loadDefaultConfig().get('search', {}) - - def test_values_have_correct_types(self): - cfg = settings.loadDefaultConfig() - assert isinstance(cfg['search']['threads'], int) - assert isinstance(cfg['search']['precursor_tolerance_ppm'], float) - assert isinstance(cfg['search']['mz_bin_width'], float) - assert isinstance(cfg['convert']['gzip'], bool) - - def test_default_score_function_is_xcorr(self): - assert settings.loadDefaultConfig()['search']['score_function'] == SCORE_FUNC_HIGH_RES - - def test_default_mz_bin_width_is_high_res(self): - assert settings.loadDefaultConfig()['search']['mz_bin_width'] == MZ_BIN_WIDTH_HIGH_RES - - def test_default_fixed_mods_does_not_contain_carbamidomethyl(self): - '''Check carbamidomethylation is not in default config''' - mods = settings.loadDefaultConfig()['index']['fixed_mods'] - assert CARBAMIDOMETHYL_MOD not in mods - -# -- Define tests for _flatten utility function class TestFlatten: def test_flat_dict_unchanged(self): - d = {'a': 1, 'b': 2} - assert _flatten(d) == {'a': 1, 'b': 2} + assert _flatten({'a': 1}) == {'a': 1} def test_nested_dict_flattened(self): - d = {'outer': {'inner': 42}} - assert _flatten(d) == {'outer.inner': 42} + assert _flatten({'outer': {'inner': 42}}) == {'outer.inner': 42} def test_deeply_nested(self): - d = {'a': {'b': {'c': 'x'}}} - assert _flatten(d) == {'a.b.c': 'x'} - - def test_mixed_depth(self): - d = {'top': 1, 'nested': {'key': 2}} - result = _flatten(d) - assert result['top'] == 1 - assert result['nested.key'] == 2 + assert _flatten({'a': {'b': {'c': 'x'}}}) == {'a.b.c': 'x'} def test_empty_dict(self): assert _flatten({}) == {} def test_default_config_flattens_without_error(self): - cfg = settings.loadDefaultConfig() - flat = _flatten(cfg) - assert all('.' in k for k in flat if any( - cfg.get(k.split('.')[0], None) and isinstance(cfg[k.split('.')[0]], dict) - for _ in [None] - )) - assert isinstance(flat, dict) - assert len(flat) > 0 - -# -- Define tests for checking configuration file exists -class TestConfigCheck: - def test_exists_true_when_file_present(self, tmp_path): - p = tmp_path / 'config.toml' - p.write_text('[global]\nverbose = false\n') - assert _configCheck(p, exists=True) is True - - def test_exists_true_fails_when_file_absent(self, tmp_path): - p = tmp_path / 'config.toml' - assert _configCheck(p, exists=True) is False - - def test_exists_false_passes_when_file_absent(self, tmp_path): - p = tmp_path / 'config.toml' - assert _configCheck(p, exists=False) is True - - def test_exists_false_fails_when_file_present(self, tmp_path): - p = tmp_path / 'config.toml' - p.write_text('[global]\nverbose = false\n') - assert _configCheck(p, exists=False) is False - -# -- Define tests for writing and loading configuration files -class TestWriteLoadConfig: - def test_write_and_reload_preserves_content(self, isolated_config_dir): - defaults = settings.loadDefaultConfig() - _writeConfig(defaults) - loaded = _loadConfigFile() - assert loaded == defaults - - def test_load_raises_when_no_config(self, isolated_config_dir): - with pytest.raises(FileNotFoundError): - _loadConfigFile() - -# -- Define tests for config init subcommand -class TestConfigInit: - def test_creates_config_file(self, isolated_config_dir): - config_init() - assert settings.globalConfigPath().exists() - - def test_created_file_is_valid_toml(self, isolated_config_dir): - config_init() - with settings.globalConfigPath().open('rb') as f: - result = tomllib.load(f) - assert isinstance(result, dict) - - def test_does_not_overwrite_existing(self, initialised_config): - settings.globalConfigPath().write_text('[global]\nverbose = true\n') - with pytest.raises(SystemExit) as exc: - config_init() - assert exc.value.code != 0 - -# -- Define tests for config exists subcommand -class TestConfigExists: - def test_exits_nonzero_when_absent(self, isolated_config_dir): - with pytest.raises(SystemExit) as exc: - config_exists() - assert exc.value.code != 0 - - def test_does_not_raise_when_present(self, initialised_config): - config_exists() - -# -- Define tests for config verify subcommand -class TestConfigVerify: - def test_valid_config_passes(self, initialised_config): - config_verify() - - def test_missing_key_exits_nonzero(self, initialised_config): - # Remove one required key by writing a truncated config - settings.globalConfigPath().write_text('[global]\nverbose = false\n') - with pytest.raises(SystemExit) as exc: - config_verify() - assert exc.value.code != 0 - - def test_exits_nonzero_when_no_config(self, isolated_config_dir): - with pytest.raises(SystemExit) as exc: - config_verify() - assert exc.value.code != 0 - -# -- Define tests for config reset subcommand -class TestConfigReset: - def test_reset_with_force_restores_defaults(self, initialised_config): - # Corrupt the config - settings.globalConfigPath().write_text('[global]\nverbose = true\n') - config_reset(force=True) - assert _loadConfigFile() == settings.loadDefaultConfig() - - def test_reset_without_force_prompts(self, initialised_config, monkeypatch): - # Simulate user declining the prompt - monkeypatch.setattr('typer.confirm', lambda *a, **kw: False) - with pytest.raises(SystemExit) as exc: - config_reset(force=False) - assert exc.value.code in (0, None) - - def test_reset_without_force_proceeds_on_confirm(self, isolated_config_dir, monkeypatch): - settings.globalConfigPath().write_text('[global]\nverbose = true\n') - monkeypatch.setattr('typer.confirm', lambda *a, **kw: True) - config_reset(force=False) - assert _loadConfigFile() == settings.loadDefaultConfig() - -# -- Define tests for _apply_custom_mod_mod helper function -class TestApplyCustom: - def test_adds_custom_entry_to_empty_string(self): - result = _apply_custom_mod('', '1K+28.0313') - assert '1K+28.0313' in result - - def test_adds_custom_entry_to_existing_string(self): - result = _apply_custom_mod('1K+28.0313', '1R+14.0157') - assert '1K+28.0313' in result - assert '1R+14.0157' in result - - def test_empty_string_clears_all_custom_mods(self): - result = _apply_custom_mod('1K+28.0313,1R+14.0157', '') - assert result == '' - - def test_duplicate_entry_not_added(self): - result = _apply_custom_mod('1K+28.0313', '1K+28.0313') - assert result.count('1K+28.0313') == 1 - - def test_managed_cys_mod_rejected_with_warning(self, capsys): - result = _apply_custom_mod('', '1C+57.0215') - assert '1C+57.0215' not in result - - def test_managed_met_mod_rejected_with_warning(self, capsys): - result = _apply_custom_mod('', '1M+15.9949') - assert '1M+15.9949' not in result - - def test_managed_phos_mod_rejected_with_warning(self, capsys): - result = _apply_custom_mod('', '1STY+79.966331') - assert '1STY+79.966331' not in result - - def test_unmanaged_entry_accepted(self): - result = _apply_custom_mod('', '1K+28.0313') - assert result == '1K+28.0313' - - def test_no_leading_or_trailing_commas(self): - result = _apply_custom_mod('', '1K+28.0313') - assert not result.startswith(',') - assert not result.endswith(',') - -# -- Define tests for _apply_mod helper function -class TestApplyMod: - def test_adds_mod_to_empty_spec(self): - assert MET_OX_MOD in _apply_mod('', mod=MET_OX_MOD) - def test_adds_mod_to_existing_spec(self): - result = _apply_mod('1Q-17.027', mod=MET_OX_MOD) - assert MET_OX_MOD in result - assert '1Q-17.027' in result - def test_prepends_mod(self): - result = _apply_mod('1Q-17.027', mod=MET_OX_MOD) - assert result.startswith(MET_OX_MOD) - - def test_adding_same_mod_twice_does_not_duplicate(self): - result = _apply_mod(MET_OX_MOD, mod=MET_OX_MOD) - assert result.count(MET_OX_MOD) == 1 - - def test_removal_with_exclusive_pattern(self): - spec = f'{MET_OX_MOD},{PHOSPHO_MOD}' - result = _apply_mod(spec, mod='', exclusive_pattern=r'^\d*M\+15\.9949') - assert MET_OX_MOD not in result - assert PHOSPHO_MOD in result - - def test_exclusive_pattern_replaces_on_add(self): - spec = f'1M+15.9949,{PHOSPHO_MOD}' - result = _apply_mod(spec, mod='2M+15.9949', exclusive_pattern=r'^\d*M\+15\.9949') - assert '2M+15.9949' in result - assert '1M+15.9949' not in result - assert PHOSPHO_MOD in result - - def test_no_leading_or_trailing_commas(self): - result = _apply_mod('', mod=MET_OX_MOD) - assert not result.startswith(',') - assert not result.endswith(',') - - def test_no_double_commas(self): - result = _apply_mod('1Q-17.027', mod=MET_OX_MOD) - assert ',,' not in result - - def test_empty_mod_with_no_pattern(self): - spec = '1M+15.9949' - assert _apply_mod(spec, mod='') == spec - - def test_removal_of_absent_mod(self): - spec = '1M+15.9949' - result = _apply_mod(spec, mod='', exclusive_pattern=r'^\d*STY\+79\.966331') - assert result == spec - -# -- Define tests for _apply_organism helper function -class TestApplyOrganism: - def test_sets_organism_section(self): - cfg = {'organism': {}, 'search': {'threads': 2}} - result = _apply_organism(cfg, {'Test1': 'TEST1', 'Test2': 'TEST2'}) - assert result['organism'] == {'Test1': 'TEST1', 'Test2': 'TEST2'} - - def test_replaces_existing_organism_section(self): - cfg = {'organism': {'OldTest', 'OLDTEST'}} - result = _apply_organism(cfg, {'NewTest': 'NEWTEST'}) - assert 'OldTest' not in result['organism'] - assert result['organism'] == {'NewTest': 'NEWTEST'} - - def test_does_not_touch_other_sections(self): - cfg = {'organism': {}, 'search': {'threads': 2}, 'percolator': {'psm_fdr': 0.01}} - _apply_organism(cfg, {'Test': 'TEST'}) - assert cfg['search']['threads'] == 2 - assert cfg['percolator']['psm_fdr'] == 0.01 - - def test_empty_dict_clears_organism_section(self): - cfg = {'organism': {'Test', 'TEST'}} - result = _apply_organism(cfg, {}) - assert result['organism'] == {} - - def test_returns_cfg(self): - cfg = {'organism': {}} - result = _apply_organism(cfg, {'Test': 'TEST'}) - assert result is cfg - -# -- Define tests for _apply_iodo helper function -class TestApplyIodo: - def test_iodo_adds_carbamidomethyl_to_empty_fixed_mods(self): - result = _apply_iodo('', iodo=True) - assert CARBAMIDOMETHYL_MOD in result - - def test_iodo_adds_carbamidomethyl_to_existing_fixed_mods(self): - result = _apply_iodo('someother_mod', iodo=True) - assert CARBAMIDOMETHYL_MOD in result - assert 'someother_mod' in result - - def test_iodo_prepends_carbamidomethyl(self): - result = _apply_iodo('someother_mod', iodo=True) - assert result.startswith(CARBAMIDOMETHYL_MOD) - - def test_no_iodo_removes_carbamidomethyl(self): - spec = f'{CARBAMIDOMETHYL_MOD},someother_mod' - result = _apply_iodo(spec, iodo=False) - assert CARBAMIDOMETHYL_MOD not in result - assert 'someother_mod' in result - - def test_no_iodo_on_empty_fixed_mods_returns_empty(self): - assert _apply_iodo('', iodo=False) == 'C+0' - - def test_no_iodo_on_spec_without_carbamidomethyl_is_noop(self): - spec = 'someother_mod' - assert _apply_iodo(spec, iodo=False) == 'C+0,'+spec - - def test_iodo_is_idempotent(self): - result = _apply_iodo(_apply_iodo('', iodo=True), iodo=True) - assert result.count(CARBAMIDOMETHYL_MOD) == 1 - - def test_result_has_no_leading_or_trailing_commas(self): - assert not _apply_iodo('', iodo=True).startswith(',') - assert not _apply_iodo('', iodo=True).endswith(',') - - def test_result_has_no_double_commas(self): - assert ',,' not in _apply_iodo('someother_mod', iodo=True) - - def test_mod_string_has_no_count_prefix(self): - result = _apply_iodo('', iodo=True) - assert result == CARBAMIDOMETHYL_MOD - -# -- Define tests for _apply_protocol_flags helper function -class TestApplyProtocolFlags: - def _base_cfg(self): - return settings.loadDefaultConfig() - - def test_iodo_none_does_not_touch_mods_spec(self): - cfg = self._base_cfg() - original = cfg['index']['mods_spec'] - result = _apply_protocol_flags(cfg) - assert result['index']['mods_spec'] == original - - def test_low_res_none_does_not_touch_search(self): - cfg = self._base_cfg() - original_bw = cfg['search']['mz_bin_width'] - original_sf = cfg['search']['score_function'] - result = _apply_protocol_flags(cfg) - assert result['search']['mz_bin_width'] == original_bw - assert result['search']['score_function'] == original_sf - - def test_low_res_true_sets_bin_width_and_score(self): - cfg = _apply_protocol_flags(self._base_cfg(), low_res=True) - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_LOW_RES - assert cfg['search']['score_function'] == SCORE_FUNC_LOW_RES - - def test_low_res_false_sets_high_res_bin_width_and_score(self): - cfg = _apply_protocol_flags(self._base_cfg(), low_res=False) - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_HIGH_RES - assert cfg['search']['score_function'] == SCORE_FUNC_HIGH_RES - - def test_combined_iodo_and_low_res(self): - cfg = _apply_protocol_flags(self._base_cfg(), iodo=True, low_res=True) - assert CARBAMIDOMETHYL_MOD in cfg['index']['fixed_mods'] - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_LOW_RES - assert cfg['search']['score_function'] == SCORE_FUNC_LOW_RES + flat = _flatten(loadDefaultConfig()) + assert isinstance(flat, dict) and len(flat) > 0 + + +class TestPrintTable: + def test_renders_without_raising_when_matching_defaults(self): + defaults = loadDefaultConfig() + _printTable(_flatten(defaults), _flatten(defaults)) + + def test_renders_without_raising_with_divergence(self): + defaults = loadDefaultConfig() + current = dict(defaults) + current['search'] = dict(current['search']) + current['search']['threads'] = 999 + _printTable(_flatten(current), _flatten(defaults)) + + +class TestPrintDiffSummary: + def test_no_changes_prints_message(self, capsys): + _print_diff_summary({'a': 1}, {'a': 1}) + assert 'No changes' in capsys.readouterr().out + + def test_changed_key_shows_old_and_new(self, capsys): + _print_diff_summary({'search.threads': 4}, {'search.threads': 8}) + out = capsys.readouterr().out + assert '4' in out and '8' in out + + def test_multiple_changes_all_reported(self, capsys): + _print_diff_summary({'a': 1, 'b': 2}, {'a': 9, 'b': 9}) + out = capsys.readouterr().out + assert out.count('✓') == 2 + + +class TestResolveOrCreate: + def test_global_returns_global_path(self, isolated_config_dir): + assert _resolve_or_create(None, use_global=True) == isolated_config_dir / 'config.toml' + + def test_explicit_path_resolves_to_comms_config(self, tmp_path): + target = _resolve_or_create(tmp_path, use_global=False) + assert target == tmp_path / 'comms' / 'config.toml' + + def test_creates_file_from_defaults_if_absent(self, tmp_path): + target = _resolve_or_create(tmp_path, use_global=False) + assert target.exists() + with target.open('rb') as f: + assert tomllib.load(f) == loadDefaultConfig() + + def test_bare_and_nested_config_both_present_raises(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / 'config.toml').write_text('[search]\n') + (tmp_path / 'comms').mkdir() + (tmp_path / 'comms' / 'config.toml').write_text('[search]\n') + with pytest.raises(SystemExit): + _resolve_or_create(None, use_global=False) - def test_combined_iodo_and_high_res(self): - cfg = _apply_protocol_flags(self._base_cfg(), iodo=True, low_res=False) - assert CARBAMIDOMETHYL_MOD in cfg['index']['fixed_mods'] - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_HIGH_RES - assert cfg['search']['score_function'] == SCORE_FUNC_HIGH_RES - - def test_only_relevant_keys_touched_by_low_res(self): - cfg_before = self._base_cfg() - cfg_after = _apply_protocol_flags(self._base_cfg(), low_res=True) - for key in ('threads', 'precursor_tolerance_ppm', 'min_peaks'): - assert cfg_after['search'][key] == cfg_before['search'][key], (f'_apply_protocol_flags unexpectedly changed search.{key}') - - def test_non_search_sections_untouched(self): - cfg_before = self._base_cfg() - cfg_after = _apply_protocol_flags(self._base_cfg(), iodo=True, low_res=True) - for section in ('global', 'convert', 'percolator', 'quantify'): - assert cfg_after.get(section) == cfg_before.get(section), (f'_apply_protocol_flags unexpectedly changed section [{section}]') - -class TestApplyProtocolFlagsMods: - def _base_cfg(self): - return settings.loadDefaultConfig() - - def test_iodo_true_adds_to_fixed_mods(self): - cfg = _apply_protocol_flags(self._base_cfg(), iodo=True) - assert CARBAMIDOMETHYL_MOD in cfg['index']['fixed_mods'] + def test_prefers_bare_config_when_only_bare_present(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / 'config.toml').write_text('[search]\n') + target = _resolve_or_create(None, use_global=False) + assert target == tmp_path / 'config.toml' + + def test_prompts_and_creates_nested_when_neither_present(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with patch('comms.commands.config._confirm', return_value=True): + target = _resolve_or_create(None, use_global=False) + assert target == tmp_path / 'comms' / 'config.toml' + + def test_declining_prompt_exits_zero(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with patch('comms.commands.config._confirm', return_value=False): + with pytest.raises(SystemExit) as exc: + _resolve_or_create(None, use_global=False) + assert exc.value.code == 0 - def test_iodo_false_removes_from_fixed_mods(self): - cfg = _apply_protocol_flags(self._base_cfg(), iodo=True) - cfg = _apply_protocol_flags(cfg, iodo=False) - assert CARBAMIDOMETHYL_MOD not in cfg['index']['fixed_mods'] - - def test_iodo_does_not_touch_mods_spec(self): - cfg = self._base_cfg() - original_mods = cfg['index']['mods_spec'] - cfg = _apply_protocol_flags(cfg, iodo=True) - assert cfg['index']['mods_spec'] == original_mods - - def test_ox_true_adds_met_mod(self): - cfg = _apply_protocol_flags(self._base_cfg(), ox=True) - assert MET_OX_MOD in cfg['index']['mods_spec'] - - def test_ox_false_removes_met_mod(self): - cfg = _apply_protocol_flags(self._base_cfg(), ox=True) - cfg = _apply_protocol_flags(cfg, ox=False) - assert MET_OX_MOD not in cfg['index']['mods_spec'] - - def test_ox_none_does_not_touch_mods_spec(self): - cfg = self._base_cfg() - original = cfg['index']['mods_spec'] - cfg = _apply_protocol_flags(cfg) - assert cfg['index']['mods_spec'] == original - - def test_phos_true_adds_phos_mod(self): - cfg = _apply_protocol_flags(self._base_cfg(), phos=True) - assert PHOSPHO_MOD in cfg['index']['mods_spec'] - - def test_phos_false_removes_phos_mod(self): - cfg = _apply_protocol_flags(self._base_cfg(), phos=True) - cfg = _apply_protocol_flags(cfg, phos=False) - assert PHOSPHO_MOD not in cfg['index']['mods_spec'] - - def test_n_cyc_true_adds_to_nterm_peptide_key(self): - cfg = _apply_protocol_flags(self._base_cfg(), n_cyc=True) - assert NCYC_MOD in cfg['index']['nterm_peptide_mods_spec'] - - def test_n_cyc_false_removes_from_nterm_peptide_key(self): - cfg = _apply_protocol_flags(self._base_cfg(), n_cyc=True) - cfg = _apply_protocol_flags(cfg, n_cyc=False) - assert NCYC_MOD not in cfg['index']['nterm_peptide_mods_spec'] - - def test_n_ace_true_adds_to_nterm_protein_key(self): - cfg = _apply_protocol_flags(self._base_cfg(), n_ace=True) - assert NACE_MOD in cfg['index']['nterm_protein_mods_spec'] - - def test_n_ace_false_removes_from_nterm_protein_key(self): - cfg = _apply_protocol_flags(self._base_cfg(), n_ace=True) - cfg = _apply_protocol_flags(cfg, n_ace=False) - assert NACE_MOD not in cfg['index']['nterm_protein_mods_spec'] - - def test_n_cyc_does_not_touch_mods_spec(self): - cfg = self._base_cfg() - original_mods = cfg['index']['mods_spec'] - cfg = _apply_protocol_flags(cfg, n_cyc=True) - assert cfg['index']['mods_spec'] == original_mods - - def test_n_ace_does_not_touch_mods_spec(self): - cfg = self._base_cfg() - original_mods = cfg['index']['mods_spec'] - cfg = _apply_protocol_flags(cfg, n_ace=True) - assert cfg['index']['mods_spec'] == original_mods - - def test_ox_and_iodo_coexist(self): - cfg = _apply_protocol_flags(self._base_cfg(), iodo=True, ox=True) - assert CARBAMIDOMETHYL_MOD in cfg['index']['fixed_mods'] - assert MET_OX_MOD in cfg['index']['mods_spec'] - - def test_iodo_does_not_remove_ox(self): - cfg = _apply_protocol_flags(self._base_cfg(), ox=True) - cfg = _apply_protocol_flags(cfg, iodo=True) - assert MET_OX_MOD in cfg['index']['mods_spec'] - - def test_all_flags_none_changes_nothing(self): - cfg_before = self._base_cfg() - cfg_after = _apply_protocol_flags(self._base_cfg()) - assert cfg_after['index']['mods_spec'] == cfg_before['index']['mods_spec'] - -# -- Define tests for _parse_organism_arg helper function -class TestParseOrganismArg: - def test_parses_single_pair(self): - result = _parse_organism_arg(['TestOrg=TEST']) - assert result == {'TestOrg': 'TEST'} - - def test_parses_multiple_pairs(self): - result = _parse_organism_arg(['TestOrg1=TEST1', 'TestOrg2=TEST2']) - assert result == {'TestOrg1': 'TEST1', 'TestOrg2': 'TEST2'} - - def test_strips_whitespace_from_key_and_pattern(self): - result = _parse_organism_arg(['Test = TEST']) - assert result == {'Test': 'TEST'} - - def test_preserves_regex_characters_in_pattern(self): - result = _parse_organism_arg(['Test=TEST$']) - assert result['Test'] == 'TEST$' - - def test_raises_system_exit_when_no_equals_sign(self): - with pytest.raises(SystemExit): - _parse_organism_arg(['TestTEST']) - def test_raises_system_exit_when_key_is_empty(self): +class TestConfigList: + def test_prints_config_path_and_table(self, tmp_path, capsys): + config_list(tmp_path, False) + out = capsys.readouterr().out + assert 'Current config' in out + + +class TestConfigVerify: + def test_valid_config_passes(self, tmp_path): + config_verify(tmp_path, False) # created from defaults, so valid + + def test_missing_key_exits_nonzero(self, tmp_path): + target = _resolve_or_create(tmp_path, use_global=False) + with target.open('rb') as f: + cfg = tomllib.load(f) + del cfg['search']['threads'] + import tomli_w + with target.open('wb') as f: + tomli_w.dump(cfg, f) with pytest.raises(SystemExit): - _parse_organism_arg(['=TEST']) + config_verify(tmp_path, False) - def test_raises_system_exit_when_pattern_is_empty(self): + def test_unexpected_key_exits_nonzero(self, tmp_path): + target = _resolve_or_create(tmp_path, use_global=False) + with target.open('rb') as f: + cfg = tomllib.load(f) + cfg['search']['not_a_real_key'] = 1 + import tomli_w + with target.open('wb') as f: + tomli_w.dump(cfg, f) with pytest.raises(SystemExit): - _parse_organism_arg(['Test=']) + config_verify(tmp_path, False) + - def test_returns_dict(self): - assert isinstance(_parse_organism_arg(['Test=TEST']), dict) +class TestConfigReset: + def test_force_resets_without_prompting(self, tmp_path): + target = _resolve_or_create(tmp_path, use_global=False) + with target.open('rb') as f: + cfg = tomllib.load(f) + cfg['search']['threads'] = 999 + import tomli_w + with target.open('wb') as f: + tomli_w.dump(cfg, f) + config_reset(tmp_path, False, force=True) + with target.open('rb') as f: + assert tomllib.load(f) == loadDefaultConfig() - def test_empty_list_returns_empty_dict(self): - result = _parse_organism_arg([]) - assert result == {} + def test_without_force_prompts_and_only_resets_on_accept(self, tmp_path): + with patch('comms.commands.config._confirm', return_value=False): + with pytest.raises(SystemExit) as exc: + config_reset(tmp_path, False, force=False) + assert exc.value.code == 0 - def test_pattern_containing_equals_sign_is_preserved(self): - result = _parse_organism_arg(['Test=TE=ST']) - assert result == {'Test': 'TE=ST'} -# Define tests for config set subcommand class TestConfigSet: - def test_creates_config_if_absent(self, isolated_config_dir): - '''config_set should auto-create the user config if none exists''' - config_set(iodo=True) - assert settings.globalConfigPath().exists() - - def test_created_config_is_valid_toml(self, isolated_config_dir): - config_set(iodo=True) - with settings.globalConfigPath().open('rb') as f: - result = tomllib.load(f) - assert isinstance(result, dict) - - def test_low_res_sets_bin_width_and_score(self, initialised_config): - config_set(low_res=True) - cfg = _loadConfigFile() - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_LOW_RES - assert cfg['search']['score_function'] == SCORE_FUNC_LOW_RES - - def test_high_res_sets_bin_width_and_score(self, initialised_config): - config_set(low_res=False) - cfg = _loadConfigFile() - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_HIGH_RES - assert cfg['search']['score_function'] == SCORE_FUNC_HIGH_RES - - def test_low_res_is_idempotent(self, initialised_config): - config_set(low_res=True) - config_set(low_res=True) - cfg = _loadConfigFile() - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_LOW_RES - assert cfg['search']['score_function'] == SCORE_FUNC_LOW_RES - - def test_sets_organism_in_config(self, initialised_config): - config_set(organism=['Test1=TEST1', 'Test2=TEST2']) - cfg = _loadConfigFile() - assert cfg['organism'] == {'Test1': 'TEST1', 'Test2': 'TEST2'} - - def test_set_organism_replaces_existing(self, initialised_config): - config_set(organism=['Test1=TEST1', 'Test2=TEST2']) - config_set(organism=['Test1=NEWTEST1']) - cfg = _loadConfigFile() - assert cfg['organism'] == {'Test1': 'NEWTEST1'} - assert 'Test2' not in cfg['organism'] - - def test_organism_section_written_as_toml_table(self, initialised_config): - config_set(organism=['Test1=TEST1', 'Test2=TEST2']) - cfg = _loadConfigFile() - assert isinstance(cfg['organism'], dict) - assert isinstance(cfg['organism']['Test1'], str) - - def test_combined_organism_and_low_res(self, initialised_config): - config_set(low_res=True, organism=['Test1=TEST1']) - cfg = _loadConfigFile() - assert cfg['organism'] == {'Test1': 'TEST1'} - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_LOW_RES - - def test_combined_organism_and_high_res(self, initialised_config): - config_set(low_res=False, organism=['Test1=TEST1']) - cfg = _loadConfigFile() - assert cfg['organism'] == {'Test1': 'TEST1'} - assert cfg['search']['mz_bin_width'] == MZ_BIN_WIDTH_HIGH_RES - - def test_no_flags_exits_nonzero(self, isolated_config_dir): - with pytest.raises(SystemExit) as exc: - config_set() - assert exc.value.code!= 0 - - def test_all_other_config_keys_unchanged_after_set(self, initialised_config): - before = _loadConfigFile() - config_set(low_res=True) - after = _loadConfigFile() - for section, values in before.items(): - if section == 'search': - for key, val in values.items(): - if key not in ('mz_bin_width', 'score_function'): - assert after[section][key] == val, ( - f'config_set unexpectedly changed {section}.{key}' - ) - else: - assert after[section] == values, ( - f'config_set unexpectedly changed section [{section}]' - ) - - def test_other_sections_unchanged_after_organism_set(self, initialised_config): - before = _loadConfigFile() - config_set(organism=['Mt=MEDTR']) - after = _loadConfigFile() - for section in ('search', 'percolator', 'quantify', 'convert', 'global'): - assert after.get(section) == before.get(section), ( - f'config_set --organism unexpectedly changed section [{section}]' - ) - - def test_iodo_adds_carbamidomethyl_to_fixed_mods(self, initialised_config): - config_set(iodo=True) - assert CARBAMIDOMETHYL_MOD in _loadConfigFile()['index']['fixed_mods'] - - def test_no_iodo_removes_carbamidomethyl_from_fixed_mods(self, initialised_config): - config_set(iodo=True) - config_set(iodo=False) - assert CARBAMIDOMETHYL_MOD not in _loadConfigFile()['index']['fixed_mods'] - - def test_iodo_does_not_add_to_mods_spec(self, initialised_config): - before_mods = _loadConfigFile()['index']['mods_spec'] - config_set(iodo=True) - assert _loadConfigFile()['index']['mods_spec'] == before_mods - - def test_iodo_is_idempotent(self, initialised_config): - config_set(iodo=True) - config_set(iodo=True) - assert _loadConfigFile()['index']['fixed_mods'].count(CARBAMIDOMETHYL_MOD) == 1 - - def test_ox_adds_met_mod(self, initialised_config): - config_set(ox=True) - assert MET_OX_MOD in _loadConfigFile()['index']['mods_spec'] - - def test_no_ox_removes_met_mod(self, initialised_config): - config_set(ox=True) - config_set(ox=False) - assert MET_OX_MOD not in _loadConfigFile()['index']['mods_spec'] - - def test_ox_is_idempotent(self, initialised_config): - config_set(ox=True) - config_set(ox=True) - assert _loadConfigFile()['index']['mods_spec'].count(MET_OX_MOD) == 1 - - def test_phos_adds_phos_mod(self, initialised_config): - config_set(phos=True) - assert PHOSPHO_MOD in _loadConfigFile()['index']['mods_spec'] - - def test_no_phos_removes_phos_mod(self, initialised_config): - config_set(phos=True) - config_set(phos=False) - assert PHOSPHO_MOD not in _loadConfigFile()['index']['mods_spec'] - - def test_n_cyc_adds_to_nterm_peptide_spec(self, initialised_config): - config_set(n_cyc=True) - assert NCYC_MOD in _loadConfigFile()['index']['nterm_peptide_mods_spec'] - - def test_no_n_cyc_removes_from_nterm_peptide_spec(self, initialised_config): - config_set(n_cyc=True) - config_set(n_cyc=False) - assert NCYC_MOD not in _loadConfigFile()['index']['nterm_peptide_mods_spec'] - - def test_n_ace_adds_to_nterm_protein_spec(self, initialised_config): - config_set(n_ace=True) - assert NACE_MOD in _loadConfigFile()['index']['nterm_protein_mods_spec'] - - def test_no_n_ace_removes_from_nterm_protein_spec(self, initialised_config): - config_set(n_ace=True) - config_set(n_ace=False) - assert NACE_MOD not in _loadConfigFile()['index']['nterm_protein_mods_spec'] - - def test_custom_adds_entry(self, initialised_config): - config_set(custom='1K+28.0313') - assert '1K+28.0313' in _loadConfigFile()['index']['custom_mods'] - - def test_custom_is_additive(self, initialised_config): - config_set(custom='1K+28.0313') - config_set(custom='1R+14.0157') - mods = _loadConfigFile()['index']['custom_mods'] - assert '1K+28.0313' in mods - assert '1R+14.0157' in mods - - def test_custom_empty_string_clears_all(self, initialised_config): - config_set(custom='1K+28.0313') - config_set(custom='') - assert _loadConfigFile()['index']['custom_mods'] == '' - - def test_custom_managed_mod_not_added(self, initialised_config): - config_set(custom='1M+15.9949') - assert '1M+15.9949' not in _loadConfigFile()['index']['custom_mods'] - - def test_custom_reaches_resolved_modifications(self, initialised_config): - config_set(custom='1K+28.0313') - cfg = _loadConfigFile() - assert '1K+28.0313' in settings.resolvedModifications(cfg) - - def test_n_cyc_does_not_change_mods_spec(self, initialised_config): - before = _loadConfigFile()['index']['mods_spec'] - config_set(n_cyc=True) - assert _loadConfigFile()['index']['mods_spec'] == before - - def test_n_ace_does_not_change_mods_spec(self, initialised_config): - before = _loadConfigFile()['index']['mods_spec'] - config_set(n_ace=True) - assert _loadConfigFile()['index']['mods_spec'] == before - - def test_all_other_config_keys_unchanged_after_new_mod_set(self, initialised_config): - before = _loadConfigFile() - config_set(ox=True, phos=True, n_cyc=True, n_ace=True) - after = _loadConfigFile() - for section, values in before.items(): - if section == 'index': - for key, val in values.items(): - if key not in ('mods_spec', 'fixed_mods', 'nterm_peptide_mods_spec', 'nterm_protein_mods_spec'): - assert after[section][key] == val - else: - assert after[section] == values \ No newline at end of file + def test_all_none_returns_false_and_no_write(self, tmp_path): + changed = config_set(tmp_path, False) + assert changed is False + + def test_protocol_flag_roundtrips(self, tmp_path): + config_set(tmp_path, False, iodo=True) + target = tmp_path / 'comms' / 'config.toml' + with target.open('rb') as f: + cfg = tomllib.load(f) + assert CARBAMIDOMETHYL_MOD in cfg['index']['fixed_mods'] + + def test_organism_flag_roundtrips(self, tmp_path): + config_set(tmp_path, False, organism=['Mt=MEDTR']) + target = tmp_path / 'comms' / 'config.toml' + with target.open('rb') as f: + cfg = tomllib.load(f) + assert cfg['organism'] == {'Mt': 'MEDTR'} + + def test_custom_mod_roundtrips(self, tmp_path): + config_set(tmp_path, False, custom='1K+28.0313') + target = tmp_path / 'comms' / 'config.toml' + with target.open('rb') as f: + cfg = tomllib.load(f) + assert '1K+28.0313' in cfg['index']['custom_mods'] + + @pytest.mark.parametrize('section, key, flag, value', [ + ('convert', 'gzip', 'gzip', True), + ('search', 'threads', 'threads', 16), + ('percolator', 'picked_protein', 'picked_protein', False), + ('quantify', 'measure', 'measure', 'dNSAF'), + ('report', 'lfc_threshold', 'lfc_threshold', 2.0), + ]) + def test_direct_flags_roundtrip(self, tmp_path, section, key, flag, value): + config_set(tmp_path, False, **{flag: value}) + target = tmp_path / 'comms' / 'config.toml' + with target.open('rb') as f: + cfg = tomllib.load(f) + assert cfg[section][key] == value + + def test_unrelated_sections_untouched(self, tmp_path): + target = _resolve_or_create(tmp_path, use_global=False) + with target.open('rb') as f: + before = tomllib.load(f) + config_set(tmp_path, False, iodo=True) + with target.open('rb') as f: + after = tomllib.load(f) + for section in ('search', 'percolator', 'quantify', 'convert'): + assert after.get(section) == before.get(section) + + def test_prints_diff_summary(self, tmp_path, capsys): + config_set(tmp_path, False, iodo=True) + assert '✓' in capsys.readouterr().out + + +class TestConfigSetLocalTarget: + def test_writes_to_local_path_not_global(self, tmp_path, isolated_config_dir): + config_set(tmp_path, False, iodo=True) + assert (tmp_path / 'comms' / 'config.toml').exists() + assert not globalConfigPath().exists() \ No newline at end of file diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index 007b228..fd6c3ea 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -3,12 +3,32 @@ ''' # -- Import external dependencies -import pytest +import json, pytest from pathlib import Path from unittest.mock import patch # -- Import functions under test -from comms.commands.report import _resolve_r_script, _write_index, run_report +from comms.commands.report import ( + _log_organism_outcomes, + _read_status, + _resolve_r_script, + _section_status, + _write_index, + run_report, +) + +def _assert_config_sidecar(out_dir: Path, command: str, overrides_given: bool, expect_key: tuple[str, str] | None = None, expect_value=None): + sidecar = out_dir / f'{command}.config.toml' + if not overrides_given: + assert not sidecar.exists() + return + assert sidecar.exists() + import tomllib + with sidecar.open('rb') as f: + cfg = tomllib.load(f) + if expect_key: + section, key = expect_key + assert cfg[section][key] == expect_value # -- Define tests for resolving R script paths class TestResolveRScript: @@ -49,6 +69,22 @@ def test_parameters_included_in_index(self, tmp_path): _write_index(tmp_path, {'organism_prefix': 'Mtrun'}, {}, {}) assert 'organism_prefix' in (tmp_path / 'index.md').read_text() + def test_partial_status_glyph(self, tmp_path): + _write_index(tmp_path, {}, {'da': 'partial'}, {'da': {'Mt': 'ok', 'Ri': 'failed'}}) + assert 'PARTIAL' in (tmp_path / 'index.md').read_text() + + def test_skipped_status_glyph(self, tmp_path): + _write_index(tmp_path, {}, {'qc': 'skipped'}, {}) + assert 'SKIPPED' in (tmp_path / 'index.md').read_text() + + def test_per_organism_lines_nested_under_section(self, tmp_path): + _write_index(tmp_path, {}, {'da': 'partial'}, {'da': {'Mt': 'ok', 'Ri': 'failed'}}) + content = (tmp_path / 'index.md').read_text() + da_line_index = content.index('- da:') + mt_line_index = content.index('Mt: ok') + assert mt_line_index > da_line_index + + # -- Define shared fixtures # -- _make_quantify_dir: returns Path to example quantify output def _make_quantify_dir(tmp_path: Path) -> Path: @@ -82,6 +118,7 @@ def _run_report_with_mocks(tmp_path, experiment_ctx, sections, **kwargs): min_reps=2, lfc_threshold=1.0, fdr_threshold=0.05, + top_n=20, overwrite=True, rscript='Rscript', in_pipeline = False, @@ -108,6 +145,7 @@ def test_raises_when_no_spectral_count_files(self, tmp_path, experiment_ctx): min_reps=2, lfc_threshold=1.0, fdr_threshold=0.05, + top_n=20, sections=['qc'], overwrite=True, rscript='Rscript', @@ -130,6 +168,7 @@ def test_raises_when_output_exists_without_overwrite(self, tmp_path, experiment_ min_reps=2, lfc_threshold=1.0, fdr_threshold=0.05, + top_n=20, sections=['qc'], overwrite=False, rscript='Rscript', @@ -150,6 +189,7 @@ def test_raises_when_rscript_not_found(self, tmp_path, experiment_ctx): min_reps=2, lfc_threshold=1.0, fdr_threshold=0.05, + top_n=20, sections=['qc'], overwrite=True, rscript='Rscript', @@ -184,4 +224,101 @@ def test_da_section_receives_lfc_and_fdr_as_positional_args(self, tmp_path, expe def test_logger_named_report(self, tmp_path, experiment_ctx): from comms.utils.log import logMsg _run_report_with_mocks(tmp_path, experiment_ctx, sections=['qc']) - assert logMsg._instance.logger.name == 'report' \ No newline at end of file + assert logMsg._instance.logger.name == 'report' + + def test_override_writes_report_config_sidecar(self, tmp_path, experiment_ctx): + _run_report_with_mocks(tmp_path, experiment_ctx, sections=['qc'], lfc_threshold=2.0) + sidecar = experiment_ctx.root / 'comms/results/report/report.config.toml' + assert sidecar.exists() + import tomllib + with sidecar.open('rb') as f: + cfg = tomllib.load(f) + assert cfg['report']['lfc_threshold'] == 2.0 + + def test_no_override_writes_no_sidecar(self, tmp_path, experiment_ctx): + defaults = dict( + quantify_dir=_make_quantify_dir(tmp_path), sample_sheet=_make_sample_sheet(tmp_path), + ctx=experiment_ctx, lfq_dir=None, ref_info=None, cont_csv=None, + organism_prefix='Mtrun', min_reps=None, lfc_threshold=None, + fdr_threshold=None, top_n=None, overwrite=True, rscript='Rscript', in_pipeline=False, + ) + with patch('comms.commands.report._run_r_section', return_value=True), patch('shutil.which', return_value='/usr/bin/Rscript'): + run_report(sections=['qc'], **defaults) + sidecar = experiment_ctx.root / 'comms/results/report/report.config.toml' + assert not sidecar.exists() + + def test_lfq_dir_falls_back_to_conventional_location(self, tmp_path, experiment_ctx): + lfq_dir = experiment_ctx.root / 'comms/results/lfq' + lfq_dir.mkdir(parents=True) + mock_run = _run_report_with_mocks(tmp_path, experiment_ctx, sections=['concordance'], lfq_dir=None) + called = [c.kwargs['section'] for c in mock_run.call_args_list] + assert 'concordance' in called + + def test_ref_info_falls_back_to_context_value(self, tmp_path, experiment_ctx): + ref = tmp_path / 'ref.txt' + ref.touch() + experiment_ctx.metadata['report'] = {'ref_info': str(ref)} + _run_report_with_mocks(tmp_path, experiment_ctx, sections=['qc'], ref_info=None) # should not raise + +class TestReadStatus: + def test_missing_file_returns_empty_dicts(self, tmp_path): + assert _read_status(tmp_path) == ({}, {}) + + def test_malformed_json_returns_empty_dicts(self, tmp_path): + (tmp_path / '_status.json').write_text('{not valid json') + assert _read_status(tmp_path) == ({}, {}) + + def test_well_formed_file_parsed(self, tmp_path): + (tmp_path / '_status.json').write_text(json.dumps({ + 'organisms': {'Mt': 'ok', 'Ri': 'failed'}, + 'reasons': {'Ri': 'insufficient replicates'}, + })) + organisms, reasons = _read_status(tmp_path) + assert organisms == {'Mt': 'ok', 'Ri': 'failed'} + assert reasons == {'Ri': 'insufficient replicates'} + + def test_unrecognised_status_values_dropped(self, tmp_path): + (tmp_path / '_status.json').write_text(json.dumps({ + 'organisms': {'Mt': 'ok', 'Ri': 'bogus_status'}, + 'reasons': {}, + })) + organisms, _ = _read_status(tmp_path) + assert organisms == {'Mt': 'ok'} + +class TestSectionStatus: + def test_no_organisms_proc_ok_true_is_skipped(self): + assert _section_status(proc_ok=True, organisms={}) == 'skipped' + + def test_no_organisms_proc_ok_false_is_failed(self): + assert _section_status(proc_ok=False, organisms={}) == 'failed' + + def test_all_ok_is_succeeded(self): + assert _section_status(True, {'Mt': 'ok', 'Ri': 'ok'}) == 'succeeded' + + def test_mixed_ok_and_failed_is_partial(self): + assert _section_status(True, {'Mt': 'ok', 'Ri': 'failed'}) == 'partial' + + def test_all_failed_is_failed(self): + assert _section_status(True, {'Mt': 'failed', 'Ri': 'failed'}) == 'failed' + + def test_all_skipped_none_ok_or_failed_is_skipped(self): + assert _section_status(True, {'Mt': 'skipped', 'Ri': 'skipped'}) == 'skipped' + +class TestLogOrganismOutcomes: + def test_logs_one_line_per_organism(self, caplog): + import logging + with caplog.at_level(logging.INFO): + _log_organism_outcomes('da', {'Mt': 'ok', 'Ri': 'failed'}, {}) + assert 'Mt' in caplog.text and 'Ri' in caplog.text + + def test_includes_reason_when_present(self, caplog): + import logging + with caplog.at_level(logging.WARN): + _log_organism_outcomes('da', {'Ri': 'failed'}, {'Ri': 'insufficient replicates'}) + assert 'insufficient replicates' in caplog.text + + def test_omits_parenthetical_when_no_reason(self, caplog): + import logging + with caplog.at_level(logging.WARN): + _log_organism_outcomes('da', {'Mt': 'ok'}, {}) + assert '()' not in caplog.text \ No newline at end of file diff --git a/tests/unit/test_samples.py b/tests/unit/test_samples.py index 9901e89..003eec5 100644 --- a/tests/unit/test_samples.py +++ b/tests/unit/test_samples.py @@ -12,20 +12,20 @@ # -- Define tests for loading sample sheet class TestLoadSampleSheet: - def test_loads_valid_tsv(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_loads_valid_tsv(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) assert isinstance(df, pd.DataFrame) - def test_returns_correct_row_count(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_returns_correct_row_count(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) assert len(df) == 2 - def test_column_names_are_lowercased(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_column_names_are_lowercased(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) assert all(c == c.lower() for c in df.columns) - def test_required_columns_present(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_required_columns_present(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) for col in REQUIRED_COLUMNS: assert col in df.columns @@ -48,8 +48,8 @@ def test_loads_csv_as_well_as_tsv(self, tmp_path): df = loadSampleSheet(p) assert len(df) == 1 - def test_optional_batch_column_allowed(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_optional_batch_column_allowed(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) assert 'batch' in df.columns # present in fixture, should not cause error def test_strips_whitespace_from_column_names(self, tmp_path): @@ -62,73 +62,75 @@ def test_strips_whitespace_from_column_names(self, tmp_path): # -- Define tests for filtering samples by treatment class TestGetSamplesByTreatment: - def test_filters_correctly(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) - result = getSamplesByTreatment(df, 'CONTROL') + def test_filters_correctly(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) + result = getSamplesByTreatment(df, 'MOCK') assert len(result) == 1 assert result.iloc[0]['sample_id'] == 'S1' - def test_case_insensitive(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) - result = getSamplesByTreatment(df, 'control') + def test_case_insensitive(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) + result = getSamplesByTreatment(df, 'mock') assert len(result) == 1 - def test_returns_empty_for_unknown_treatment(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_returns_empty_for_unknown_treatment(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) result = getSamplesByTreatment(df, 'NONEXISTENT') assert len(result) == 0 - def test_returns_copy_not_view(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) - result = getSamplesByTreatment(df, 'CONTROL') + def test_returns_copy_not_view(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) + result = getSamplesByTreatment(df, 'MOCK') result['sample_id'] = 'MODIFIED' - original = loadSampleSheet(valid_sample_sheet) + original = loadSampleSheet(sample_sheet_factory(['WCL'])) assert original.iloc[0]['sample_id'] == 'S1' # -- Define tests for filtering samples by fraction class TestGetSamplesByFraction: - def test_filters_correctly(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_filters_correctly(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) result = getSamplesByFraction(df, 'WCL') assert len(result) == 2 assert result.iloc[0]['sample_id'] == 'S1' - def test_case_insensitive(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_case_insensitive(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) result = getSamplesByFraction(df, 'wcl') assert len(result) == 2 - def test_returns_empty_for_unknown_treatment(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_returns_empty_for_unknown_treatment(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) result = getSamplesByFraction(df, 'NONEXISTENT') assert len(result) == 0 - def test_returns_copy_not_view(self, valid_sample_sheet): - df = loadSampleSheet(valid_sample_sheet) + def test_returns_copy_not_view(self, sample_sheet_factory): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) result = getSamplesByFraction(df, 'WCL') result['sample_id'] = 'MODIFIED' - original = loadSampleSheet(valid_sample_sheet) + original = loadSampleSheet(sample_sheet_factory(['WCL'])) assert original.iloc[0]['sample_id'] == 'S1' # -- Define tests for creating file map class TestGetRawFileMap: - def test_maps_existing_files(self, valid_sample_sheet, tmp_path): - df = loadSampleSheet(valid_sample_sheet) + def test_maps_existing_files(self, sample_sheet_factory, tmp_path): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) # Create the dummy mzML file in tmp_path so it is "found" - (tmp_path / 'synthetic.mzML').touch() + (tmp_path / 'sample_mock_wcl_1.RAW').touch() + (tmp_path / 'sample_treat_wcl_1.RAW').touch() file_map = getRawFileMap(df, tmp_path) assert 'S1' in file_map assert 'S2' in file_map - def test_omits_missing_files(self, valid_sample_sheet, tmp_path): - df = loadSampleSheet(valid_sample_sheet) + def test_omits_missing_files(self, sample_sheet_factory, tmp_path): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) # Do NOT create synthetic.mzML — file does not exist file_map = getRawFileMap(df, tmp_path) assert file_map == {} - def test_returns_path_objects(self, valid_sample_sheet, tmp_path): - df = loadSampleSheet(valid_sample_sheet) - (tmp_path / 'synthetic.mzML').touch() + def test_returns_path_objects(self, sample_sheet_factory, tmp_path): + df = loadSampleSheet(sample_sheet_factory(['WCL'])) + (tmp_path / 'sample_mock_wcl_1.RAW').touch() + (tmp_path / 'sample_treat_wcl_1.RAW').touch() file_map = getRawFileMap(df, tmp_path) for v in file_map.values(): assert isinstance(v, Path) diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index 2b14a32..5befdd0 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -7,7 +7,7 @@ from pathlib import Path # -- Import internal functions -from comms.utils.settings import loadDefaultConfig, globalConfigPath, resolveConfig, resolvedModifications +from comms.utils.settings import globalConfigPath, initComms, loadDefaultConfig, resolve_config_value, resolveConfig, resolvedModifications # -- Define tests for validating user config path class TestUserConfigPath: @@ -46,42 +46,64 @@ def test_default_when_nothing_present(self, isolated_config_dir, tmp_path): assert isinstance(cfg, dict) and 'search' in cfg assert 'default' in source - def test_global_used_when_present(self, isolated_config_dir): - from comms.commands.config import config_init - config_init() - cfg, source = resolveConfig(None) - assert source.startswith('global') + def test_default_when_nothing_present(self, isolated_config_dir, tmp_path): + cfg, source = resolveConfig(tmp_path / 'comms') + assert isinstance(cfg, dict) and 'search' in cfg + assert 'default' in source - def test_local_preferred_over_global(self, isolated_config_dir, tmp_path): - from comms.commands.config import config_init - from comms.utils.settings import loadDefaultConfig - import tomli_w - config_init() # global exists - comms = tmp_path / 'comms'; comms.mkdir(parents=True) - local = loadDefaultConfig(); local['search']['threads'] = 7 - with (comms / 'config.toml').open('wb') as f: - tomli_w.dump(local, f) - cfg, source = resolveConfig(comms) - assert cfg['search']['threads'] == 7 + def test_local_preferred_when_present(self, isolated_config_dir, tmp_path): + comms_dir = tmp_path / 'comms' + comms_dir.mkdir() + local_cfg = loadDefaultConfig() + local_cfg['search']['threads'] = 99 + with (comms_dir / 'config.toml').open('wb') as f: + import tomli_w + tomli_w.dump(local_cfg, f) + cfg, source = resolveConfig(comms_dir) + assert cfg['search']['threads'] == 99 assert source.startswith('local') + + def test_global_used_when_no_local(self, isolated_config_dir, tmp_path): + import tomli_w + global_cfg = loadDefaultConfig() + global_cfg['search']['threads'] = 77 + with (isolated_config_dir / 'config.toml').open('wb') as f: + tomli_w.dump(global_cfg, f) + cfg, source = resolveConfig(tmp_path / 'comms') + assert cfg['search']['threads'] == 77 + assert source.startswith('global') + +# -- Define tests for resolving a config value +class TestResolveConfigValue: + def test_override_returned_when_given(self): + assert resolve_config_value({'search': {'threads': 4}}, 'search', 'threads', 8) == 8 + + def test_falls_back_to_config_value(self): + assert resolve_config_value({'search': {'threads': 4}}, 'search', 'threads', None) == 4 + + def test_raises_keyerror_when_absent_from_both(self): + with pytest.raises(KeyError): + resolve_config_value({}, 'search', 'threads', None) + # -- Define tests for falling back to default configuration class TestConfigFallback: - def test_falls_back_to_defaults_when_no_user_config(self, isolated_config_dir, monkeypatch): - ''' - When globalConfigPath() returns a path that does not exist, the module - should load bundled defaults. We simulate this by importing settings - with a patched path pointing to a non-existent file. - ''' - import importlib - import comms.utils.settings as settings_mod - - monkeypatch.setattr(settings_mod, 'globalConfigPath', Path('/nonexistent/config.toml')) - - # Reload to re-run the module-level config loading logic - # (We test the function directly since reloading modules is fragile in pytest) - defaults = loadDefaultConfig() - assert isinstance(defaults, dict) - assert 'search' in defaults + def test_resolve_config_falls_back_to_defaults(self, isolated_config_dir, tmp_path, monkeypatch): + monkeypatch.setattr( + 'comms.utils.settings.globalConfigPath', + lambda: tmp_path / '_nonexistent_config.toml', + ) + cfg, source = resolveConfig(tmp_path / 'comms_that_does_not_exist') + assert cfg == loadDefaultConfig() + assert 'bundled' in source or 'default' in source + +# -- Define tests for initComms function +class TestInitComms: + def test_runs_without_raising(self, capsys): + initComms() + + def test_prints_comms_name(self, capsys): + initComms() + assert 'comMS' in capsys.readouterr().out # -- Define tests for resolvedModifications function class TestResolvedModsSpec: From 37e7ac33f19f5844dc9ab42037cfcde52407e07f Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 13:00:41 +0100 Subject: [PATCH 076/108] test: add unit tests - Added tests/unit/test_installrdeps.py to define unit tests for utility functions used to install R dependencies. - Added tests/unit/test_readiness.py to define unit tests for utility functions used to check readiness of commands. --- tests/unit/test_installrdeps.py | 114 ++++++++++++++++++++++++++++++++ tests/unit/test_readiness.py | 58 ++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 tests/unit/test_installrdeps.py create mode 100644 tests/unit/test_readiness.py diff --git a/tests/unit/test_installrdeps.py b/tests/unit/test_installrdeps.py new file mode 100644 index 0000000..6b230a7 --- /dev/null +++ b/tests/unit/test_installrdeps.py @@ -0,0 +1,114 @@ +''' +Unit tests for src/comms/utils/installrdeps.py +''' +import json +from unittest.mock import patch, MagicMock +import pytest + +from comms.utils.installrdeps import ( + check_r_dependencies, install_r_dependencies, install_r_dependencies_terminal, _print_dependency_table, +) + + +class TestCheckRDependencies: + def test_returns_none_when_rscript_not_on_path(self): + with patch('shutil.which', return_value=None): + assert check_r_dependencies() is None + + def test_parses_well_formed_json(self): + payload = json.dumps({'installed': ['limma'], 'missing': ['iq']}) + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=0, stdout=payload, stderr='')): + result = check_r_dependencies() + assert result == {'installed': ['limma'], 'missing': ['iq']} + + def test_nonzero_return_code_returns_none(self): + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=1, stdout='', stderr='boom')): + assert check_r_dependencies() is None + + def test_malformed_json_returns_none(self): + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=0, stdout='not json', stderr='')): + assert check_r_dependencies() is None + + +class TestPrintDependencyTable: + def test_only_installed_line_when_nothing_missing(self, caplog): + import logging + with caplog.at_level(logging.INFO): + _print_dependency_table({'installed': ['limma'], 'missing': []}) + assert 'Installed' in caplog.text and 'Missing' not in caplog.text + + def test_both_lines_when_something_missing(self, caplog): + import logging + with caplog.at_level(logging.INFO): + _print_dependency_table({'installed': ['limma'], 'missing': ['iq']}) + assert 'Missing' in caplog.text + assert 'r-utils install' in caplog.text + + def test_neither_line_when_both_empty(self, caplog): + import logging + with caplog.at_level(logging.INFO): + _print_dependency_table({'installed': [], 'missing': []}) + assert caplog.text == '' + + +class TestInstallRDependencies: + def test_returns_false_when_rscript_not_on_path(self): + with patch('shutil.which', return_value=None): + assert install_r_dependencies() is False + + def test_streams_stdout_lines(self, caplog): + import logging + proc = MagicMock() + proc.stdout = iter(['installing limma...\n', 'done\n']) + proc.returncode = 0 + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.Popen', return_value=proc), caplog.at_level(logging.INFO): + result = install_r_dependencies() + assert result is True + assert 'installing limma' in caplog.text + + def test_nonzero_exit_returns_false(self): + proc = MagicMock() + proc.stdout = iter([]) + proc.returncode = 1 + with patch('shutil.which', return_value='/usr/bin/Rscript'), patch('subprocess.Popen', return_value=proc): + assert install_r_dependencies() is False + + +class TestInstallRDependenciesTerminal: + def test_nothing_missing_informs_without_prompting(self, caplog): + import logging + payload = json.dumps({'installed': ['limma'], 'missing': []}) + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=0, stdout=payload, stderr='')), \ + caplog.at_level(logging.INFO): + install_r_dependencies_terminal() + assert 'No dependencies missing' in caplog.text + + def test_missing_and_user_confirms_installs(self): + payload = json.dumps({'installed': [], 'missing': ['iq']}) + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=0, stdout=payload, stderr='')), \ + patch('comms.utils.installrdeps.logMsg.input', return_value='y'), \ + patch('comms.utils.installrdeps.install_r_dependencies') as mock_install: + install_r_dependencies_terminal() + mock_install.assert_called_once() + + def test_missing_and_user_declines_cancels(self): + payload = json.dumps({'installed': [], 'missing': ['iq']}) + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=0, stdout=payload, stderr='')), \ + patch('comms.utils.installrdeps.logMsg.input', return_value='n'), \ + patch('comms.utils.installrdeps.install_r_dependencies') as mock_install: + install_r_dependencies_terminal() + mock_install.assert_not_called() + + def test_malformed_output_logs_error_without_raising(self, caplog): + import logging + with patch('shutil.which', return_value='/usr/bin/Rscript'), \ + patch('subprocess.run', return_value=MagicMock(returncode=0, stdout='not json', stderr='')), \ + caplog.at_level(logging.ERROR): + install_r_dependencies_terminal() # should not raise \ No newline at end of file diff --git a/tests/unit/test_readiness.py b/tests/unit/test_readiness.py new file mode 100644 index 0000000..e7aedd9 --- /dev/null +++ b/tests/unit/test_readiness.py @@ -0,0 +1,58 @@ +''' +Unit tests for src/comms/utils/readiness.py +''' +from comms.utils.readiness import missing_requirements, COMMANDS + + +def _all_true(**overrides): + base = dict( + has_data=True, has_database=True, has_sample_sheet=True, has_organism_prefix=True, + multispecies=True, has_organism_tags=True, has_trfp=True, has_crux=True, has_r_deps=True, + ) + base.update(overrides) + return base + + +class TestMissingRequirements: + def test_all_satisfied_every_command_empty(self): + result = missing_requirements(**_all_true()) + assert all(gaps == [] for gaps in result.values()) + + def test_missing_data_appears_only_in_relevant_commands(self): + result = missing_requirements(**_all_true(has_data=False)) + assert 'data files' in result['convert'] + assert 'data files' in result['search'] + assert 'data files' in result['lfq'] + assert 'data files' not in result['index'] + assert 'data files' not in result['quantify'] + + def test_missing_crux_appears_in_every_crux_dependent_command(self): + result = missing_requirements(**_all_true(has_crux=False)) + for cmd in ('index', 'search', 'rescore', 'lfq', 'quantify'): + assert 'Crux' in result[cmd] + assert 'Crux' not in result['convert'] + assert 'Crux' not in result['report'] + + def test_missing_trfp_only_affects_convert(self): + result = missing_requirements(**_all_true(has_trfp=False)) + assert 'ThermoRawFileParser' in result['convert'] + assert all('ThermoRawFileParser' not in gaps for cmd, gaps in result.items() if cmd != 'convert') + + def test_missing_r_deps_only_affects_report(self): + result = missing_requirements(**_all_true(has_r_deps=False)) + assert 'R dependencies' in result['report'] + + def test_pipeline_is_union_of_all_other_commands(self): + result = missing_requirements(**_all_true(has_data=False, has_crux=False)) + expected = set() + for cmd, gaps in result.items(): + if cmd != 'pipeline': + expected.update(gaps) + assert set(result['pipeline']) == expected + + # -- the §1.2 assumption, pinned explicitly -- + def test_rescore_requires_organism_patterns_only_when_multispecies(self): + single = missing_requirements(**_all_true(multispecies=False, has_organism_tags=False)) + multi = missing_requirements(**_all_true(multispecies=True, has_organism_tags=False)) + assert 'organism patterns' not in single['rescore'] + assert 'organism patterns' in multi['rescore'] \ No newline at end of file From 93baa26e8efe9a28dff3272966729179ed343883 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 13:03:46 +0100 Subject: [PATCH 077/108] test: update integration tests - Modified tests/integration/test_pipeline.py to refactor integration tests for LFQ quantification. --- tests/integration/test_pipeline.py | 84 +++++++++++++++--------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index dd7cb4d..d000d2a 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -716,22 +716,22 @@ def _side_effect(**kwargs): # LFQ # =========================================================================== class TestRunLfqOutputDirectories: - def test_lfq_root_directory_is_created(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, tmp_path, experiment_ctx): + def test_lfq_root_directory_is_created(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, tmp_path, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True): run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) assert (tmp_path / 'comms' / 'results' / 'lfq').exists() - def test_single_fraction_creates_one_output_directory(self, crux_bin, single_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_single_fraction, tmp_path, experiment_ctx): + def test_single_fraction_creates_one_output_directory(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, tmp_path, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True): run_lfq( - rescore_dir=single_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1']), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_single_fraction, + sample_sheet=sample_sheet_factory(['WCL']), ctx=experiment_ctx, ) lfq_root = tmp_path / 'comms' / 'results' / 'lfq' @@ -739,12 +739,12 @@ def test_single_fraction_creates_one_output_directory(self, crux_bin, single_fra assert len(subdirs) == 1 assert subdirs[0].name == 'WCL' - def test_creates_per_fraction_output_directories(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, tmp_path, experiment_ctx): + def test_creates_per_fraction_output_directories(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, tmp_path, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True): run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) lfq_root = tmp_path / 'comms' / 'results' / 'lfq' @@ -753,64 +753,64 @@ def test_creates_per_fraction_output_directories(self, crux_bin, multi_fraction_ assert (lfq_root / 'PUR').exists() class TestRunLfqCruxCalls: - def test_lfq_called_once_per_fraction(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, experiment_ctx): + def test_lfq_called_once_per_fraction(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True) as mock_lfq: run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) assert mock_lfq.call_count == 3 - def test_lfq_called_with_correct_fraction_psm_files(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, experiment_ctx): + def test_lfq_called_with_correct_fraction_psm_files(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True) as mock_lfq: run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) all_psm_files = [c.kwargs['psm_files'] for c in mock_lfq.call_args_list] for files in all_psm_files: assert len(files) == 2 - def test_lfq_not_called_for_unmatched_psm_files(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, experiment_ctx): - orphan = multi_fraction_psm_dir / 'orphan_file.percolator.target.psms.txt' + def test_lfq_not_called_for_unmatched_psm_files(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx): + orphan = psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]) / 'orphan_file.percolator.target.psms.txt' orphan.touch() with patch('comms.commands.lfq.cruxutil.lfq', return_value=True) as mock_lfq: run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) assert mock_lfq.call_count == 3 - def test_lfq_receives_correct_fileroot_per_fraction(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, experiment_ctx): + def test_lfq_receives_correct_fileroot_per_fraction(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True) as mock_lfq: run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) fileroots = {c.kwargs['fileroot'] for c in mock_lfq.call_args_list} assert fileroots == {'WCL', 'ECF', 'PUR'} class TestRunLfqEarlyExit: - def test_raises_system_exit_when_no_psm_files(self, crux_bin, synthetic_mzml, valid_sample_sheet_multiple_fractions, tmp_path, experiment_ctx): + def test_raises_system_exit_when_no_psm_files(self, crux_bin, synthetic_mzml, sample_sheet_factory, tmp_path, experiment_ctx): empty_rescore_dir = tmp_path / 'empty_rescore' empty_rescore_dir.mkdir() with pytest.raises(SystemExit): run_lfq( rescore_dir=empty_rescore_dir, data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) - def test_lfq_called_per_fraction_even_when_no_mzml_matches(self, crux_bin, multi_fraction_psm_dir, valid_sample_sheet_multiple_fractions, tmp_path, experiment_ctx): + def test_lfq_called_per_fraction_even_when_no_mzml_matches(self, crux_bin, psm_dir_factory, sample_sheet_factory, tmp_path, experiment_ctx): ''' cruxutil.lfq is called for each fraction even when the supplied mzML file does not match any PSM file stem (the mock returns False for each call) ''' @@ -818,45 +818,45 @@ def test_lfq_called_per_fraction_even_when_no_mzml_matches(self, crux_bin, multi dummy_mzml.touch() with patch('comms.commands.lfq.cruxutil.lfq', return_value=False) as mock_lfq: run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[dummy_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) assert mock_lfq.call_count == 3 class TestRunLfqWarnings: - def test_logs_warning_when_lfq_fails_for_fraction(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, experiment_ctx, caplog): + def test_logs_warning_when_lfq_fails_for_fraction(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx, caplog): with patch('comms.commands.lfq.cruxutil.lfq', return_value=False), caplog.at_level(logging.WARNING): run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) assert 'LFQ failed' in caplog.text or 'failed' in caplog.text.lower() - def test_completes_remaining_fractions_even_if_one_fails(self, crux_bin, multi_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_multiple_fractions, experiment_ctx): + def test_completes_remaining_fractions_even_if_one_fails(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx): call_count = {'n': 0} def _mock_lfq(**kwargs): call_count['n'] += 1 return call_count['n'] != 1 with patch('comms.commands.lfq.cruxutil.lfq', side_effect=_mock_lfq): run_lfq( - rescore_dir=multi_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1', 'sample_mock_ecf_1', 'sample_treat_ecf_1', 'sample_mock_pur_1', 'sample_treat_pur_1',]), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_multiple_fractions, + sample_sheet=sample_sheet_factory(['WCL', 'ECF', 'PUR']), ctx=experiment_ctx, ) assert call_count['n'] == 3 class TestRunLfqLogger: - def test_logger_is_named_lfq(self, crux_bin, single_fraction_psm_dir, synthetic_mzml, valid_sample_sheet_single_fraction, experiment_ctx): + def test_logger_is_named_lfq(self, crux_bin, psm_dir_factory, synthetic_mzml, sample_sheet_factory, experiment_ctx): with patch('comms.commands.lfq.cruxutil.lfq', return_value=True): run_lfq( - rescore_dir=single_fraction_psm_dir, + rescore_dir=psm_dir_factory(['sample_mock_wcl_1', 'sample_treat_wcl_1']), data_files=[synthetic_mzml], - sample_sheet=valid_sample_sheet_single_fraction, + sample_sheet=sample_sheet_factory(['WCL']), ctx=experiment_ctx, ) assert logMsg._instance.logger.name == 'lfq' @@ -910,11 +910,11 @@ def test_quantify_finds_flat_output( # Pipeline (end-to-end) # =========================================================================== class TestRunPipeline: - def test_pipeline_completes_without_raising(self, crux_bin, synthetic_fixtures, valid_sample_sheet, experiment_ctx): + def test_pipeline_completes_without_raising(self, crux_bin, synthetic_fixtures, sample_sheet_factory, experiment_ctx): fasta, mzml = synthetic_fixtures try: run_pipeline( - sample_sheet=valid_sample_sheet, + sample_sheet=sample_sheet_factory(['WCL']), database=fasta, data=[mzml], ctx=experiment_ctx, @@ -932,11 +932,11 @@ def test_pipeline_completes_without_raising(self, crux_bin, synthetic_fixtures, 'Check that synthetic fixtures are valid and Crux is working.' ) - def test_pipeline_creates_results_tree(self, crux_bin, synthetic_fixtures, valid_sample_sheet, tmp_path, experiment_ctx): + def test_pipeline_creates_results_tree(self, crux_bin, synthetic_fixtures, sample_sheet_factory, tmp_path, experiment_ctx): fasta, mzml = synthetic_fixtures try: run_pipeline( - sample_sheet=valid_sample_sheet, + sample_sheet=sample_sheet_factory(['WCL']), database=fasta, data=[mzml], ctx=experiment_ctx, @@ -956,11 +956,11 @@ def test_pipeline_creates_results_tree(self, crux_bin, synthetic_fixtures, valid stage_dir = results_root / stage assert stage_dir.exists(), f'Expected results directory for stage: {stage}' - def test_comms_logger_is_pipeline(self, crux_bin, synthetic_fixtures, valid_sample_sheet, tmp_path, experiment_ctx): + def test_comms_logger_is_pipeline(self, crux_bin, synthetic_fixtures, sample_sheet_factory, tmp_path, experiment_ctx): fasta, mzml = synthetic_fixtures try: run_pipeline( - sample_sheet=valid_sample_sheet, + sample_sheet=sample_sheet_factory(['WCL']), database=fasta, data=[mzml], ctx=experiment_ctx, From 39f09bd71ad3363044e88458e5a24c136ccb0fa7 Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 13:04:43 +0100 Subject: [PATCH 078/108] chore: update pyproject.toml - Modified pyproject.toml to fix pytest-cov not displaying test coverage. --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5671528..153afc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,13 @@ build-backend = "uv_build" [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "--tb=short -ra --cov=comms --cov-report=term-missing" +addopts = "--tb=short -ra --cov=src/comms --cov-report=term-missing" minversion = "7.0" markers = [ "crux: requires the Crux binary to be present under bin/ (auto-skipped if absent)", "trfp: requires ThermoRawFileParser.exe to be present under bin/ (auto-skipped if absent)", ] + +[tool.coverage.run] +source = ["src"] +relative_files = true \ No newline at end of file From 19200756395668e6ff05574d8f4f97059a0212ab Mon Sep 17 00:00:00 2001 From: Sam Holland Date: Wed, 12 Aug 2026 13:54:48 +0100 Subject: [PATCH 079/108] docs(TEST): update TESTS.md - Modified tests/TESTS.md to reflect changes in test suite. --- tests/TESTS.md | 273 +++++++++++++++++++++++++++++++------------------ 1 file changed, 175 insertions(+), 98 deletions(-) diff --git a/tests/TESTS.md b/tests/TESTS.md index 1c901ed..ef5c078 100644 --- a/tests/TESTS.md +++ b/tests/TESTS.md @@ -5,14 +5,16 @@ This document outlines the comMS test suite: its structure, shared fixtures, and - [Running the test suite](#running-the-test-suite) - [Test markers](#test-markers) - [Shared fixtures](#shared-fixtures) - - [Binary availability fixtures](#binary-availability-fixtures) + - [Bin directory and binary availability fixtures](#bin-directory-and-binary-availability-fixtures) - [Synthetic file fixtures](#synthetic-file-fixtures) - [Sample sheet fixtures](#sample-sheet-fixtures) - [Config fixtures](#config-fixtures) - [Synthetic Percolator results](#synthetic-percolator-results) - - [LFQ fixtures](#lfq-fixtures) - - [Rescore integration fixtures](#rescore-integration-fixtures) + - [PSM directory fixtures](#psm-directory-fixtures) - [GUI fixtures](#gui-fixtures) + - [Experiment context fixtures](#experiment-context-fixtures) + - [Experiment builder fixture](#experiment-builder-fixture) + - [Rescore/crux integration fixtures](#rescorecrux-integration-fixtures) - [Synthetic fixture generator](#synthetic-fixture-generator) - [Running standalone](#running-standalone) - [Synthetic proteome](#synthetic-proteome) @@ -21,21 +23,27 @@ This document outlines the comMS test suite: its structure, shared fixtures, and - [Real `.RAW` fixture](#real-raw--fixture) - [Unit tests](#unit-tests) - [`tests/unit/test_config.py`](#testsunittest_configpy) + - [`tests/unit/test_modspec.py`](#testsunittest_modspecpy) - [`tests/unit/test_context.py`](#testsunittest_contextpy) - [`tests/unit/test_experiment.py`](#testsunittest_experimentpy) - [`tests/unit/test_fasta.py`](#testsunittest_fastapy) + - [`tests/unit/test_installrdeps.py`](#testsunittest_installrdepspy) - [`tests/unit/test_lfq.py`](#testsunittest_lfqpy) + - [`tests/unit/test_license.py`](#testsunittest_licensepy) - [`tests/unit/test_parammedic.py`](#testsunittest_parammedicpy) - [`tests/unit/test_paths.py`](#testsunittest_pathspy) + - [`tests/unit/test_readiness.py`](#testsunittest_readinesspy) - [`tests/unit/test_report.py`](#testsunittest_reportpy) - [`tests/unit/test_rescore.py`](#testsunittest_rescorepy) - [`tests/unit/test_samples.py`](#testsunittest_samplespy) - [`tests/unit/test_settings.py`](#testsunittest_settingspy) + - [`tests/unit/test_sheet.py`](#testsunittest_sheetpy) - [`tests/unit/test_uninstall.py`](#testsunittest_uninstallpy) - [`tests/unit/test_validate.py`](#testsunittest_validatepy) - [`tests/unit/test_version.py`](#testsunittest_versionpy) - [`tests/unit/gui/test_gui_models.py`](#testsunitguitest_gui_modelspy) - [`tests/unit/gui/test_gui_panels.py`](#testsunitguitest_gui_panelspy) + - [`tests/unit/gui/test_gui_readiness.py`](#testsunitguitest_gui_readinesspy) - [`tests/unit/gui/test_gui_status.py`](#testsunitguitest_gui_statuspy) - [`tests/unit/gui/test_gui_widgets.py`](#testsunitguitest_gui_widgetspy) - [Integration tests](#integration-tests) @@ -44,6 +52,7 @@ This document outlines the comMS test suite: its structure, shared fixtures, and - [`tests/integration/test_pipeline.py`](#testsintegrationtest_pipelinepy) - [`tests/integration/test_trfp.py`](#testsintegrationtest_trfppy) - [R tests](#r-tests) + - [`tests/r/helper.R`](#testsrhelperr) - [`tests/r/test_utils_import.R`](#testsrtest_utils_importr) - [`tests/r/test_utils_normalise.R`](#testsrtest_utils_normaliser) @@ -83,8 +92,8 @@ Two custom markers control which tests are run depending on external binary avai | Marker | Requirement | |---|---| -| `crux` | Requires the Crux binary under `bin/` | -| `trfp` | Requires ThermoRawFileParser under `bin/` | +| `crux` | Requires the Crux binary under `tests/bin/` | +| `trfp` | Requires ThermoRawFileParser under `tests/bin/` | Tests decorated with these markers are skipped automatically (with a message) if the corresponding binary is not found, and the rest of the suite continue unaffected. To run only unmarked tests (i.e. those with no external dependency): @@ -98,16 +107,18 @@ uv run pytest -m "not crux and not trfp" ## Shared fixtures Shared fixtures are defined in `tests/conftest.py` and are described below. -### Binary availability fixtures -Two session-scoped fixtures are defined to locate external binaries, which both use the same regular expression/globbing logic as the source code. If a binary is absent, any test depending on that fixture will be skipped. +### Bin directory and binary availability fixtures +A session-scoped, autouse fixture (`_comms_bin_dir_env`) sets the `COMMS_BIN_DIR` environment variable to `tests/bin` for the whole test session, so every test — not just those requesting `crux_bin`/`trfp_exe` directly — resolves binaries against the bundled test `bin/` directory rather than a real installation. + +Two session-scoped fixtures locate external binaries within that directory, mirroring the resolution logic in `comms.utils.crux.findCrux` and `comms.utils.trfp.findTRFP`. If a binary is absent, any test depending on that fixture is skipped. Fixture | Description -- | -- -`crux_bin` | Resolves to Crux binary path under `bin/`; requires `pytest.mark.crux` -`trfp_exe` | Resolves to `ThermoRawFileParser.exe` under `bin`; requires `pytest.mark.trfp` +`crux_bin` | Resolves to the highest-versioned `crux*/bin/crux` under `tests/bin/`; requires `pytest.mark.crux` +`trfp_exe` | Resolves to `*/ThermoRawFileParser(.exe)` under `tests/bin/`; requires `pytest.mark.trfp` ### Synthetic file fixtures -Fixture | Description +Fixture | Description ---|--- `synthetic_fasta(tmp_path)` | Writes `synthetic_proteome.fasta` to a temporary directory and returns its path `synthetic_mzml(tmp_path)` | Writes `synthetic.mzML` to a temporary directory and returns its path @@ -116,55 +127,55 @@ Fixture | Description ### Sample sheet fixtures Fixture | Description ---|--- -`valid_sample_sheet(tmp_path)` | Two samples across two treatments with a `fraction` column (`WCL`) and optional `batch` column, one replicate each; written as TSV +`sample_sheet_factory(tmp_path)` | Returns a function `_make(fractions=('WCL',), batch=True)` that writes a minimal TSV sample sheet (two treatments, `MOCK`/`TREAT`, one replicate each, per given fraction) and returns its path. Replaces the old fixed `valid_sample_sheet*` fixtures, letting each test ask for exactly the fractions it needs `sample_sheet_missing_col(tmp_path)` | Missing the required `treatment` column; used to test validation errors `sample_sheet_duplicate_ids(tmp_path)` | Duplicate `sample_id` values; used to test duplicate detection ### Config fixtures Fixture | Description ---|--- -`isolated_config_dir(tmp_path, monkeypatch)` | Monkeypatches `globalConfigPath()` in both `settings` and `config` modules to point at a temporary directory, so tests don't access the real OS config file +`isolated_config_dir(tmp_path, monkeypatch)` | Monkeypatches `globalConfigPath()` in the `settings`, `config`, and `uninstall` modules to point at a temporary directory, so tests don't touch the real OS config file ### Synthetic Percolator results - Fixture | Description ---|--- `synthetic_percolator_results(tmp_path)` | Writes a minimal synthetic Percolator PSM file at `rescore/EUK/synthetic.EUK.percolator.target.psms.txt`, matching the per-organism subdirectory structure produced by `run_rescore` round 2 and bypassing the need to run Percolator on synthetic data (which does not provide enough PSMs for convergence) -### LFQ fixtures - -Fixture | Description ----|--- -`valid_sample_sheet_single_fraction(tmp_path)` | Single fraction (`WCL`), two treatments; used to test the single-fraction edge case in `_groupPsmsByFraction` -`valid_sample_sheet_multiple_fractions(tmp_path)` | Three fractions (`WCL`, `ECF`, `PUR`), two treatments, one replicate each; written as TSV -`single_fraction_psm_dir(tmp_path)` | Writes two synthetic Percolator PSM files (one fraction) to `comms/results/rescore/`, matching `valid_sample_sheet_single_fraction`; returns the directory path -`multi_fraction_psm_dir(tmp_path)` | Writes six synthetic Percolator PSM files (two per fraction) to `comms/results/rescore/`, matching `valid_sample_sheet_multiple_fractions`; returns the directory path - -### Rescore integration fixtures - -The following fixtures are defined within `tests/integration/test_pipeline.py` for use by the rescore integration test classes. - +### PSM directory fixtures Fixture | Description ---|--- -`two_organism_fasta(tmp_path)` | Writes a combined FASTA containing one TESTEUK protein, one TESTPRO protein, and one cRAP contaminant; returns the path -`synthetic_tide_search_dir(tmp_path)` | Writes a minimal synthetic Tide-search target PSM file to `tmp_path / 'search'` and returns the directory path, bypassing the need to run `tide-search` in rescore tests - -A module-level helper function `_write_per_organism_psm_files(rescore_dir, file_base, labels)` is also defined in `test_pipeline.py`. This is not a fixture but is used as a mock side effect within `TestRunRescoreMergedOutput` to write synthetic per-organism Percolator output files so that `_mergeRescoredPsms` has something to read without Percolator running. +`psm_dir_factory(tmp_path)` | Returns a function `_make(stems)` that writes one synthetic Percolator PSM file per given stem to `comms/results/rescore/` and returns that directory. Replaces the old fixed `single_fraction_psm_dir`/`multi_fraction_psm_dir` fixtures ### GUI fixtures -`QT_QPA_PLATFORM=offscreen` is set at the top of `conftest.py` so Qt widgets can be constructed and painted without a display +`QT_QPA_PLATFORM=minimal` is set at the top of `conftest.py` so Qt widgets can be constructed and painted without a display. A custom Qt message handler (installed at import time via `qInstallMessageHandler`) silently drops the small set of known-harmless warnings the offscreen/minimal platform plugin emits (e.g. "does not support grabbing the keyboard/mouse") and forwards everything else to stderr as Qt would by default. Fixture | Description -- | -- -`qapp` | Session-scoped `QApplication` (created with `QApplication.instance() or QApplication([])`) so GUI widgets can be built in tests; requested via `pytestmark = pytest.mark.usefixtures('qapp')` at the top of each GUI test module +`qapp` | Session-scoped `QApplication` (created with `QApplication.instance() or QApplication([''])`) so GUI widgets can be built in tests; requested via `pytestmark = pytest.mark.usefixtures('qapp')` at the top of each GUI test module ### Experiment context fixtures +Fixture | Description +-- | -- +`experiment_ctx(tmp_path, isolated_config_dir)` | Returns `ExperimentContext.resolve(tmp_path)`: an experiment context rooted at `tmp_path` with no `experiment.toml`, so config resolves to the bundled defaults and `bin_dir` is `None`. Also depends on `isolated_config_dir` so global-config fallback in tests never touches the real OS config -A bare experiment context is provided for the command-level integration tests, which now receive an ExperimentContext rather than a raw output directory. - +### Experiment builder fixture Fixture | Description -- | -- -`experiment_ctx(tmp_path)` | Returns ExperimentContext.resolve(tmp_path): an experiment context rooted at tmp_path with no experiment.toml, so config resolves to the bundled defaults and bin_dir is None +`experiment_builder(tmp_path, isolated_config_dir, sample_sheet_factory, psm_dir_factory)` | Returns a chainable builder for composing a `comms/` directory from only the pieces a test needs: `.with_sample_sheet(fractions, batch)` writes a sample sheet and records it under `[files]`; `.with_stage_output(stage, files)` writes placeholder output for a pipeline stage (PSM files via `psm_dir_factory` for `'rescore'`, otherwise empty placeholder files); `.with_metadata(**kwargs)` merges arbitrary sections into `experiment.toml`; `.build()` writes `experiment.toml` and returns `(root, ctx)` via `ExperimentContext.resolve` + +### Rescore/crux integration fixtures +The following fixtures are defined within `tests/integration/test_pipeline.py` and `tests/integration/test_crux.py` for use by their respective integration test classes. + +Fixture | Description +---|--- +`two_organism_fasta(tmp_path)` | Writes a combined FASTA containing one TESTEUK protein, one TESTPRO protein, and one cRAP contaminant; returns the path +`synthetic_tide_search_dir(tmp_path)` | Writes a minimal synthetic Tide-search target PSM file to `tmp_path / 'search'` and returns the directory path, bypassing the need to run `tide-search` in rescore tests +`pipeline_index(crux_bin, tmp_path_factory)` | Module-scoped: builds one shared Tide index for all `TestRunSearch*` tests in `test_pipeline.py`, skipping the module's tests if `run_index` fails +`pipeline_search(crux_bin, pipeline_index, tmp_path_factory)` | Module-scoped: runs `run_search` once and shares `(search_dir, fasta, work)` across `TestRunRescore` tests, skipping if `run_search` fails +`built_index(crux_bin, tmp_path_factory)` | Module-scoped, in `test_crux.py`: builds one shared Tide index via `tideIndex` for all `TestTideSearch` tests +`search_results(crux_bin, built_index, tmp_path_factory)` | Module-scoped, in `test_crux.py`: runs `tideSearch` once and returns `(out_dir, target_file, fasta)` + +Module-level helper functions `_write_combined_percolator_output`, `_write_split_psm_files`, and `_write_combined_psm` are also defined in `test_pipeline.py`/`test_rescore.py`. These are not fixtures but are used as mock side effects to write synthetic per-round output so that downstream logic (`_splitPsmsByOrganism`, the per-organism Percolator round) has something to read without Percolator or Tide-search actually running. ---

^ Back to top

@@ -182,7 +193,7 @@ By default, the script will write files to the directory containing the script. python tests/fixtures/generate_fixtures.py path/to/output/dir ``` ### Synthetic proteome -The synthetic proteome is written to `synthetic_proteome.fasta`. It contains five protein sequences, each with one or two tryptic peptides which exclusively map to the source protein. The protein IDs and peptide sequences are: +The synthetic proteome is written to `synthetic_proteome.fasta`. It contains five protein sequences; PROT1's sequence (`ACDEFGHIKLMNPQRSTVWYK`) is a single tryptic run yielding two of the target peptides, so that a multi-peptide protein is represented without duplicating sequence content. The protein IDs and peptide sequences are: Protein ID | Tryptic peptides | ---|--- @@ -222,7 +233,7 @@ Peptide mass = sum(residues) + water b-ions and y-ions are singly charged and skip the terminal ions (b1 and y1), following the standard convention. ### Real `.RAW ` fixture -Valid synthetic `.RAW` file cannot be generated without the ThermoFisher vendor SDK, therefore integration tests which would require a `.RAW` file are gated behind the `REAL_RAW_FIXTURE` guard. To run these tests, place a valid file at: +Valid synthetic `.RAW` file cannot be generated without the ThermoFisher vendor SDK, therefore integration tests which would require a `.RAW` file are gated behind the `REAL_RAW_FIXTURE` guard (a constant defined in `test_trfp.py` and imported by `test_convert.py`). To run these tests, place a valid file at: ``` tests/fixtures/real_sample.RAW ``` @@ -235,47 +246,53 @@ If this file is absent, the relevant tests are skipped automatically. Unit tests cover logic in isolation, i.e. they do not require external binaries and do not write to the local filesystem beyond `tmp_path`. All config-touching tests use the `isolated_config_dir` fixture described [above](#config-fixtures). ### `tests/unit/test_config.py` -Unit tests covering `src/comms/commands/config.py`, and indirectly `src/comms/utils/settings.py`: +Unit tests covering `src/comms/commands/config.py` following the config system overhaul (mod/protocol-flag logic has moved to `comms/utils/modspec.py`, covered separately below): Class | Test description -- | -- -`TestLoadDefaultConfig` | returns a dict; contains expected top-level sections; search section has required keys; `fragment_tolerance_da` key has been removed; key values have correct types; default score function is xcorr; default mz_bin_width is high-res; default fixed_mods does not contain carbamidomethyl -`TestFlatten` | flat dict unchanged; nested dict flattened; deeply nested; mixed depth; empty dict; default config flattens without error -`TestConfigCheck` | returns `True`/`False` correctly for exists/absent file under both `exists=True` and `exists=False` modes -`TestWriteLoadConfig` | round-trip preserves content; raises `FileNotFoundError` when no config present -`TestApplyMod` | adds mod to empty spec; adds to existing spec; prepends mod; duplicate entry not added; removal with exclusive pattern; exclusive pattern replaces on add; no leading/trailing commas; no double commas; empty mod with no pattern is no-op; removal of absent mod is no-op -`TestApplyIodo` | adds carbamidomethyl to empty and non-empty `fixed_mods` string; prepends; removes carbamidomethyl; no-op when not present; idempotent; result has no count prefix; no leading/trailing commas; no double commas -`TestApplyCustom` | adds entry to empty string; adds to existing; empty string clears all; duplicate not added; managed Met/Cys/phos mods rejected with warning; unmanaged entry accepted; no leading/trailing commas -`TestApplyOrganism` | sets organism section; replaces existing; does not touch other sections; empty dict clears; returns cfg -`TestApplyProtocolFlags` | iodo/low_res/mbr None leaves keys unchanged; low_res True/False sets bin width and score function; combined iodo+low_res; only relevant keys touched; non-search sections untouched -`TestApplyProtocolFlagsMods` | iodo True/False writes to `fixed_mods`, not `mods_spec`; ox/phos/n_cyc/n_ace True adds correct mod to correct key; False removes; None is no-op; n_cyc/n_ace do not touch mods_spec; ox and iodo coexist across different keys; iodo does not remove ox; all flags None changes nothing -`TestParseOrganismArg` | single and multiple pairs; strips whitespace; preserves regex chars; raises `SystemExit` on no `=`, empty key, empty pattern; returns dict; empty list returns empty dict; `=` in pattern preserved -`TestConfigInit` | creates config file; file is valid TOML; does not overwrite existing -`TestConfigExists` | exits nonzero when absent; does not raise when present -`TestConfigVerify` | valid config passes; missing key exits nonzero; exits nonzero when no config -`TestConfigReset` | `--force` restores defaults; without force prompts; confirms and resets on accept -`TestConfigSet` | creates config if absent; created config is valid TOML; all named mod flags add/remove correct mod in correct key; idempotent for ox/phos/n_cyc/n_ace; n_cyc/n_ace do not change mods_spec; custom adds entry; custom is additive; custom empty string clears; custom managed mod not added; iodo flags unchanged from original tests; low_res/organism/mbr unchanged from original tests; combined flags work together; no-flags exits nonzero; all unrelated config keys unchanged after set -`TestResolveConfigTarget` | None resolves to the global user config path; "global"/"GLOBAL" resolve case-insensitively to the global path; any other string is returned verbatim as a Path -`TestConfigSetLocalTarget` | writes to the supplied local path and produces valid TOML; the global user config is left untouched when a local target is given +`TestFlatten` | flat dict unchanged; nested dict flattened; deeply nested; empty dict; the bundled default config flattens without error +`TestPrintTable` | renders without raising when current matches defaults; renders without raising when a value diverges from defaults +`TestPrintDiffSummary` | no-changes case prints a "No changes" message; a changed key shows both old and new values; multiple changes are all reported (one ✓ per change) +`TestResolveOrCreate` | `use_global=True` resolves to the global config path; `use_global=False` resolves to `/comms/config.toml`; creates a defaults-derived file if none exists; raises `SystemExit` when both a bare `config.toml` and a nested `comms/config.toml` are present (ambiguous); prefers the bare config when only that is present; prompts and creates the nested config when neither is present (accepted via `_confirm`); declining the prompt exits with code 0 +`TestConfigList` | prints the config path and a table (`config_list`) +`TestConfigVerify` | a config created from defaults passes; missing a required key exits non-zero; an unexpected/extra key exits non-zero +`TestConfigReset` | `--force` resets to bundled defaults without prompting; without force, prompts via `_confirm` and exits with code 0 only when declined +`TestConfigSet` | all-`None` flags returns `False` and writes nothing; a protocol flag (`iodo`) round-trips into `index.fixed_mods`; `organism` round-trips into the `organism` section; `custom` round-trips into `index.custom_mods`; direct flags (`gzip`, `threads`, `picked_protein`, `measure`, `lfc_threshold`) round-trip into their respective sections (parametrised); unrelated sections (`search`, `percolator`, `quantify`, `convert`) are untouched by an unrelated flag; a diff summary (✓) is printed after a successful set +`TestConfigSetLocalTarget` | writes to the local `/comms/config.toml`, not the global user config, which remains untouched + +--- +### `tests/unit/test_modspec.py` +Unit tests covering `src/comms/utils/modspec.py` — the modification-spec and protocol-flag logic previously tested as part of `test_config.py`, now in its own module alongside the mod-name constants (`CARBAMIDOMETHYL_MOD`, `MET_OX_MOD`, `PHOSPHO_MOD`, `NCYC_MOD`, `NACE_MOD`) and resolution constants (`MZ_BIN_WIDTH_HIGH_RES`/`LOW_RES`, `SCORE_FUNC_HIGH_RES`/`LOW_RES`): + +Class | Test description +-- | -- +`TestApplyMod` | adds mod to empty spec; adds to existing spec; prepends mod; duplicate entry not added; removal via exclusive pattern; exclusive pattern replaces on add; no leading/trailing commas; removal of an absent mod is a no-op +`TestApplyIodo` | adds carbamidomethyl when `iodo=True`; removes it when `iodo=False`; `iodo=False` on an empty spec adds `C+0`; idempotent (re-applying does not duplicate) +`TestApplyCustomMod` | adds entry to empty string; empty string clears all; duplicate not added; managed mods (Met ox, carbamidomethyl, STY phospho) are rejected and silently dropped (parametrised); `MANAGED_MOD_PATTERNS` is a `dict[str, str]` of pattern to flag name +`TestApplyOrganism` | sets the `organism` section; replaces an existing one; does not touch other sections +`TestParseOrganismArg` | single and multiple `key=value` pairs; strips whitespace; raises `SystemExit` on a missing `=`, empty key, or empty pattern; empty list returns an empty dict +`TestApplyProtocolFlags` | `low_res=True`/`False` sets `mz_bin_width` and `score_function` to the low-/high-res constants; `None` flags are a no-op; **new:** `clip_met=True`/`False` sets `index.clip_n_met` as a genuine bool (never a string) and `None` is a no-op; **new:** `missed_cleavages` sets `index.missed_cleavages` and `None` is a no-op; `clip_met` and `missed_cleavages` coexist and don't disturb mod-flag handling (`ox` still writes to `mods_spec`) --- ### `tests/unit/test_context.py` -Unit tests covering `src/comms/utils/context.py`. +Unit tests covering `src/comms/utils/context.py`: Class | Test description -- | -- `TestNormaliseDirs` | a plain root returns `(root, root/comms)`; a path whose final component is `comms` returns `(parent, comms)`; a directory containing `experiment.toml` directly is treated as a `comms` directory and returns `(parent, dir)` `TestResolve` | with no `experiment.toml` present, config falls back to the bundled default, `bin_dir` is `None`, and root equals the input directory; a local `comms/config.toml` is preferred and `config_source` begins with `"local"`; a `bin_dir` set in `experiment.toml` is parsed to a `Path` and exposed on the context -`TestExperimentContextProperties` | each stored-input `@property` (`data_files`, `database`, `sample_sheet`, `analysis_mode`, `multispecies`, `organism_prefix`, `ref_info`, `cont_csv`) returns the correct typed value when the corresponding `metadata` key is present, and `None` / empty list when absent; properties are **not** constructor parameters — they are read from `self.metadata` -`TestResultsDir` | returns the canonical `/comms/results/` path; the path is correct even when the directory does not yet exist -`TestChoose` | override is returned when given; a warning containing the label is logged when override differs from stored; no warning when override equals stored; stored is returned when no override; raises `SystemExit` when neither is supplied; raises `SystemExit` when resolved path is missing and `must_exist=True`; returns a non-existent path without raising when `must_exist=False` +`TestResolveResultsInput` | delegates to `results_dir` for the default (no-override) case; `must_exist=False` suppresses the existence check even though the directory doesn't exist; an explicit override wins over the computed path +`TestResolveReport` | `report_enabled=True` and no override means do-not-skip (`resolve_report` returns `False`); `report_enabled=False` means skip (`True`); `report_enabled=None` (unset) defaults to do-not-skip; an override agreeing with the context logs no warning; an override disagreeing with the context wins and logs a warning; the polarity is pinned explicitly — `resolve_report` returns *skip*, not *enabled* +`TestExperimentContextProperties` | each stored-input `@property` (`data_files`, `database`, `sample_sheet`, `analysis_mode`, `multispecies`, `organism_prefix`, `ref_info`, `cont_csv`, `report_enabled`) returns the correct typed value when the corresponding metadata key is present, and `None`/empty list when absent; `multispecies` falls back to checking whether `config['organism']` is non-empty when no `[experiment].analysis` mode is stored; properties are read from `self.metadata`, not constructor parameters +`TestResultsDir` | `results_dir(ctx, command)` returns the canonical `/comms/results/` path; correct even when the directory does not yet exist +`TestChoose` | `_choose` returns the override when given; logs a warning naming the label when override differs from stored; no warning when override equals stored; returns stored when no override; raises `SystemExit` when neither is supplied; raises `SystemExit` when the resolved path is missing and `must_exist=True`; returns a non-existent path without raising when `must_exist=False` `TestCheckFiles` | all-existing files returns a `list[Path]` equal to the inputs (not `True`); a missing file raises `SystemExit`; empty input returns an empty list (not `True`) `TestResolveDataFiles` | stored list returned when no override; override wins and a warning is logged when it differs from stored; raises `SystemExit` when neither stored nor override is present; raises `SystemExit` when any file is missing; result is a `list[Path]` -`TestResolveMzmlFiles` | explicit override list returned directly; raises when override file missing; globs `*.mzML` and `*.mzML.gz` from the convert results directory when no override; raises when no files found in convert directory -`TestResolveSingleFileInputs` | `resolve_database` and `resolve_sample_sheet` return stored value; override wins with warning; raises when neither supplied; raises when resolved file is missing -`TestResolveOrganismPrefix` | stored prefix returned; override wins with warning; raises when neither is available; return type is `str` +`TestResolveMzmlFiles` | explicit override list returned directly; raises when an override file is missing; globs `*.mzML` and `*.mzML.gz` from the convert results directory when no override; raises when no files found in the convert directory +`TestResolveSingleFileInputs` | `resolve_database` and `resolve_sample_sheet` return the stored value; override wins with a warning; raises when neither is supplied; raises when the resolved file is missing +`TestResolveOrganismPrefix` | stored prefix returned; override wins with a warning; raises when neither is available; return type is `str` --- @@ -286,7 +303,7 @@ Class | Test description -- | -- `TestLaunchExperimentGui` | exits with `run_app`'s return code; logs a launch message; `logMsg` instance is named `'experiment'` `TestMainWindowCloseLogging` | closing the window logs a message containing "closed"; `logMsg` instance is named `'experiment'` -`TestRunExperimentHeadless` | with `typer.prompt`/`typer.confirm` patched and a temporary `.mzML` file, writes `sample_sheet.tsv`, `config.toml` and `experiment.toml` under `/comms/`; the prompt sequence now includes the database FASTA prompt (between bin-dir and treatments) and an explicit data-file list via `_prompt_list('data file')` (between the input directory and per-file assignment); requires at least one treatment and one fraction (exits non-zero otherwise); records a `bin_dir` in `experiment.toml` only when one is supplied +`TestRunExperimentHeadless` | with `typer.prompt`/`typer.confirm` patched and a temporary `.mzML` file, writes `sample_sheet.tsv`, `config.toml` and `experiment.toml` under `/comms/`; the prompt sequence includes the combined database FASTA prompt (between bin-dir and treatments) and an explicit data-file list via `_prompt_list('data file')` (between the input directory and per-file assignment); records a `bin_dir` in `experiment.toml` only when one is supplied --- @@ -302,6 +319,18 @@ Class | Test description --- +### `tests/unit/test_installrdeps.py` +Unit tests covering `src/comms/utils/installrdeps.py` — checking, printing and installing the R package dependencies used by the `report` command. No R installation is required; `shutil.which`, `subprocess.run`/`Popen` are mocked throughout. + +Class | Test description +-- | -- +`TestCheckRDependencies` | returns `None` when `Rscript` is not on `PATH`; parses a well-formed JSON `{installed, missing}` payload from stdout; a non-zero return code returns `None`; malformed JSON returns `None` +`TestPrintDependencyTable` | only an "Installed" line is logged when nothing is missing; both "Installed" and "Missing" lines (plus the `r-utils install` hint) are logged when something is missing; nothing is logged when both lists are empty +`TestInstallRDependencies` | returns `False` when `Rscript` is not on `PATH`; streams `Popen` stdout lines to the log as they arrive and returns `True` on a zero exit code; a non-zero exit code returns `False` +`TestInstallRDependenciesTerminal` | nothing missing informs the user without prompting; something missing and the user confirms (`logMsg.input` returns `'y'`) calls `install_r_dependencies`; the user declining (`'n'`) does not call it; malformed check output logs an error without raising + +--- + ### `tests/unit/test_lfq.py` Unit tests covering `_groupPsmsByFraction` in `src/comms/commands/lfq.py`. No external binaries are required. @@ -311,6 +340,15 @@ Class | Test description --- +### `tests/unit/test_license.py` +Unit tests covering `src/comms/commands/license.py`: + +Class | Test description +-- | -- +`TestPrintLicense` | raises `SystemExit` with code 0; reads and pages the license file without raising (`pydoc.pager` is mocked and asserted called once) + +--- + ### `tests/unit/test_parammedic.py` Unit tests covering `_parseParamMedicOutput` and `_runParamMedic` in `src/comms/commands/search.py`. No external binaries are required; `cruxutil.paramMedic` is mocked throughout `TestRunParamMedic`. @@ -326,61 +364,85 @@ Unit tests covering `src/comms/utils/paths.py`: Class | Test description -- | -- -`TestGenerateOutputFileStructure` | creates the expected `comms/results//` subdirectory; creates directories if absent; returns existing path unchanged if already correct; works for all supported commands -`TestCheckUniqueFileName` | returns expected base name when no conflict; increments suffix on conflict; increments correctly through multiple conflicts; correct naming patterns for all commands (`search`, `quantify`, `rescore`, `report`); returned path is within `out_dir` +`TestGenerateOutputFileStructure` | creates the expected `comms/results//` subdirectory; creates directories if absent; returns existing path unchanged if already correct; works for `convert`, `index`, `search`, `rescore`, `quantify` (parametrised) +`TestCheckUniqueFileName` | returns the expected base name when there's no conflict; increments the numeric suffix on conflict, and again through multiple conflicts; `quantify` naming (`.spectral-counts.txt`); `rescore` naming (`.percolator.psms.txt`); `report` naming takes a `fmt` kwarg instead of `orig_name` (`comms-report.`); an unrecognised command still produces a usable fallback name (`comms--output...`) rather than raising; the returned path's parent is `out_dir` `TestRepoBinDir` | an explicit `experiment_bin_dir` takes precedence over everything and is returned unchanged; explicit value beats `COMMS_BIN_DIR` even when both are set; `COMMS_BIN_DIR` is used when no explicit value is given; falls back to repo-root `bin/` path when neither is set; `experiment_bin_dir=None` is equivalent to omitting the argument; returns a `Path` object --- -### `tests/unit/test_report.py` -Unit tests covering `src/comms/commands/report.py`: +### `tests/unit/test_readiness.py` +Unit tests covering `src/comms/utils/readiness.py` — the command-readiness gap calculator that backs the GUI readiness panel (and, indirectly, the CLI's pre-flight checks): Class | Test description -- | -- -`TestResolveRScript` | returns a `Path`; path ends with the requested script name; auxiliary scripts under `aux/` subdirectory are resolved correctly -`TestWriteIndex` | creates `index.md`; contains all section names; failed sections marked FAILED; passed sections marked ✓; parameters block is included -`TestRunReportValidation` | raises `SystemExit` when no spectral-counts files in quantify directory; raises `SystemExit` when output directory exists without `--overwrite`; raises `SystemExit` when Rscript binary is not on PATH; silently drops concordance when `--lfq-dir` absent; creates output directory; writes `index.md`; passes `lfc_threshold` and `fdr_threshold` as positional args to the `da` section; `logMsg` instance is named `'report'` +`TestMissingRequirements` | `missing_requirements(**all_true)` returns an empty gap list for every command in `COMMANDS`; a missing `has_data` flag surfaces "data files" only in `convert`/`search`/`lfq`, not `index`/`quantify`; a missing `has_crux` flag surfaces "Crux" in every Crux-dependent command (`index`, `search`, `rescore`, `lfq`, `quantify`) but not `convert`/`report`; a missing `has_trfp` flag surfaces "ThermoRawFileParser" only in `convert`; a missing `has_r_deps` flag surfaces "R dependencies" only in `report`; `pipeline`'s gap list is exactly the union of every other command's gaps; rescore requires "organism patterns" only when `multispecies=True` (pinned explicitly as a regression guard for the §1.2 assumption) -N.B. `_run_r_section` and `shutil.which` are mocked throughout — no R installation is required. +--- + +### `tests/unit/test_report.py` +Unit tests covering helper functions and `run_report` in `src/comms/commands/report.py`. This module now tracks and reports per-organism outcomes for each report section, not just a single pass/fail per section. + +Class | Test description +-- | -- +`TestResolveRScript` | returns a `Path`; path ends with the requested script name; auxiliary scripts under `aux/` (e.g. `aux/ev-markers.R`) are resolved correctly +`TestWriteIndex` | creates `index.md`; contains all section names; a failed section is marked `FAILED`; a passed section is marked with `✓`; a `partial` section (mixed per-organism outcomes) is marked `PARTIAL`; a `skipped` section is marked `SKIPPED`; per-organism outcome lines are nested under their parent section line in the rendered output; the parameters block is included +`TestRunReportValidation` | raises `SystemExit` when no spectral-counts files are in the quantify directory; raises `SystemExit` when the output directory exists without `--overwrite`; raises `SystemExit` when the `Rscript` binary is not on `PATH`; the `concordance` section is silently dropped when no `--lfq-dir` is available (falls back to the conventional `comms/results/lfq` location, and only drops the section if that's also absent); `ref_info` falls back to the value stored on the experiment context when not passed explicitly; creates the output directory and writes `index.md`; passes `lfc_threshold` and `fdr_threshold` as positional args (not kwargs) to the `da` section; `logMsg` instance is named `'report'`; a config override (e.g. `lfc_threshold`) writes a `report.config.toml` sidecar recording the overridden value, while an unmodified run (all overrides `None`) writes no sidecar +`TestReadStatus` | a missing `_status.json` returns `({}, {})`; malformed JSON returns `({}, {})`; a well-formed file returns its `organisms` and `reasons` dicts; unrecognised status values are dropped from the parsed `organisms` dict +`TestSectionStatus` | no organisms and `proc_ok=True` is `'skipped'`; no organisms and `proc_ok=False` is `'failed'`; all-`ok` organisms is `'succeeded'`; a mix of `ok`/`failed` is `'partial'`; all-`failed` is `'failed'`; all-`skipped` (none ok or failed) is `'skipped'` +`TestLogOrganismOutcomes` | logs one line per organism; includes the failure reason in parentheses when present; omits the empty parenthetical when there's no reason --- ### `tests/unit/test_rescore.py` -Unit tests covering helper functions in `src/comms/commands/rescore.py`. No external binaries are required; tests use synthetic PSM files written directly to `tmp_path`. +Unit tests covering helper functions in `src/comms/commands/rescore.py`. No external binaries are required; tests use synthetic PSM files written directly to `tmp_path`. Note the organism split now happens on **Tide-search** target/decoy output (`_splitPsmsByOrganism`, ahead of the per-organism Percolator round), not on already-rescored Percolator PSMs — the protein-ID column position is located dynamically via `_findProteinIdsIndex` rather than assumed to be the last column. Class | Test description -- | -- -`TestParseOrganismTags` | parses two-organism comma-separated string; parses single-organism string; strips internal and leading/trailing whitespace; preserves regex characters in values; raises `SystemExit` on odd item count, single item, or empty string; returns `dict[str, str]`; keys and values are strings -`TestClassifyPsmRow` | returns a `list` for matching rows; returns `['EUK']` for a matching EUK row; returns `['PRO']` for a matching PRO row; returns the string `'contaminants'` for an unmatched row; returns `'contaminants'` for an empty row; uses the last tab-delimited column as the protein ID; returns a list with multiple labels when the protein ID matches more than one organism tag -`TestSplitPsmsByOrganism` | returns `True` on success; creates per-organism target files in labelled subdirectories; creates per-organism decoy files; EUK file contains only EUK rows; PRO file contains only PRO rows; contaminant rows go to a `contaminants/` bucket; header is preserved in each output file; returns a bool without raising when the target file is missing; skips a missing decoy file gracefully and still succeeds for the target; output files are non-empty; with `shared_policy='drop'`, rows matching more than one organism are excluded from all output files; with `shared_policy='include'`, rows matching more than one organism appear in all matching output files +`TestParseOrganismTags` | parses two-organism comma-separated string; parses single-organism string; strips internal and leading/trailing whitespace; preserves regex characters in values; raises `SystemExit` on odd item count, single item, or empty string; returns `dict[str, str]` +`TestClassifyPsmRow` | `_classifyPsmRow(row, id_index, organism_tags)` returns a `list` for matching rows; returns `['EUK']`/`['PRO']` for matching rows; returns `['contaminants']` for an unmatched or empty row; uses the column at the supplied `id_index` for the protein ID (rather than assuming the last column); returns a list with multiple labels when the protein ID matches more than one organism tag +`TestFindProteinIdsIndex` | `_findProteinIdsIndex(header, 'protein id')` returns the correct 0-based column index for a mid-row column; returns `0` when it's the first column +`TestSplitPsmsByOrganism` | operates on a combined Tide-search target/decoy file pair; returns `True` on success; creates per-organism target and decoy files under labelled subdirectories, named `.