From eb70a2db1d1733e62183b85e4d8ff2780df9a557 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Thu, 16 Jul 2026 15:19:10 +0200 Subject: [PATCH 1/9] feat: extend section tsv parser to section sep val parser for csv and others --- src/tsoppy/general/file_parser.py | 89 +++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/src/tsoppy/general/file_parser.py b/src/tsoppy/general/file_parser.py index cdf3749..b40b6e2 100644 --- a/src/tsoppy/general/file_parser.py +++ b/src/tsoppy/general/file_parser.py @@ -7,12 +7,58 @@ logger = logging.getLogger(__name__) +class sectionIdx: + empty = False + length = 0 + start = 0 + + def __init__(self, name: str, start: int, length: int = 0): + self.name = name + self.start = start + self.length = length + + def __eq__(self, other): + if not isinstance(other, sectionIdx): + return False + attr_to_compare = ["empty", "length", "name", "start"] + return all(vars(self).get(k) == vars(other).get(k) for k in attr_to_compare) + + def finalize(self): + self.set_length(self.length) + return self + + def included(self, idx_list: list[sectionIdx]) -> bool: + # Ignore empty sections + if self.name == "": + return True + return any(item.name == self.name for item in idx_list) + + def set_length(self, length: int): + if length == 0: + self.empty = True + self.length = length + + def Parse_section_tsv( path: str, key_value_sections: list[str] ) -> tuple[list[str], dict[str, polars.DataFrame]]: """Parse a sectioned TSV file into headers and a mapping of section names to DataFrames.""" + return _parse_section_sep_val(path, key_value_sections, "\t") + + +def Parse_section_csv( + path: str, key_value_sections: list[str] +) -> tuple[list[str], dict[str, polars.DataFrame]]: + """Parse a sectioned CSV file into headers and a mapping of section names to DataFrames.""" + return _parse_section_sep_val(path, key_value_sections, ",") + + +def _parse_section_sep_val( + path: str, key_value_sections: list[str], sep: str +) -> tuple[list[str], dict[str, polars.DataFrame]]: + """Parse a sectioned file containing separated values (TSV, CSV, etc.) into headers and a mapping of section names to DataFrames.""" try: - df = polars.read_csv(path, separator="\t", has_header=False) + df = polars.read_csv(path, separator=sep, has_header=False) except FileNotFoundError: logger.error(f"File {path} not found.") raise @@ -21,57 +67,56 @@ def Parse_section_tsv( raise section_idx = _get_section_idx(df) headers = [] - if section_idx[0][1] != 1: - headers = _parse_headers(df, section_idx[0][1] - 1) + if section_idx[0].start != 1: + headers = _parse_headers(df, section_idx[0].start - 1) section_dfs = {} for section in section_idx: # create slice of dataframe for the section - df_slice = df.slice(section[1], section[2]) + if section.empty: + section_dfs[section.name] = polars.DataFrame() + continue + df_slice = df.slice(section.start, section.length) # check if row contains null values if any(item is None for item in df_slice.row(0)): df_slice = _handle_row_with_nulls(df_slice) # check if section is a key value section - if section[0] in key_value_sections: + if section.name in key_value_sections: # check that the section only contains two columns and transpose else log a warning if df_slice.width == 2: df_slice = df_slice.transpose() else: logger.warning( - f"Section {section[0]} is supposed to be a key value section but contains more than two columns." + f"Section {section.name} is supposed to be a key value section but contains more than two columns." ) # assume first row contains column names df_header = df_slice.head(1).to_dicts().pop() # remove first row and rename columns and link it to the section name - section_dfs[section[0]] = df_slice.rename(df_header).slice(1) + section_dfs[section.name] = df_slice.rename(df_header).slice(1) return headers, section_dfs -def _get_section_idx(df: polars.DataFrame) -> list[tuple[str, int, int]]: +def _get_section_idx(df: polars.DataFrame) -> list[tuple[sectionIdx]]: """Get the the name, start index and length of each section in the DataFrame.""" - section = "" - section_start = 0 - section_length = 0 + section = sectionIdx("", 0) section_idx = [] for row in df.with_row_index().iter_rows(): if row[1]: match = re.search(r"^\[(?P
.*)\]$", row[1]) if match: - section_start = row[0] + 1 - section = match.group("section") + if not section.included(section_idx): + section_idx.append(section.finalize()) + section = sectionIdx(match.group("section"), row[0] + 1) + continue + else: + section.set_length(row[0] - section.start + 1) if all(item is None for item in row[1:]): - section_length = row[0] - section_start - if section_start > 0 and section_length > 0: - section_idx.append((section, section_start, section_length)) - section_start = 0 - section_length = 0 - if row[0] == len(df) - 1: - section_length = row[0] - section_start + 1 - if section_start > 0 and section_length > 0: - section_idx.append((section, section_start, section_length)) + continue + if not section.included(section_idx): + section_idx.append(section.finalize()) return section_idx From f77ccd2d43657ef2356f68b27172897154288651 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Thu, 16 Jul 2026 15:19:36 +0200 Subject: [PATCH 2/9] test: add unit tests for section sep val parser --- tests/general_file_parser_test.py | 101 ++++++++++++++---- .../empty.tsv | 0 .../empty_first_column_name.tsv | 0 .../parse_section_sep_val/empty_section.tsv | 5 + .../extra_empty_lines.tsv | 0 .../key_value.tsv | 0 .../multiple_sections.tsv | 0 .../no_headers.tsv | 0 .../null_columns.tsv | 0 .../parse_section_sep_val/standard_csv.tsv | 6 ++ .../standard_tsv.tsv} | 0 11 files changed, 94 insertions(+), 18 deletions(-) rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/empty.tsv (100%) rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/empty_first_column_name.tsv (100%) create mode 100644 tests/test_data/general_file_parser/parse_section_sep_val/empty_section.tsv rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/extra_empty_lines.tsv (100%) rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/key_value.tsv (100%) rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/multiple_sections.tsv (100%) rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/no_headers.tsv (100%) rename tests/test_data/general_file_parser/{parse_section_tsv => parse_section_sep_val}/null_columns.tsv (100%) create mode 100644 tests/test_data/general_file_parser/parse_section_sep_val/standard_csv.tsv rename tests/test_data/general_file_parser/{parse_section_tsv/standard.tsv => parse_section_sep_val/standard_tsv.tsv} (100%) diff --git a/tests/general_file_parser_test.py b/tests/general_file_parser_test.py index 500d66e..08059e6 100644 --- a/tests/general_file_parser_test.py +++ b/tests/general_file_parser_test.py @@ -5,7 +5,8 @@ from pytest import mark, raises from tsoppy.general.file_parser import ( - Parse_section_tsv, + sectionIdx, + _parse_section_sep_val, _get_section_idx, _handle_row_with_nulls, _parse_headers, @@ -19,8 +20,29 @@ "inputs, exception, want", [ ( - # Standard case with one section and headers - (path.join(test_data_dir, "parse_section_tsv/standard.tsv"), []), + # Standard tsv case with one section and headers + ( + path.join(test_data_dir, "parse_section_sep_val/standard_tsv.tsv"), + [], + "\t", + ), + nullcontext(), + ( + ["header1"], + { + "section1": polars.DataFrame( + {"col1": ["value1", "value2"], "col2": ["value3", "value4"]} + ) + }, + ), + ), + ( + # Standard csv case with one section and headers + ( + path.join(test_data_dir, "parse_section_sep_val/standard_csv.tsv"), + [], + ",", + ), nullcontext(), ( ["header1"], @@ -33,7 +55,11 @@ ), ( # Standard case with multiple sections and headers - (path.join(test_data_dir, "parse_section_tsv/multiple_sections.tsv"), []), + ( + path.join(test_data_dir, "parse_section_sep_val/multiple_sections.tsv"), + [], + "\t", + ), nullcontext(), ( ["header1"], @@ -49,7 +75,11 @@ ), ( # No headers - (path.join(test_data_dir, "parse_section_tsv/no_headers.tsv"), []), + ( + path.join(test_data_dir, "parse_section_sep_val/no_headers.tsv"), + [], + "\t", + ), nullcontext(), ( [], @@ -60,9 +90,31 @@ }, ), ), + ( + # Empty section + ( + path.join(test_data_dir, "parse_section_sep_val/empty_section.tsv"), + [], + "\t", + ), + nullcontext(), + ( + [], + { + "empty": polars.DataFrame(), + "section1": polars.DataFrame( + {"col1": ["value1", "value2"], "col2": ["value3", "value4"]} + ), + }, + ), + ), ( # Extra empty lines between sections and headers - (path.join(test_data_dir, "parse_section_tsv/extra_empty_lines.tsv"), []), + ( + path.join(test_data_dir, "parse_section_sep_val/extra_empty_lines.tsv"), + [], + "\t", + ), nullcontext(), ( ["header1"], @@ -75,7 +127,11 @@ ), ( # Columns containing only null values - (path.join(test_data_dir, "parse_section_tsv/null_columns.tsv"), []), + ( + path.join(test_data_dir, "parse_section_sep_val/null_columns.tsv"), + [], + "\t", + ), nullcontext(), ( ["header1"], @@ -90,9 +146,10 @@ # Empty first column name ( path.join( - test_data_dir, "parse_section_tsv/empty_first_column_name.tsv" + test_data_dir, "parse_section_sep_val/empty_first_column_name.tsv" ), [], + "\t", ), nullcontext(), ( @@ -106,7 +163,11 @@ ), ( # Key-value pairs instead of tabular data - (path.join(test_data_dir, "parse_section_tsv/key_value.tsv"), ["section1"]), + ( + path.join(test_data_dir, "parse_section_sep_val/key_value.tsv"), + ["section1"], + "\t", + ), nullcontext(), ( ["header1"], @@ -119,13 +180,17 @@ ), ( # Non-existent file - (path.join(test_data_dir, "parse_section_tsv/non-existent.tsv"), []), + ( + path.join(test_data_dir, "parse_section_sep_val/non-existent.tsv"), + [], + "\t", + ), raises(FileNotFoundError), ([], {}), ), ( # Empty file - (path.join(test_data_dir, "parse_section_tsv/empty.tsv"), []), + (path.join(test_data_dir, "parse_section_sep_val/empty.tsv"), [], "\t"), raises(polars.exceptions.NoDataError), ([], {}), ), @@ -133,7 +198,7 @@ ) def test_parse_section_tsv(inputs, exception, want): with exception: - got = Parse_section_tsv(inputs[0], inputs[1]) + got = _parse_section_sep_val(inputs[0], inputs[1], inputs[2]) assert got[0] == want[0] for key in want[1].keys(): assert key in got[1] @@ -151,7 +216,7 @@ def test_parse_section_tsv(inputs, exception, want): "col2": [None, "col2", "value2"], } ), - [("section1", 1, 2)], + [sectionIdx("section1", 1, 2)], ), ( # Standard case with one section and headers @@ -161,7 +226,7 @@ def test_parse_section_tsv(inputs, exception, want): "col2": [None, None, None, "col2", "value2"], } ), - [("section1", 3, 2)], + [sectionIdx("section1", 3, 2)], ), ( # Extra empty lines prior the top section, no headers @@ -171,7 +236,7 @@ def test_parse_section_tsv(inputs, exception, want): "col2": [None, None, None, "col2", "value2"], } ), - [("section1", 3, 2)], + [sectionIdx("section1", 3, 2)], ), ( # Missing column name in one section @@ -181,7 +246,7 @@ def test_parse_section_tsv(inputs, exception, want): "col2": [None, None, None, "col2", "value2"], } ), - [("section1", 3, 2)], + [sectionIdx("section1", 3, 2)], ), ( # Multiple sections with extra empty lines @@ -211,7 +276,7 @@ def test_parse_section_tsv(inputs, exception, want): ], } ), - [("section1", 3, 2), ("section2", 7, 2)], + [sectionIdx("section1", 3, 2), sectionIdx("section2", 7, 2)], ), ( # Column only contains null values @@ -222,7 +287,7 @@ def test_parse_section_tsv(inputs, exception, want): "col3": [None, None, None], } ), - [("section1", 1, 2)], + [sectionIdx("section1", 1, 2)], ), ], ) diff --git a/tests/test_data/general_file_parser/parse_section_tsv/empty.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/empty.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/empty.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/empty.tsv diff --git a/tests/test_data/general_file_parser/parse_section_tsv/empty_first_column_name.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/empty_first_column_name.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/empty_first_column_name.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/empty_first_column_name.tsv diff --git a/tests/test_data/general_file_parser/parse_section_sep_val/empty_section.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/empty_section.tsv new file mode 100644 index 0000000..ec4c457 --- /dev/null +++ b/tests/test_data/general_file_parser/parse_section_sep_val/empty_section.tsv @@ -0,0 +1,5 @@ +[empty] +[section1] +col1 col2 +value1 value3 +value2 value4 diff --git a/tests/test_data/general_file_parser/parse_section_tsv/extra_empty_lines.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/extra_empty_lines.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/extra_empty_lines.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/extra_empty_lines.tsv diff --git a/tests/test_data/general_file_parser/parse_section_tsv/key_value.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/key_value.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/key_value.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/key_value.tsv diff --git a/tests/test_data/general_file_parser/parse_section_tsv/multiple_sections.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/multiple_sections.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/multiple_sections.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/multiple_sections.tsv diff --git a/tests/test_data/general_file_parser/parse_section_tsv/no_headers.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/no_headers.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/no_headers.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/no_headers.tsv diff --git a/tests/test_data/general_file_parser/parse_section_tsv/null_columns.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/null_columns.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/null_columns.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/null_columns.tsv diff --git a/tests/test_data/general_file_parser/parse_section_sep_val/standard_csv.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/standard_csv.tsv new file mode 100644 index 0000000..2aa30d2 --- /dev/null +++ b/tests/test_data/general_file_parser/parse_section_sep_val/standard_csv.tsv @@ -0,0 +1,6 @@ +header1, +, +[section1], +col1,col2 +value1,value3 +value2,value4 diff --git a/tests/test_data/general_file_parser/parse_section_tsv/standard.tsv b/tests/test_data/general_file_parser/parse_section_sep_val/standard_tsv.tsv similarity index 100% rename from tests/test_data/general_file_parser/parse_section_tsv/standard.tsv rename to tests/test_data/general_file_parser/parse_section_sep_val/standard_tsv.tsv From 3e8e1d9b8cea23797275bde7354ac5ed3f1eee69 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Fri, 17 Jul 2026 11:33:52 +0200 Subject: [PATCH 3/9] docs: include doc strings for sectionIdx class --- src/tsoppy/general/file_parser.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tsoppy/general/file_parser.py b/src/tsoppy/general/file_parser.py index b40b6e2..c0a943d 100644 --- a/src/tsoppy/general/file_parser.py +++ b/src/tsoppy/general/file_parser.py @@ -8,6 +8,14 @@ class sectionIdx: + """Section index class to save information about section + + Attributes: + empty: Empty state of section (bool) + length: Length of section in rows (int) + name: Section name (str) + start: Start row index of section (int) + """ empty = False length = 0 start = 0 @@ -135,7 +143,8 @@ def _handle_row_with_nulls(df: polars.DataFrame) -> polars.DataFrame: # remove any columns that are completely null (no column header nor values) df = df.select( - [polars.col(col) for col in df.columns if not df[col].null_count() == df.height] + [polars.col(col) + for col in df.columns if not df[col].null_count() == df.height] ) # avoid null values by filling with "-" From 5dd60b7a7f5b4706aca40cb4aeea713e97e46918 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Wed, 22 Jul 2026 15:13:36 +0200 Subject: [PATCH 4/9] feat: add samplesheet to WorkflowOutput and add sample check method and meta data --- src/tsoppy/general/classes.py | 255 +++++++++++++++++++++++++++++----- 1 file changed, 224 insertions(+), 31 deletions(-) diff --git a/src/tsoppy/general/classes.py b/src/tsoppy/general/classes.py index 6ee639e..3e3a1ca 100644 --- a/src/tsoppy/general/classes.py +++ b/src/tsoppy/general/classes.py @@ -1,36 +1,113 @@ import gzip import logging import os +import re from pathlib import Path import cyvcf2 import msgspec import polars -from tsoppy.general.file_parser import Parse_section_tsv +from tsoppy.general.file_parser import Parse_section_csv, Parse_section_tsv # Use logger that was set up in CLI logger = logging.getLogger(__name__) class WorkflowConfig(msgspec.Struct): - """Config class for workflow output file path format strings""" + """Config class for workflow output file path format strings. + + Attributes: + metrics_output_tsv: Paths to metrics output tsv (dict[str, str]) + samplesheet: Paths to samplesheet (dict[str, str]) + small_variant_genome_vcf: Paths to small variant genome vcf (dict[str, str]) + tmb_trace_tsv: Paths to tmb trace tsv (dict[str, str]) + variants_annotated_json: Paths to variants annotated json (dict[str, str]) + """ metrics_output_tsv: dict[str, str] + samplesheet: dict[str, str] small_variant_genome_vcf: dict[str, str] tmb_trace_tsv: dict[str, str] variants_annotated_json: dict[str, str] def __eq__(self, other): + """Assess if two instances of this class are equal.""" if not isinstance(other, WorkflowConfig): return False - if self.metrics_output_tsv != other.metrics_output_tsv: - return False - if self.small_variant_genome_vcf != other.small_variant_genome_vcf: + attr_to_compare = [ + "metrics_output_tsv", + "samplesheet", + "small_variant_genome_vcf", + "tmb_trace_tsv", + "variants_annotated_json", + ] + return all(vars(self).get(k) == vars(other).get(k) for k in attr_to_compare) + + +class InPredNomenclatureCode(msgspec.Struct): + """Class for inpred nomenclature code. + + Attributes: + code: Letters representing the nomenclature code (str) + choices: Possible choices for nomenclature code (dict[str, str]) + """ + + code: str + choices: dict[str, str] + + def __eq__(self, other): + """Assess if two instances of this class are equal.""" + if not isinstance(other, InPredNomenclatureCode): return False - if self.tmb_trace_tsv != other.tmb_trace_tsv: + attr_to_compare = ["code", "choices"] + return all(vars(self).get(k) == vars(other).get(k) for k in attr_to_compare) + + +class InPredNomenclature(msgspec.Struct): + """Class for inpred nomenclature. + + Attributes: + format: sample id format (str) + project_code: Code representing project (InPredNomenclatureCode) + patient_code: Code representing patient (dict[str, str]) + nucleic_acid_input_type_code: Code representing nucleic acid input type (InPredNomenclatureCode) + assay_type_code: Code representing assay type (InPredNomenclatureCode) + sample_type_code: Code representing sample type (InPredNomenclatureCode) + library_preparation_attempt: Code representing library preparation (InPredNomenclatureCode) + biological_replicate: Code representing biological replicate (dict[str, str]) + sample_material: Code representing sample material (InPredNomenclatureCode) + tumor_site: Code representing tumor site (InPredNomenclatureCode) + """ + + format: str + project_code: InPredNomenclatureCode + patient_code: dict[str, str] + nucleic_acid_input_type_code: InPredNomenclatureCode + assay_type_code: InPredNomenclatureCode + sample_type_code: InPredNomenclatureCode + library_preparation_attempt: InPredNomenclatureCode + biological_replicate: dict[str, str] + sample_material: InPredNomenclatureCode + tumor_site: InPredNomenclatureCode + + def __eq__(self, other): + """Assess if two instances of this class are equal.""" + if not isinstance(other, InPredNomenclature): return False - return self.variants_annotated_json == other.variants_annotated_json + attr_to_compare = [ + "format", + "project_code", + "patient_code", + "nucleic_acid_input_type_code", + "assay_type_code", + "sample_type_code", + "library_preparation_attempt", + "biological_replicate", + "sample_material", + "tumor_site", + ] + return all(vars(self).get(k) == vars(other).get(k) for k in attr_to_compare) class WorkflowOutput: @@ -38,18 +115,43 @@ class WorkflowOutput: Attributes: config: Configuration (WorkflowConfig) + nomenclature: InPred nomenclature (InPredNomenclature) root: Root path (Path) + samples: Data section of samplesheet (polars.DataFrame) workflow_type: Detected workflow type (str) workflow_version: Detected workflow version (str) """ - def __init__(self, config_yaml: str | Path, root_path: str | Path): + sample_id_rex = r"^(?P\w{3})(?P\d{4})-(?P\w)(?P\d{2})-(?P\w)(?P\d)(?P\d)-(?P\w)(?P\d{2})" + + def __init__( + self, + config_yaml: str | Path, + inpred_nomenclature: str | Path, + root_path: str | Path, + ): """Initialize WorkflowOutput.""" self.root = Path(root_path) with open(config_yaml, "r") as yaml_file: self.config = msgspec.yaml.decode(yaml_file.read(), type=WorkflowConfig) + with open(inpred_nomenclature, "r") as yaml_file: + self.nomenclature = msgspec.yaml.decode( + yaml_file.read(), type=InPredNomenclature + ) self._detect_type_and_version() + self._parse_samples() + + def __eq__(self, other): + """Assess if two instances of this class are equal.""" + if not isinstance(other, WorkflowOutput): + return False + if self.config != other.config: + return False + if self.samples.equals(other.samples): + return False + attr_to_compare = ["root", "workflow_type", "workflow_version"] + return all(vars(self).get(k) == vars(other).get(k) for k in attr_to_compare) def _detect_type_and_version(self): """Detect which workflow type and version is present based on information in MetricsOutput.tsv.""" @@ -57,9 +159,10 @@ def _detect_type_and_version(self): # Get all values for MetricsOutput.tsv paths and check if they are the same info_src = list(self.config.metrics_output_tsv.values()) if len(set(info_src)) != 1: - raise ValueError( - f"Got {info_src} but need exactly one file to detect workflow id" + logger.error( + f"Got {info_src} but need exactly one file to detect workflow id." ) + raise ValueError # Parse MetricsOutput.tsv headers, sections = Parse_section_tsv( @@ -77,18 +180,86 @@ def _detect_type_and_version(self): row=0, column="Workflow Version" ) - def __eq__(self, other): - if not isinstance(other, WorkflowOutput): - return False - if self.config != other.config: - return False - if self.root != other.root: - return False - if self.workflow_type != other.workflow_type: - return False - return self.workflow_version == other.workflow_version + def _parse_samples(self): + """Parse samples from samplesheet.""" + samplesheet_path = os.path.join( + self.root, self.config.samplesheet[self.workflow_id] + ) + + # Parse SampleSheet.csv + _, sections = Parse_section_csv(samplesheet_path, []) + + # Find and check that there is only one Data section present + data_sections = [section for section in list(sections) if "Data" in section] + if len(data_sections) != 1: + logger.error( + f"Expected one data section in samplesheet {samplesheet_path} - got {len(data_sections)}." + ) + raise KeyError + + # Ensure that the table contains a Sample_ID column + if "Sample_ID" not in sections[data_sections[0]]: + logger.error( + f"Sample_ID column is missing from samplesheet {samplesheet_path}." + ) + raise KeyError - def workflow_id(self): + # Store the complete table + self.samples = sections[data_sections[0]] + + def _translate_code(self, code: str, choice: str) -> str: + code_dict = getattr(self.nomenclature, code) + if not hasattr(code_dict, "choices"): + return choice + else: + choices = list(code_dict.choices.keys()) + if choice not in choices: + logger.warning(f"{choice} is not valid - expected any: {choices}") + return choice + else: + return code_dict.choices[choice] + + def sample_exists(self, sample_id: str) -> bool: + """Check if sample exists.""" + return sample_id in self.sample_list + + @property + def sample_list(self) -> list[str]: + """Return all samples present in samplsheet.""" + return self.samples["Sample_ID"].to_list() + + def sample_meta(self, sample_id: str) -> dict[str:str]: + """Parse meta information from sample id and return as dict""" + match = re.search(self.sample_id_rex, sample_id) + if not match: + logger.warning( + "Sample {sample_id} does not follow the InPreD sample ID nomenclature." + ) + return None + return { + "project": self._translate_code("project_code", match.group("project")), + "patient": self._translate_code("patient_code", match.group("patient")), + "nucleic_acid_input_type": self._translate_code( + "nucleic_acid_input_type_code", match.group("input") + ), + "assay_type": self._translate_code("assay_type_code", match.group("assay")), + "sample_type": self._translate_code( + "sample_type_code", match.group("sample") + ), + "library_preparation_attempt": self._translate_code( + "library_preparation_attempt", match.group("preparation") + ), + "biological_replicate": self._translate_code( + "biological_replicate", match.group("replicate") + ), + "sample_material": self._translate_code( + "sample_material", match.group("material") + ), + "tumor_site": self._translate_code("tumor_site", match.group("tumor")), + } + + @property + def workflow_id(self) -> str: """Return combined string for workflow type and version.""" return f"{self.workflow_type}_{self.workflow_version}" @@ -102,9 +273,15 @@ class SmallVariantGenomeVcf(WorkflowOutput): vcf: Parsed VCF object (cyvcf2.VCF) """ - def __init__(self, config_yaml: str | Path, root_path: str | Path, sample_id: str): + def __init__( + self, + config_yaml: str | Path, + inpred_nomenclature: str | Path, + root_path: str | Path, + sample_id: str, + ): """Initialize SmallVariantGenomeVcf""" - super().__init__(config_yaml, root_path) + super().__init__(config_yaml, inpred_nomenclature, root_path) self.sample_id = sample_id self._parse() @@ -115,11 +292,15 @@ def create(cls, workflow_output: WorkflowOutput, sample_id: str): obj.__dict__.update(workflow_output.__dict__) obj.sample_id = sample_id obj._parse() - return obj + if not workflow_output.sample_exists(sample_id): + logger.error(f"Sample {sample_id} does not exist") + raise ValueError + else: + return obj def _parse(self): """Parse the small variant genome VCF file""" - fmt = self.config.small_variant_genome_vcf[self.workflow_id()] + fmt = self.config.small_variant_genome_vcf[self.workflow_id] self.path = Path( os.path.join(self.root, fmt.format(self.sample_id, self.sample_id)) ) @@ -140,9 +321,15 @@ class TmbTraceTsv(WorkflowOutput): sample_id: Sample identifier (str) """ - def __init__(self, config_yaml: str | Path, root_path: str | Path, sample_id: str): + def __init__( + self, + config_yaml: str | Path, + inpred_nomenclature: str | Path, + root_path: str | Path, + sample_id: str, + ): """Initialize TmTraceTsv.""" - super().__init__(config_yaml, root_path) + super().__init__(config_yaml, inpred_nomenclature, root_path) self.sample_id = sample_id self._parse() @@ -157,7 +344,7 @@ def create(cls, workflow_output: WorkflowOutput, sample_id: str): def _parse(self): """Parse the TMB trace tsv.""" - fmt = self.config.tmb_trace_tsv[self.workflow_id()] + fmt = self.config.tmb_trace_tsv[self.workflow_id] self.path = Path( os.path.join(self.root, fmt.format(self.sample_id, self.sample_id)) ) @@ -178,9 +365,15 @@ class VariantsAnnotatedJson(WorkflowOutput): sample_id: Sample identifier (str) """ - def __init__(self, config_yaml: str | Path, root_path: str | Path, sample_id: str): + def __init__( + self, + config_yaml: str | Path, + inpred_nomenclature: str | Path, + root_path: str | Path, + sample_id: str, + ): """Initialize VariantsAnnotatedJson.""" - super().__init__(config_yaml, root_path) + super().__init__(config_yaml, inpred_nomenclature, root_path) self.sample_id = sample_id self._parse() @@ -195,7 +388,7 @@ def create(cls, workflow_output: WorkflowOutput, sample_id: str): def _parse(self): """Parse the variants annotated JSON file""" - fmt = self.config.variants_annotated_json[self.workflow_id()] + fmt = self.config.variants_annotated_json[self.workflow_id] self.path = Path( os.path.join(self.root, fmt.format(self.sample_id, self.sample_id)) ) From 06e65d3931badc46225d62679ddaac4e4a06e994 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Wed, 22 Jul 2026 15:14:22 +0200 Subject: [PATCH 5/9] test: add tests and test data for workflow output extension --- tests/general_classes_test.py | 142 +++++++++++++++--- .../SampleSheet_Intermediate.csv | 3 + .../260210_123411_SampleSheet.csv | 3 + .../general_classes/nomenclature.yaml | 107 +++++++++++++ 4 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 tests/test_data/general_classes/dragen/standard/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv create mode 100644 tests/test_data/general_classes/localapp/standard/Logs_Intermediates/SamplesheetValidation/260210_123411_SampleSheet.csv create mode 100644 tests/test_data/general_classes/nomenclature.yaml diff --git a/tests/general_classes_test.py b/tests/general_classes_test.py index bbe14fb..9724705 100644 --- a/tests/general_classes_test.py +++ b/tests/general_classes_test.py @@ -20,33 +20,89 @@ "inputs, exception, want", [ ( - ("config.yaml", path.join(test_data_dir, "dragen/standard")), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/standard"), + ), nullcontext(), - "dragen_2.6.2.4", + ("dragen_2.6.2.4", "sample1"), ), ( - ("config.yaml", path.join(test_data_dir, "localapp/standard")), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "localapp/standard"), + ), nullcontext(), - "localapp_ruo-2.2.0.12", + ("localapp_ruo-2.2.0.12", "sample1"), ), ( - ("config.yaml", path.join(test_data_dir, "localapp/non-existent")), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "localapp/non-existent"), + ), raises(FileNotFoundError), - "", + ("", ""), ), ], ) def test_workflowoutput_init(inputs, exception, want): with exception: - got = WorkflowOutput(inputs[0], inputs[1]) - assert got.workflow_id() == want + got = WorkflowOutput(inputs[0], inputs[1], inputs[2]) + assert got.workflow_id() == want[0] + assert got.sample_exists(want[1]) + + +@mark.parametrize( + "inputs, want", + [ + ( + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/standard"), + "IPH0001-D01-P01-A00", + ), + { + "project": "InPreD HUS", + "patient": "0001", + "nucleic_acid_input_type": "DNA", + "assay_type": "TSO500 DNA", + "sample_type": "Primary tumor, naive", + "library_preparation_attempt": "First try", + "biological_replicate": "1", + "sample_material": "Archived (FFPE)", + "tumor_site": "Cancer origo incerta", + }, + ), + ( + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/standard"), + "sample1", + ), + None, + ), + ], +) +def test_workflowoutput_sample_meta(inputs, want): + got = WorkflowOutput(inputs[0], inputs[1], inputs[2]) + assert got.sample_meta(inputs[3]) == want @mark.parametrize( "inputs, exception, want", [ ( - ("config.yaml", path.join(test_data_dir, "dragen/standard"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/standard"), + "sample1", + ), nullcontext(), ( "chr1", @@ -61,7 +117,12 @@ def test_workflowoutput_init(inputs, exception, want): ), ), ( - ("config.yaml", path.join(test_data_dir, "localapp/standard"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "localapp/standard"), + "sample1", + ), nullcontext(), ( "chr1", @@ -76,7 +137,12 @@ def test_workflowoutput_init(inputs, exception, want): ), ), ( - ("config.yaml", path.join(test_data_dir, "dragen/non-existent"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/non-existent"), + "sample1", + ), raises(FileNotFoundError), ( None, @@ -94,8 +160,8 @@ def test_workflowoutput_init(inputs, exception, want): ) def test_smallvariantgenomevcf_create(inputs, exception, want): with exception: - workflow_output = WorkflowOutput(inputs[0], inputs[1]) - got = SmallVariantGenomeVcf.create(workflow_output, inputs[2]) + workflow_output = WorkflowOutput(inputs[0], inputs[1], inputs[2]) + got = SmallVariantGenomeVcf.create(workflow_output, inputs[3]) got_variants = list(got.vcf) assert len(got_variants) == 1 got_variant = got_variants[0] @@ -114,7 +180,12 @@ def test_smallvariantgenomevcf_create(inputs, exception, want): "inputs, exception, want", [ ( - ("config.yaml", path.join(test_data_dir, "dragen/standard"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/standard"), + "sample1", + ), nullcontext(), DataFrame( { @@ -149,7 +220,12 @@ def test_smallvariantgenomevcf_create(inputs, exception, want): ), ), ( - ("config.yaml", path.join(test_data_dir, "localapp/standard"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "localapp/standard"), + "sample1", + ), nullcontext(), DataFrame( { @@ -177,7 +253,12 @@ def test_smallvariantgenomevcf_create(inputs, exception, want): ), ), ( - ("config.yaml", path.join(test_data_dir, "dragen/non-existent"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/non-existent"), + "sample1", + ), raises(FileNotFoundError), None, ), @@ -185,8 +266,8 @@ def test_smallvariantgenomevcf_create(inputs, exception, want): ) def test_tmbtracetsv_create(inputs, exception, want): with exception: - workflow_output = WorkflowOutput(inputs[0], inputs[1]) - got = TmbTraceTsv.create(workflow_output, inputs[2]) + workflow_output = WorkflowOutput(inputs[0], inputs[1], inputs[2]) + got = TmbTraceTsv.create(workflow_output, inputs[3]) assert got.table.equals(want) @@ -194,17 +275,32 @@ def test_tmbtracetsv_create(inputs, exception, want): "inputs, exception, want", [ ( - ("config.yaml", path.join(test_data_dir, "dragen/standard"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/standard"), + "sample1", + ), nullcontext(), {"id": "sample1"}, ), ( - ("config.yaml", path.join(test_data_dir, "localapp/standard"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "localapp/standard"), + "sample1", + ), nullcontext(), {"id": "sample1"}, ), ( - ("config.yaml", path.join(test_data_dir, "dragen/non-existent"), "sample1"), + ( + "config.yaml", + path.join(test_data_dir, "nomenclature.yaml"), + path.join(test_data_dir, "dragen/non-existent"), + "sample1", + ), raises(FileNotFoundError), None, ), @@ -212,6 +308,6 @@ def test_tmbtracetsv_create(inputs, exception, want): ) def test_variantsannotatedjson_create(inputs, exception, want): with exception: - workflow_output = WorkflowOutput(inputs[0], inputs[1]) - got = VariantsAnnotatedJson.create(workflow_output, inputs[2]) + workflow_output = WorkflowOutput(inputs[0], inputs[1], inputs[2]) + got = VariantsAnnotatedJson.create(workflow_output, inputs[3]) assert got.data == want diff --git a/tests/test_data/general_classes/dragen/standard/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv b/tests/test_data/general_classes/dragen/standard/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv new file mode 100644 index 0000000..63ba9ef --- /dev/null +++ b/tests/test_data/general_classes/dragen/standard/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv @@ -0,0 +1,3 @@ +[TSO500S_Data],,,,, +Sample_ID,Index_ID,Sample_Type,Pair_ID,Sample_Feature,Sample_Description +sample1,IDX0001,DNA,sample1,HRD, \ No newline at end of file diff --git a/tests/test_data/general_classes/localapp/standard/Logs_Intermediates/SamplesheetValidation/260210_123411_SampleSheet.csv b/tests/test_data/general_classes/localapp/standard/Logs_Intermediates/SamplesheetValidation/260210_123411_SampleSheet.csv new file mode 100644 index 0000000..63ba9ef --- /dev/null +++ b/tests/test_data/general_classes/localapp/standard/Logs_Intermediates/SamplesheetValidation/260210_123411_SampleSheet.csv @@ -0,0 +1,3 @@ +[TSO500S_Data],,,,, +Sample_ID,Index_ID,Sample_Type,Pair_ID,Sample_Feature,Sample_Description +sample1,IDX0001,DNA,sample1,HRD, \ No newline at end of file diff --git a/tests/test_data/general_classes/nomenclature.yaml b/tests/test_data/general_classes/nomenclature.yaml new file mode 100644 index 0000000..202a495 --- /dev/null +++ b/tests/test_data/general_classes/nomenclature.yaml @@ -0,0 +1,107 @@ +format: "PPPyyyy-Ann-Sxz-Mll" +project_code: + code: PPP + choices: + IPA: "InPreD AHUS" + IPD: "InPreD OUS" + IPH: "InPreD HUS" + IPO: "InPreD St. Olav" +patient_code: + code: yyyy +nucleic_acid_input_type_code: + code: A + choices: + C: "Cell-free samples" + D: "DNA" + R: "RNA" +assay_type_code: + code: nn + choices: + "01": "TSO500 DNA" + "02": "Archer FusionPlex Lung assay" + "03": "TSO500 RNA" + "04": "EPIC DNA methylation" + "05": "Whole-genome DNA sequencing (WGS)" + "06": "TSO500 HRD (TSO500 solid + HRD)" + "07": "Whole-transcriptome RNA sequencing (WTS)" + "50": "[External] Twist Human Core Exome Plus (DNA)" + "51": "[External] TruSeq Stranded mRNA" +sample_type_code: + code: S + choices: + A: "Post Allotransplantation" + C: "Cell-line" + D: "Distal metastasis, naive" + d: "Distal metastasis, post-treatment" + E: "Naive" + e: "Post treatment" + L: "Liquid" + M: "Metastasis" + N: "Normal/control" + P: "Primary tumor, naive" + p: "Primary tumor, post-treatment" + R: "Regional metastasis, naive" + r: "Regional metastasis, post-treatment" + T: "Primary tumor" + X: "Unknown" +library_preparation_attempt: + code: x + choices: + "0": "First try" + "1": "Second try (e.g., after cleaning, new extraction)" + "2": "Third try" + "3": "Fourth try" + "4": "Fifth try" + "5": "Sixth try" + "6": "Seventh try" + "7": "Eighth try" + "8": "Nineth try" + "9": "Validation/verification test;" +biological_replicate: + code: z +sample_material: + code: M + choices: + A: "Archived (FFPE)" + B: "Blood" + C: "Cytology" + E: "Extramedullary" + F: "Fresh Frozen" + M: "Fresh bone Marrow" + S: "Buccal Swab" + X: "Unknown" +tumor_site: + code: ll + choices: + "00": "Cancer origo incerta" + "01": "Adrenal Gland" + "02": "Ampulla of Vater" + "03": "Biliary Tract" + "04": "Bladder/Urinary Tract" + "05": "Bone" + "06": "Breast" + "07": "Cervix" + "08": "CNS/Brain" + "09": "Colon/Rectum" + "10": "Esophagus/Stomach" + "11": "Eye" + "12": "Head and Neck" + "13": "Kidney" + "14": "Liver" + "15": "Lung" + "16": "Lymphoid" + "17": "Myeloid" + "18": "Ovary/Fallopian Tube" + "19": "Pancreas" + "20": "Peripheral Nervous System" + "21": "Peritoneum" + "22": "Pleura" + "23": "Prostate" + "24": "Skin" + "25": "Soft Tissue" + "26": "Testis" + "27": "Thymus" + "28": "Thyroid" + "29": "Uterus" + "30": "Vulva/Vagina" + "XX": "Unknown" \ No newline at end of file From 7c0a1b98afccbab891d4b830eeb815f44c49903e Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Wed, 22 Jul 2026 15:14:44 +0200 Subject: [PATCH 6/9] chore: add samplesheet paths to config file --- config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.yaml b/config.yaml index 3563fb1..d285e5b 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,9 @@ metrics_output_tsv: dragen_2.6.2.4: Logs_Intermediates/MetricsOutput/MetricsOutput.tsv localapp_ruo-2.2.0.12: Logs_Intermediates/MetricsOutput/MetricsOutput.tsv +samplesheet: + dragen_2.6.2.4: Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv + localapp_ruo-2.2.0.12: Logs_Intermediates/SamplesheetValidation/*_SampleSheet.csv small_variant_genome_vcf: dragen_2.6.2.4: Logs_Intermediates/DnaDragenCaller/{}/{}.hard-filtered.gvcf.gz localapp_ruo-2.2.0.12: Logs_Intermediates/VariantMatching/{}/{}_MergedSmallVariants.genome.vcf From 994f8b32bc4e36346c9a63727634a286906b8920 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Wed, 22 Jul 2026 15:15:12 +0200 Subject: [PATCH 7/9] chore: add rainbow csv extension --- .devcontainer/devcontainer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 033a256..8bd937e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -48,6 +48,7 @@ "gitlab.gitlab-workflow", "kahole.magit", "ms-vsliveshare.vsliveshare", + "mechatroner.rainbow-csv", "tamasfe.even-better-toml" ] } From b6ce05ceeaf3476f190aa268471fb5c6fcea87c2 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Wed, 22 Jul 2026 15:20:12 +0200 Subject: [PATCH 8/9] chore: ruff linting --- src/tsoppy/general/file_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsoppy/general/file_parser.py b/src/tsoppy/general/file_parser.py index c0a943d..880bd26 100644 --- a/src/tsoppy/general/file_parser.py +++ b/src/tsoppy/general/file_parser.py @@ -16,6 +16,7 @@ class sectionIdx: name: Section name (str) start: Start row index of section (int) """ + empty = False length = 0 start = 0 @@ -143,8 +144,7 @@ def _handle_row_with_nulls(df: polars.DataFrame) -> polars.DataFrame: # remove any columns that are completely null (no column header nor values) df = df.select( - [polars.col(col) - for col in df.columns if not df[col].null_count() == df.height] + [polars.col(col) for col in df.columns if not df[col].null_count() == df.height] ) # avoid null values by filling with "-" From 4938784d089f6191d409e831264009f4ddd68aa1 Mon Sep 17 00:00:00 2001 From: Martin Rippin Date: Wed, 22 Jul 2026 15:21:16 +0200 Subject: [PATCH 9/9] test: don't call property --- tests/general_classes_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/general_classes_test.py b/tests/general_classes_test.py index 9724705..c673bd1 100644 --- a/tests/general_classes_test.py +++ b/tests/general_classes_test.py @@ -51,7 +51,7 @@ def test_workflowoutput_init(inputs, exception, want): with exception: got = WorkflowOutput(inputs[0], inputs[1], inputs[2]) - assert got.workflow_id() == want[0] + assert got.workflow_id == want[0] assert got.sample_exists(want[1])