From 810791d0179cfb76f52211e6d278d1c59aca2e8e Mon Sep 17 00:00:00 2001 From: Martin Husbyn Date: Wed, 19 Aug 2026 14:39:17 +0100 Subject: [PATCH] feat(offline): add ext.offline fixture path for the two EBI-facing processes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SRA_IDS_TO_RUNINFO and SRA_FASTQ_FTP are the only processes that talk to EBI. When `ext.offline` is set they emit a pinned runinfo TSV and generate a small FASTQ locally instead; unset (the default) they behave exactly as before. This lets flow-api's sample-import acceptance test run the pipeline end to end without gating a merge on EBI being reachable from a single runner IP, while every transform step downstream — SRA_RUNINFO_TO_FTP, SRA_TO_SAMPLESHEET, MULTIQC_MAPPINGS_CONFIG — still runs for real, so the samplesheet the test asserts on is the one production emits. `ext` rather than a profile or a param because the caller layers one config over every pipeline in its environment: a `withName` selector that matches nothing is a silent no-op, whereas an unknown param would be rejected by any pipeline validating with nf-schema. `md5sum -c` still runs in the offline path. That makes the generated bytes load-bearing: `gzip -n` keeps them deterministic, and the fixtures' fastq_md5 column is repointed at the generated read. Every other column keeps ENA's real value. An accession with no fixture fails naming the accession, rather than emitting an empty TSV that would surface later as zero samples imported. No breaking changes; no nextflow_schema.json change. --- assets/offline/ERR10677146.runinfo.tsv | 2 + assets/offline/ERR1160845.runinfo.tsv | 2 + assets/offline/README.md | 27 +++++++++++ modules/local/sra_fastq_ftp/main.nf | 26 ++++++++++- modules/local/sra_ids_to_runinfo/main.nf | 28 +++++++++--- tests/offline_ids.csv | 3 ++ tests/offline_ids_no_fixture.csv | 2 + tests/sra_offline.config | 4 ++ tests/sra_offline.nf.test | 57 ++++++++++++++++++++++++ tests/sra_offline.nf.test.snap | 55 +++++++++++++++++++++++ 10 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 assets/offline/ERR10677146.runinfo.tsv create mode 100644 assets/offline/ERR1160845.runinfo.tsv create mode 100644 assets/offline/README.md create mode 100644 tests/offline_ids.csv create mode 100644 tests/offline_ids_no_fixture.csv create mode 100644 tests/sra_offline.config create mode 100644 tests/sra_offline.nf.test create mode 100644 tests/sra_offline.nf.test.snap diff --git a/assets/offline/ERR10677146.runinfo.tsv b/assets/offline/ERR10677146.runinfo.tsv new file mode 100644 index 00000000..158f3c16 --- /dev/null +++ b/assets/offline/ERR10677146.runinfo.tsv @@ -0,0 +1,2 @@ +run_accession experiment_accession sample_accession secondary_sample_accession study_accession secondary_study_accession submission_accession run_alias experiment_alias sample_alias study_alias library_layout library_selection library_source library_strategy library_name instrument_model instrument_platform base_count read_count tax_id scientific_name sample_title experiment_title study_title sample_description fastq_md5 fastq_bytes fastq_ftp fastq_galaxy fastq_aspera +ERR10677146 ERX10144910 SAMEA8947202 ERS6629975 PRJEB11419 ERP012803 ERA19529803 qiita_ppdid_118164:10317.THDMI.BLANK5.11B qiita_ptid_10903:10317.THDMI.BLANK5.11B qiita_sid_10317:10317.THDMI.BLANK5.11B qiita_sid_10317 PAIRED PCR METAGENOMIC WGS 10317.THDMI.BLANK5.11B Illumina NovaSeq 6000 ILLUMINA 146601 978 256318 metagenome 10317.THDMI.BLANK5.11B Illumina NovaSeq 6000 sequencing: qiita_ptid_10903:10317.THDMI.BLANK5.11B American Gut Project American Gut control 2e2014fade7e73ad0500b049472748c1;2e2014fade7e73ad0500b049472748c1 39628;41645 ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz fasp.sra.ebi.ac.uk:/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;fasp.sra.ebi.ac.uk:/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz diff --git a/assets/offline/ERR1160845.runinfo.tsv b/assets/offline/ERR1160845.runinfo.tsv new file mode 100644 index 00000000..ad2dd5b2 --- /dev/null +++ b/assets/offline/ERR1160845.runinfo.tsv @@ -0,0 +1,2 @@ +run_accession experiment_accession sample_accession secondary_sample_accession study_accession secondary_study_accession submission_accession run_alias experiment_alias sample_alias study_alias library_layout library_selection library_source library_strategy library_name instrument_model instrument_platform base_count read_count tax_id scientific_name sample_title experiment_title study_title sample_description fastq_md5 fastq_bytes fastq_ftp fastq_galaxy fastq_aspera +ERR1160845 ERX1234252 SAMEA3687213 ERS994362 PRJEB11419 ERP012803 ERA541392 qiita_ppdid_706:10317.BLANK.93.3D.r22 qiita_ptid_1263:10317.BLANK.93.3D.r22 qiita_sid_10317:10317.BLANK.93.3D.r22 qiita_sid_10317 SINGLE PCR METAGENOMIC AMPLICON 10317.BLANK.93.3D.r22 Illumina HiSeq 2500 ILLUMINA 4158 33 256318 metagenome 10317.BLANK.93.3D.r22 Illumina HiSeq 2500 sequencing: qiita_ptid_1263:10317.BLANK.93.3D.r22 American Gut Project American Gut control 2e2014fade7e73ad0500b049472748c1 2491 ftp.sra.ebi.ac.uk/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz ftp.sra.ebi.ac.uk/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz fasp.sra.ebi.ac.uk:/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz diff --git a/assets/offline/README.md b/assets/offline/README.md new file mode 100644 index 00000000..666e1d3b --- /dev/null +++ b/assets/offline/README.md @@ -0,0 +1,27 @@ +# Offline runinfo fixtures + +Pinned ENA responses used by `SRA_IDS_TO_RUNINFO` when `ext.offline` is set, so a run +can complete without reaching EBI: + +```groovy +process { + withName: '.*SRA_IDS_TO_RUNINFO.*' { ext.offline = true } + withName: '.*SRA_FASTQ_FTP.*' { ext.offline = true } +} +``` + +Each file is a real ENA response for one accession, captured with the module's default +`ENA_METADATA_FIELDS`, so the column set is exactly what `sra_runinfo_to_ftp.py` and +`SRA_TO_SAMPLESHEET` expect. Everything downstream of these two processes runs for real. + +`fastq_md5` is the one column that does not hold ENA's value. Offline `SRA_FASTQ_FTP` +generates a fixed one-read FASTQ instead of downloading, and still runs `md5sum -c`, so +`fastq_md5` has to be the md5 of those generated bytes: + +``` +printf '@offline\nACGT\n+\nIIII\n' | gzip -n | md5sum +``` + +Repeat the md5 once per FASTQ, semicolon-separated, `_1` then `_2` for paired runs. +`fastq_bytes`, `fastq_ftp` and `fastq_galaxy` keep ENA's real values — nothing reads them +once the download is bypassed, and they make the fixture's origin obvious. diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf index 0b077ab4..55c892cc 100644 --- a/modules/local/sra_fastq_ftp/main.nf +++ b/modules/local/sra_fastq_ftp/main.nf @@ -19,7 +19,31 @@ process SRA_FASTQ_FTP { script: def args = task.ext.args ?: '' - if (meta.single_end) { + def offline = task.ext.offline ?: false + if (offline) { + // gzip -n keeps the bytes deterministic so the fastq_md5 pinned in assets/offline still verifies + def generate_read = "printf '@offline\\nACGT\\n+\\nIIII\\n' | gzip -n" + if (meta.single_end) { + """ + $generate_read > ${meta.id}.fastq.gz + + echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 + md5sum -c ${meta.id}.fastq.gz.md5 + """ + } else { + """ + $generate_read > ${meta.id}_1.fastq.gz + + echo "${meta.md5_1} ${meta.id}_1.fastq.gz" > ${meta.id}_1.fastq.gz.md5 + md5sum -c ${meta.id}_1.fastq.gz.md5 + + $generate_read > ${meta.id}_2.fastq.gz + + echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 + md5sum -c ${meta.id}_2.fastq.gz.md5 + """ + } + } else if (meta.single_end) { """ wget \\ $args \\ diff --git a/modules/local/sra_ids_to_runinfo/main.nf b/modules/local/sra_ids_to_runinfo/main.nf index de5c3acc..146cd2ff 100644 --- a/modules/local/sra_ids_to_runinfo/main.nf +++ b/modules/local/sra_ids_to_runinfo/main.nf @@ -18,13 +18,27 @@ process SRA_IDS_TO_RUNINFO { script: def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : '' - """ - echo $id > id.txt - sra_ids_to_runinfo.py \\ - id.txt \\ - ${id}.runinfo.tsv \\ - $metadata_fields - """ + def offline = task.ext.offline ?: false + if (offline) { + def fixture = file("${projectDir}/assets/offline/${id}.runinfo.tsv") + if (!fixture.exists()) { + error("No offline runinfo fixture for accession '${id}' (looked for ${fixture})") + } + // Inlined rather than staged so the fixture reaches the task without projectDir being mounted + """ + cat <<'END_RUNINFO' > ${id}.runinfo.tsv +${fixture.text.trim()} +END_RUNINFO + """ + } else { + """ + echo $id > id.txt + sra_ids_to_runinfo.py \\ + id.txt \\ + ${id}.runinfo.tsv \\ + $metadata_fields + """ + } stub: """ diff --git a/tests/offline_ids.csv b/tests/offline_ids.csv new file mode 100644 index 00000000..9cfbe91b --- /dev/null +++ b/tests/offline_ids.csv @@ -0,0 +1,3 @@ +accession +ERR1160845 +ERR10677146 diff --git a/tests/offline_ids_no_fixture.csv b/tests/offline_ids_no_fixture.csv new file mode 100644 index 00000000..c65bca3f --- /dev/null +++ b/tests/offline_ids_no_fixture.csv @@ -0,0 +1,2 @@ +accession +ERR1160846 diff --git a/tests/sra_offline.config b/tests/sra_offline.config new file mode 100644 index 00000000..9e896d4c --- /dev/null +++ b/tests/sra_offline.config @@ -0,0 +1,4 @@ +process { + withName: '.*SRA_IDS_TO_RUNINFO.*' { ext.offline = true } + withName: '.*SRA_FASTQ_FTP.*' { ext.offline = true } +} diff --git a/tests/sra_offline.nf.test b/tests/sra_offline.nf.test new file mode 100644 index 00000000..1e5b8588 --- /dev/null +++ b/tests/sra_offline.nf.test @@ -0,0 +1,57 @@ +nextflow_pipeline { + + name "Test pipeline with ext.offline" + script "../main.nf" + tag "pipeline" + config "./sra_offline.config" + + test("-profile test --ext.offline") { + + when { + params { + outdir = "$outputDir" + input = "$projectDir/tests/offline_ids.csv" + } + } + + then { + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + def csv_file = path(params.outdir + '/samplesheet/samplesheet.csv').csv() + def ebi_calls = [] + new File(workDir).eachFileRecurse { f -> + if (f.name == '.command.sh' && (f.text.contains('sra_ids_to_runinfo.py') || f.text.contains('wget \\'))) { + ebi_calls << f.parentFile.parentFile.name + '/' + f.parentFile.name + } + } + + assert workflow.success + assertAll( + { assert csv_file.rowCount == 2 }, + { assert stable_name.count { it.toString().endsWith('.fastq.gz') } == 3 }, + { assert ebi_calls == [] }, + { assert snapshot( + removeFromYamlMap("${params.outdir}/pipeline_info/nf_core_fetchngs_software_versions.yml", "Workflow"), + stable_name, + stable_path, + "samplesheet.csv:md5," + csv_file.sort().table.collect { row -> [row.getString("sample"), row.getString("fastq_1").replaceAll(params.outdir, ""), row.getString("fastq_2").replaceAll(params.outdir, "")].join(",") }.join("\n").md5() + ).match() } + ) + } + } + + test("-profile test --ext.offline, accession with no fixture") { + + when { + params { + outdir = "$outputDir" + input = "$projectDir/tests/offline_ids_no_fixture.csv" + } + } + + then { + assert workflow.failed + assert workflow.stdout.join('\n').contains("No offline runinfo fixture for accession 'ERR1160846'") + } + } +} diff --git a/tests/sra_offline.nf.test.snap b/tests/sra_offline.nf.test.snap new file mode 100644 index 00000000..1c76749f --- /dev/null +++ b/tests/sra_offline.nf.test.snap @@ -0,0 +1,55 @@ +{ + "-profile test --ext.offline": { + "content": [ + { + "MULTIQC_MAPPINGS_CONFIG": { + "python": "3.9.5" + }, + "SRA_FASTQ_FTP": { + "wget": "1.21.4" + }, + "SRA_IDS_TO_RUNINFO": { + "python": "3.9.5" + }, + "SRA_RUNINFO_TO_FTP": { + "python": "3.9.5" + } + }, + [ + "fastq", + "fastq/ERX10144910_ERR10677146_1.fastq.gz", + "fastq/ERX10144910_ERR10677146_2.fastq.gz", + "fastq/ERX1234252_ERR1160845.fastq.gz", + "fastq/md5", + "fastq/md5/ERX10144910_ERR10677146_1.fastq.gz.md5", + "fastq/md5/ERX10144910_ERR10677146_2.fastq.gz.md5", + "fastq/md5/ERX1234252_ERR1160845.fastq.gz.md5", + "metadata", + "metadata/ERR10677146.runinfo_ftp.tsv", + "metadata/ERR1160845.runinfo_ftp.tsv", + "pipeline_info", + "pipeline_info/nf_core_fetchngs_software_versions.yml", + "samplesheet", + "samplesheet/id_mappings.csv", + "samplesheet/multiqc_config.yml", + "samplesheet/samplesheet.csv" + ], + [ + "ERX10144910_ERR10677146_1.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90", + "ERX10144910_ERR10677146_2.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90", + "ERX1234252_ERR1160845.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90", + "ERX10144910_ERR10677146_1.fastq.gz.md5:md5,5bff0624e9f12fceaca657ef787bfdb5", + "ERX10144910_ERR10677146_2.fastq.gz.md5:md5,5b29c334d293d91306fa852f6d6cd719", + "ERX1234252_ERR1160845.fastq.gz.md5:md5,939bd1d72ed1dc24903e8f01010ef709", + "id_mappings.csv:md5,1721ea7256be67b5808434dad0285a3f", + "multiqc_config.yml:md5,9d6cd40e5729fb5c9632dd8b25d460b0" + ], + "samplesheet.csv:md5,2613ca86e10d7448548502b164765261" + ], + "timestamp": "2026-08-19T14:33:28.942194", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.04.6" + } + } +} \ No newline at end of file