feat(offline): ext.offline fixture path for the two EBI-facing processes

assets/offline/ERR10677146.runinfo.tsv

@@ -0,0 +1,2 @@
+run_accession	experiment_accession	sample_accession	secondary_sample_accession	study_accession	secondary_study_accession	submission_accession	run_alias	experiment_alias	sample_alias	study_alias	library_layout	library_selection	library_source	library_strategy	library_name	instrument_model	instrument_platform	base_count	read_count	tax_id	scientific_name	sample_title	experiment_title	study_title	sample_description	fastq_md5	fastq_bytes	fastq_ftp	fastq_galaxy	fastq_aspera
+ERR10677146	ERX10144910	SAMEA8947202	ERS6629975	PRJEB11419	ERP012803	ERA19529803	qiita_ppdid_118164:10317.THDMI.BLANK5.11B	qiita_ptid_10903:10317.THDMI.BLANK5.11B	qiita_sid_10317:10317.THDMI.BLANK5.11B	qiita_sid_10317	PAIRED	PCR	METAGENOMIC	WGS	10317.THDMI.BLANK5.11B	Illumina NovaSeq 6000	ILLUMINA	146601	978	256318	metagenome	10317.THDMI.BLANK5.11B	Illumina NovaSeq 6000 sequencing: qiita_ptid_10903:10317.THDMI.BLANK5.11B	American Gut Project	American Gut control	2e2014fade7e73ad0500b049472748c1;2e2014fade7e73ad0500b049472748c1	39628;41645	ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz	ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz	fasp.sra.ebi.ac.uk:/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;fasp.sra.ebi.ac.uk:/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz

assets/offline/ERR1160845.runinfo.tsv

@@ -0,0 +1,2 @@
+run_accession	experiment_accession	sample_accession	secondary_sample_accession	study_accession	secondary_study_accession	submission_accession	run_alias	experiment_alias	sample_alias	study_alias	library_layout	library_selection	library_source	library_strategy	library_name	instrument_model	instrument_platform	base_count	read_count	tax_id	scientific_name	sample_title	experiment_title	study_title	sample_description	fastq_md5	fastq_bytes	fastq_ftp	fastq_galaxy	fastq_aspera
+ERR1160845	ERX1234252	SAMEA3687213	ERS994362	PRJEB11419	ERP012803	ERA541392	qiita_ppdid_706:10317.BLANK.93.3D.r22	qiita_ptid_1263:10317.BLANK.93.3D.r22	qiita_sid_10317:10317.BLANK.93.3D.r22	qiita_sid_10317	SINGLE	PCR	METAGENOMIC	AMPLICON	10317.BLANK.93.3D.r22	Illumina HiSeq 2500	ILLUMINA	4158	33	256318	metagenome	10317.BLANK.93.3D.r22	Illumina HiSeq 2500 sequencing: qiita_ptid_1263:10317.BLANK.93.3D.r22	American Gut Project	American Gut control	2e2014fade7e73ad0500b049472748c1	2491	ftp.sra.ebi.ac.uk/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz	ftp.sra.ebi.ac.uk/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz	fasp.sra.ebi.ac.uk:/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz

assets/offline/README.md

@@ -0,0 +1,27 @@
+# Offline runinfo fixtures
+
+Pinned ENA responses used by `SRA_IDS_TO_RUNINFO` when `ext.offline` is set, so a run
+can complete without reaching EBI:
+
+```groovy
+process {
+    withName: '.*SRA_IDS_TO_RUNINFO.*' { ext.offline = true }
+    withName: '.*SRA_FASTQ_FTP.*'      { ext.offline = true }
+}
+```
+
+Each file is a real ENA response for one accession, captured with the module's default
+`ENA_METADATA_FIELDS`, so the column set is exactly what `sra_runinfo_to_ftp.py` and
+`SRA_TO_SAMPLESHEET` expect. Everything downstream of these two processes runs for real.
+
+`fastq_md5` is the one column that does not hold ENA's value. Offline `SRA_FASTQ_FTP`
+generates a fixed one-read FASTQ instead of downloading, and still runs `md5sum -c`, so
+`fastq_md5` has to be the md5 of those generated bytes:
+
+```
+printf '@offline\nACGT\n+\nIIII\n' | gzip -n | md5sum
+```
+
+Repeat the md5 once per FASTQ, semicolon-separated, `_1` then `_2` for paired runs.
+`fastq_bytes`, `fastq_ftp` and `fastq_galaxy` keep ENA's real values — nothing reads them
+once the download is bypassed, and they make the fixture's origin obvious.

modules/local/sra_fastq_ftp/main.nf

@@ -19,7 +19,31 @@ process SRA_FASTQ_FTP {
 
     script:
     def args = task.ext.args ?: ''
-    if (meta.single_end) {
+    def offline = task.ext.offline ?: false
+    if (offline) {
+        // gzip -n keeps the bytes deterministic so the fastq_md5 pinned in assets/offline still verifies
+        def generate_read = "printf '@offline\\nACGT\\n+\\nIIII\\n' | gzip -n"
+        if (meta.single_end) {
+            """
+            $generate_read > ${meta.id}.fastq.gz
+
+            echo "${meta.md5_1}  ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5
+            md5sum -c ${meta.id}.fastq.gz.md5
+            """
+        } else {
+            """
+            $generate_read > ${meta.id}_1.fastq.gz
+
+            echo "${meta.md5_1}  ${meta.id}_1.fastq.gz" > ${meta.id}_1.fastq.gz.md5
+            md5sum -c ${meta.id}_1.fastq.gz.md5
+
+            $generate_read > ${meta.id}_2.fastq.gz
+
+            echo "${meta.md5_2}  ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5
+            md5sum -c ${meta.id}_2.fastq.gz.md5
+            """
+        }
+    } else if (meta.single_end) {
         """
         wget \\
             $args \\

modules/local/sra_ids_to_runinfo/main.nf

@@ -18,13 +18,27 @@ process SRA_IDS_TO_RUNINFO {
 
     script:
     def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : ''
-    """
-    echo $id > id.txt
-    sra_ids_to_runinfo.py \\
-        id.txt \\
-        ${id}.runinfo.tsv \\
-        $metadata_fields
-    """
+    def offline = task.ext.offline ?: false
+    if (offline) {
+        def fixture = file("${projectDir}/assets/offline/${id}.runinfo.tsv")
+        if (!fixture.exists()) {
+            error("No offline runinfo fixture for accession '${id}' (looked for ${fixture})")
+        }
+        // Inlined rather than staged so the fixture reaches the task without projectDir being mounted
+        """
+        cat <<'END_RUNINFO' > ${id}.runinfo.tsv
+${fixture.text.trim()}
+END_RUNINFO
+        """
+    } else {
+        """
+        echo $id > id.txt
+        sra_ids_to_runinfo.py \\
+            id.txt \\
+            ${id}.runinfo.tsv \\
+            $metadata_fields
+        """
+    }
 
     stub:
     """

tests/offline_ids.csv

@@ -0,0 +1,3 @@
+accession
+ERR1160845
+ERR10677146

tests/offline_ids_no_fixture.csv

@@ -0,0 +1,2 @@
+accession
+ERR1160846

tests/sra_offline.config

@@ -0,0 +1,4 @@
+process {
+    withName: '.*SRA_IDS_TO_RUNINFO.*' { ext.offline = true }
+    withName: '.*SRA_FASTQ_FTP.*'      { ext.offline = true }
+}

tests/sra_offline.nf.test

@@ -0,0 +1,57 @@
+nextflow_pipeline {
+
+    name "Test pipeline with ext.offline"
+    script "../main.nf"
+    tag "pipeline"
+    config "./sra_offline.config"
+
+    test("-profile test --ext.offline") {
+
+        when {
+            params {
+                outdir = "$outputDir"
+                input = "$projectDir/tests/offline_ids.csv"
+            }
+        }
+
+        then {
+            def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
+            def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            def csv_file = path(params.outdir + '/samplesheet/samplesheet.csv').csv()
+            def ebi_calls = []
+            new File(workDir).eachFileRecurse { f ->
+                if (f.name == '.command.sh' && (f.text.contains('sra_ids_to_runinfo.py') || f.text.contains('wget \\'))) {
+                    ebi_calls << f.parentFile.parentFile.name + '/' + f.parentFile.name
+                }
+            }
+
+            assert workflow.success
+            assertAll(
+                { assert csv_file.rowCount == 2 },
+                { assert stable_name.count { it.toString().endsWith('.fastq.gz') } == 3 },
+                { assert ebi_calls == [] },
+                { assert snapshot(
+                    removeFromYamlMap("${params.outdir}/pipeline_info/nf_core_fetchngs_software_versions.yml", "Workflow"),
+                    stable_name,
+                    stable_path,
+                    "samplesheet.csv:md5," + csv_file.sort().table.collect { row -> [row.getString("sample"), row.getString("fastq_1").replaceAll(params.outdir, ""), row.getString("fastq_2").replaceAll(params.outdir, "")].join(",") }.join("\n").md5()
+                ).match() }
+            )
+        }
+    }
+
+    test("-profile test --ext.offline, accession with no fixture") {
+
+        when {
+            params {
+                outdir = "$outputDir"
+                input = "$projectDir/tests/offline_ids_no_fixture.csv"
+            }
+        }
+
+        then {
+            assert workflow.failed
+            assert workflow.stdout.join('\n').contains("No offline runinfo fixture for accession 'ERR1160846'")
+        }
+    }
+}

tests/sra_offline.nf.test.snap

@@ -0,0 +1,55 @@
+{
+    "-profile test --ext.offline": {
+        "content": [
+            {
+                "MULTIQC_MAPPINGS_CONFIG": {
+                    "python": "3.9.5"
+                },
+                "SRA_FASTQ_FTP": {
+                    "wget": "1.21.4"
+                },
+                "SRA_IDS_TO_RUNINFO": {
+                    "python": "3.9.5"
+                },
+                "SRA_RUNINFO_TO_FTP": {
+                    "python": "3.9.5"
+                }
+            },
+            [
+                "fastq",
+                "fastq/ERX10144910_ERR10677146_1.fastq.gz",
+                "fastq/ERX10144910_ERR10677146_2.fastq.gz",
+                "fastq/ERX1234252_ERR1160845.fastq.gz",
+                "fastq/md5",
+                "fastq/md5/ERX10144910_ERR10677146_1.fastq.gz.md5",
+                "fastq/md5/ERX10144910_ERR10677146_2.fastq.gz.md5",
+                "fastq/md5/ERX1234252_ERR1160845.fastq.gz.md5",
+                "metadata",
+                "metadata/ERR10677146.runinfo_ftp.tsv",
+                "metadata/ERR1160845.runinfo_ftp.tsv",
+                "pipeline_info",
+                "pipeline_info/nf_core_fetchngs_software_versions.yml",
+                "samplesheet",
+                "samplesheet/id_mappings.csv",
+                "samplesheet/multiqc_config.yml",
+                "samplesheet/samplesheet.csv"
+            ],
+            [
+                "ERX10144910_ERR10677146_1.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90",
+                "ERX10144910_ERR10677146_2.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90",
+                "ERX1234252_ERR1160845.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90",
+                "ERX10144910_ERR10677146_1.fastq.gz.md5:md5,5bff0624e9f12fceaca657ef787bfdb5",
+                "ERX10144910_ERR10677146_2.fastq.gz.md5:md5,5b29c334d293d91306fa852f6d6cd719",
+                "ERX1234252_ERR1160845.fastq.gz.md5:md5,939bd1d72ed1dc24903e8f01010ef709",
+                "id_mappings.csv:md5,1721ea7256be67b5808434dad0285a3f",
+                "multiqc_config.yml:md5,9d6cd40e5729fb5c9632dd8b25d460b0"
+            ],
+            "samplesheet.csv:md5,2613ca86e10d7448548502b164765261"
+        ],
+        "timestamp": "2026-08-19T14:33:28.942194",
+        "meta": {
+            "nf-test": "0.9.5",
+            "nextflow": "25.04.6"
+        }
+    }
+}