diff --git a/assets/offline/ERR10677146.runinfo.tsv b/assets/offline/ERR10677146.runinfo.tsv new file mode 100644 index 00000000..158f3c16 --- /dev/null +++ b/assets/offline/ERR10677146.runinfo.tsv @@ -0,0 +1,2 @@ +run_accession experiment_accession sample_accession secondary_sample_accession study_accession secondary_study_accession submission_accession run_alias experiment_alias sample_alias study_alias library_layout library_selection library_source library_strategy library_name instrument_model instrument_platform base_count read_count tax_id scientific_name sample_title experiment_title study_title sample_description fastq_md5 fastq_bytes fastq_ftp fastq_galaxy fastq_aspera +ERR10677146 ERX10144910 SAMEA8947202 ERS6629975 PRJEB11419 ERP012803 ERA19529803 qiita_ppdid_118164:10317.THDMI.BLANK5.11B qiita_ptid_10903:10317.THDMI.BLANK5.11B qiita_sid_10317:10317.THDMI.BLANK5.11B qiita_sid_10317 PAIRED PCR METAGENOMIC WGS 10317.THDMI.BLANK5.11B Illumina NovaSeq 6000 ILLUMINA 146601 978 256318 metagenome 10317.THDMI.BLANK5.11B Illumina NovaSeq 6000 sequencing: qiita_ptid_10903:10317.THDMI.BLANK5.11B American Gut Project American Gut control 2e2014fade7e73ad0500b049472748c1;2e2014fade7e73ad0500b049472748c1 39628;41645 ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz fasp.sra.ebi.ac.uk:/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_1.fastq.gz;fasp.sra.ebi.ac.uk:/vol1/fastq/ERR106/046/ERR10677146/ERR10677146_2.fastq.gz diff --git a/assets/offline/ERR1160845.runinfo.tsv b/assets/offline/ERR1160845.runinfo.tsv new file mode 100644 index 00000000..ad2dd5b2 --- /dev/null +++ b/assets/offline/ERR1160845.runinfo.tsv @@ -0,0 +1,2 @@ +run_accession experiment_accession sample_accession secondary_sample_accession study_accession secondary_study_accession submission_accession run_alias experiment_alias sample_alias study_alias library_layout library_selection library_source library_strategy library_name instrument_model instrument_platform base_count read_count tax_id scientific_name sample_title experiment_title study_title sample_description fastq_md5 fastq_bytes fastq_ftp fastq_galaxy fastq_aspera +ERR1160845 ERX1234252 SAMEA3687213 ERS994362 PRJEB11419 ERP012803 ERA541392 qiita_ppdid_706:10317.BLANK.93.3D.r22 qiita_ptid_1263:10317.BLANK.93.3D.r22 qiita_sid_10317:10317.BLANK.93.3D.r22 qiita_sid_10317 SINGLE PCR METAGENOMIC AMPLICON 10317.BLANK.93.3D.r22 Illumina HiSeq 2500 ILLUMINA 4158 33 256318 metagenome 10317.BLANK.93.3D.r22 Illumina HiSeq 2500 sequencing: qiita_ptid_1263:10317.BLANK.93.3D.r22 American Gut Project American Gut control 2e2014fade7e73ad0500b049472748c1 2491 ftp.sra.ebi.ac.uk/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz ftp.sra.ebi.ac.uk/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz fasp.sra.ebi.ac.uk:/vol1/fastq/ERR116/005/ERR1160845/ERR1160845.fastq.gz diff --git a/assets/offline/README.md b/assets/offline/README.md new file mode 100644 index 00000000..666e1d3b --- /dev/null +++ b/assets/offline/README.md @@ -0,0 +1,27 @@ +# Offline runinfo fixtures + +Pinned ENA responses used by `SRA_IDS_TO_RUNINFO` when `ext.offline` is set, so a run +can complete without reaching EBI: + +```groovy +process { + withName: '.*SRA_IDS_TO_RUNINFO.*' { ext.offline = true } + withName: '.*SRA_FASTQ_FTP.*' { ext.offline = true } +} +``` + +Each file is a real ENA response for one accession, captured with the module's default +`ENA_METADATA_FIELDS`, so the column set is exactly what `sra_runinfo_to_ftp.py` and +`SRA_TO_SAMPLESHEET` expect. Everything downstream of these two processes runs for real. + +`fastq_md5` is the one column that does not hold ENA's value. Offline `SRA_FASTQ_FTP` +generates a fixed one-read FASTQ instead of downloading, and still runs `md5sum -c`, so +`fastq_md5` has to be the md5 of those generated bytes: + +``` +printf '@offline\nACGT\n+\nIIII\n' | gzip -n | md5sum +``` + +Repeat the md5 once per FASTQ, semicolon-separated, `_1` then `_2` for paired runs. +`fastq_bytes`, `fastq_ftp` and `fastq_galaxy` keep ENA's real values — nothing reads them +once the download is bypassed, and they make the fixture's origin obvious. diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf index 0b077ab4..55c892cc 100644 --- a/modules/local/sra_fastq_ftp/main.nf +++ b/modules/local/sra_fastq_ftp/main.nf @@ -19,7 +19,31 @@ process SRA_FASTQ_FTP { script: def args = task.ext.args ?: '' - if (meta.single_end) { + def offline = task.ext.offline ?: false + if (offline) { + // gzip -n keeps the bytes deterministic so the fastq_md5 pinned in assets/offline still verifies + def generate_read = "printf '@offline\\nACGT\\n+\\nIIII\\n' | gzip -n" + if (meta.single_end) { + """ + $generate_read > ${meta.id}.fastq.gz + + echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 + md5sum -c ${meta.id}.fastq.gz.md5 + """ + } else { + """ + $generate_read > ${meta.id}_1.fastq.gz + + echo "${meta.md5_1} ${meta.id}_1.fastq.gz" > ${meta.id}_1.fastq.gz.md5 + md5sum -c ${meta.id}_1.fastq.gz.md5 + + $generate_read > ${meta.id}_2.fastq.gz + + echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 + md5sum -c ${meta.id}_2.fastq.gz.md5 + """ + } + } else if (meta.single_end) { """ wget \\ $args \\ diff --git a/modules/local/sra_ids_to_runinfo/main.nf b/modules/local/sra_ids_to_runinfo/main.nf index de5c3acc..146cd2ff 100644 --- a/modules/local/sra_ids_to_runinfo/main.nf +++ b/modules/local/sra_ids_to_runinfo/main.nf @@ -18,13 +18,27 @@ process SRA_IDS_TO_RUNINFO { script: def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : '' - """ - echo $id > id.txt - sra_ids_to_runinfo.py \\ - id.txt \\ - ${id}.runinfo.tsv \\ - $metadata_fields - """ + def offline = task.ext.offline ?: false + if (offline) { + def fixture = file("${projectDir}/assets/offline/${id}.runinfo.tsv") + if (!fixture.exists()) { + error("No offline runinfo fixture for accession '${id}' (looked for ${fixture})") + } + // Inlined rather than staged so the fixture reaches the task without projectDir being mounted + """ + cat <<'END_RUNINFO' > ${id}.runinfo.tsv +${fixture.text.trim()} +END_RUNINFO + """ + } else { + """ + echo $id > id.txt + sra_ids_to_runinfo.py \\ + id.txt \\ + ${id}.runinfo.tsv \\ + $metadata_fields + """ + } stub: """ diff --git a/tests/offline_ids.csv b/tests/offline_ids.csv new file mode 100644 index 00000000..9cfbe91b --- /dev/null +++ b/tests/offline_ids.csv @@ -0,0 +1,3 @@ +accession +ERR1160845 +ERR10677146 diff --git a/tests/offline_ids_no_fixture.csv b/tests/offline_ids_no_fixture.csv new file mode 100644 index 00000000..c65bca3f --- /dev/null +++ b/tests/offline_ids_no_fixture.csv @@ -0,0 +1,2 @@ +accession +ERR1160846 diff --git a/tests/sra_offline.config b/tests/sra_offline.config new file mode 100644 index 00000000..9e896d4c --- /dev/null +++ b/tests/sra_offline.config @@ -0,0 +1,4 @@ +process { + withName: '.*SRA_IDS_TO_RUNINFO.*' { ext.offline = true } + withName: '.*SRA_FASTQ_FTP.*' { ext.offline = true } +} diff --git a/tests/sra_offline.nf.test b/tests/sra_offline.nf.test new file mode 100644 index 00000000..1e5b8588 --- /dev/null +++ b/tests/sra_offline.nf.test @@ -0,0 +1,57 @@ +nextflow_pipeline { + + name "Test pipeline with ext.offline" + script "../main.nf" + tag "pipeline" + config "./sra_offline.config" + + test("-profile test --ext.offline") { + + when { + params { + outdir = "$outputDir" + input = "$projectDir/tests/offline_ids.csv" + } + } + + then { + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + def csv_file = path(params.outdir + '/samplesheet/samplesheet.csv').csv() + def ebi_calls = [] + new File(workDir).eachFileRecurse { f -> + if (f.name == '.command.sh' && (f.text.contains('sra_ids_to_runinfo.py') || f.text.contains('wget \\'))) { + ebi_calls << f.parentFile.parentFile.name + '/' + f.parentFile.name + } + } + + assert workflow.success + assertAll( + { assert csv_file.rowCount == 2 }, + { assert stable_name.count { it.toString().endsWith('.fastq.gz') } == 3 }, + { assert ebi_calls == [] }, + { assert snapshot( + removeFromYamlMap("${params.outdir}/pipeline_info/nf_core_fetchngs_software_versions.yml", "Workflow"), + stable_name, + stable_path, + "samplesheet.csv:md5," + csv_file.sort().table.collect { row -> [row.getString("sample"), row.getString("fastq_1").replaceAll(params.outdir, ""), row.getString("fastq_2").replaceAll(params.outdir, "")].join(",") }.join("\n").md5() + ).match() } + ) + } + } + + test("-profile test --ext.offline, accession with no fixture") { + + when { + params { + outdir = "$outputDir" + input = "$projectDir/tests/offline_ids_no_fixture.csv" + } + } + + then { + assert workflow.failed + assert workflow.stdout.join('\n').contains("No offline runinfo fixture for accession 'ERR1160846'") + } + } +} diff --git a/tests/sra_offline.nf.test.snap b/tests/sra_offline.nf.test.snap new file mode 100644 index 00000000..1c76749f --- /dev/null +++ b/tests/sra_offline.nf.test.snap @@ -0,0 +1,55 @@ +{ + "-profile test --ext.offline": { + "content": [ + { + "MULTIQC_MAPPINGS_CONFIG": { + "python": "3.9.5" + }, + "SRA_FASTQ_FTP": { + "wget": "1.21.4" + }, + "SRA_IDS_TO_RUNINFO": { + "python": "3.9.5" + }, + "SRA_RUNINFO_TO_FTP": { + "python": "3.9.5" + } + }, + [ + "fastq", + "fastq/ERX10144910_ERR10677146_1.fastq.gz", + "fastq/ERX10144910_ERR10677146_2.fastq.gz", + "fastq/ERX1234252_ERR1160845.fastq.gz", + "fastq/md5", + "fastq/md5/ERX10144910_ERR10677146_1.fastq.gz.md5", + "fastq/md5/ERX10144910_ERR10677146_2.fastq.gz.md5", + "fastq/md5/ERX1234252_ERR1160845.fastq.gz.md5", + "metadata", + "metadata/ERR10677146.runinfo_ftp.tsv", + "metadata/ERR1160845.runinfo_ftp.tsv", + "pipeline_info", + "pipeline_info/nf_core_fetchngs_software_versions.yml", + "samplesheet", + "samplesheet/id_mappings.csv", + "samplesheet/multiqc_config.yml", + "samplesheet/samplesheet.csv" + ], + [ + "ERX10144910_ERR10677146_1.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90", + "ERX10144910_ERR10677146_2.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90", + "ERX1234252_ERR1160845.fastq.gz:md5,ab1f953a60ce0bdc336e5653e27bdd90", + "ERX10144910_ERR10677146_1.fastq.gz.md5:md5,5bff0624e9f12fceaca657ef787bfdb5", + "ERX10144910_ERR10677146_2.fastq.gz.md5:md5,5b29c334d293d91306fa852f6d6cd719", + "ERX1234252_ERR1160845.fastq.gz.md5:md5,939bd1d72ed1dc24903e8f01010ef709", + "id_mappings.csv:md5,1721ea7256be67b5808434dad0285a3f", + "multiqc_config.yml:md5,9d6cd40e5729fb5c9632dd8b25d460b0" + ], + "samplesheet.csv:md5,2613ca86e10d7448548502b164765261" + ], + "timestamp": "2026-08-19T14:33:28.942194", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.04.6" + } + } +} \ No newline at end of file