From 7d1c0ddffca25888fd443929bf0e970e5d996842 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Wed, 13 May 2026 12:18:54 +0100 Subject: [PATCH 01/30] explicitly set test_upload = true in test configs --- ...est_assembly_coassembly_no_coverage.config | 34 ++++++++++++++++++ ...t_assembly_coassembly_with_coverage.config | 35 +++++++++++++++++++ conf/test_assembly_complete_metadata.config | 1 + ...t_assembly_no_coverage_paired_reads.config | 1 + ...t_assembly_no_coverage_single_reads.config | 1 + ...assembly_no_study_complete_metadata.config | 1 - conf/test_assembly_one_contig.config | 1 + 7 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 conf/test_assembly_coassembly_no_coverage.config create mode 100644 conf/test_assembly_coassembly_with_coverage.config diff --git a/conf/test_assembly_coassembly_no_coverage.config b/conf/test_assembly_coassembly_no_coverage.config new file mode 100644 index 0000000..eda3ca5 --- /dev/null +++ b/conf/test_assembly_coassembly_no_coverage.config @@ -0,0 +1,34 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/seqsubmit -profile test_assembly_coassembly_no_coverage, --outdir + +---------------------------------------------------------------------------------------- +*/ + +process { + resourceLimits = [ + cpus: 2, + memory: '8.GB', + time: '1.h' + ] +} + +params { + config_profile_name = 'Test --mode metagenomic_assemblies co-assembly without coverage profile' + config_profile_description = 'Co-assembly test with missing coverage and paired-end reads from two runs' + + // Input data + input = "${projectDir}/assets/samplesheet_assembly_coassembly_no_coverage.csv" + outdir = 'test_output' + + mode = "metagenomic_assemblies" + submission_study = "PRJEB98843" + centre_name = "TEST_CENTER" + + test_upload = true +} diff --git a/conf/test_assembly_coassembly_with_coverage.config b/conf/test_assembly_coassembly_with_coverage.config new file mode 100644 index 0000000..910231a --- /dev/null +++ b/conf/test_assembly_coassembly_with_coverage.config @@ -0,0 +1,35 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/seqsubmit -profile test_assembly_coassembly_with_coverage, --outdir + +---------------------------------------------------------------------------------------- +*/ + +process { + resourceLimits = [ + cpus: 2, + memory: '8.GB', + time: '1.h' + ] +} + +params { + config_profile_name = 'Test --mode metagenomic_assemblies co-assembly with coverage profile' + config_profile_description = 'Co-assembly test with pre-provided coverage and multiple run accessions' + + // Input data + input = "${projectDir}/assets/samplesheet_assembly_coassembly_with_coverage.csv" + outdir = 'test_output' + + mode = "metagenomic_assemblies" + submission_study = "PRJEB98843" + centre_name = "TEST_CENTER" + + test_upload = true + +} diff --git a/conf/test_assembly_complete_metadata.config b/conf/test_assembly_complete_metadata.config index 16e4ccf..ce366be 100644 --- a/conf/test_assembly_complete_metadata.config +++ b/conf/test_assembly_complete_metadata.config @@ -30,4 +30,5 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" + test_upload = true } diff --git a/conf/test_assembly_no_coverage_paired_reads.config b/conf/test_assembly_no_coverage_paired_reads.config index 65c73e4..4571a1c 100644 --- a/conf/test_assembly_no_coverage_paired_reads.config +++ b/conf/test_assembly_no_coverage_paired_reads.config @@ -30,4 +30,5 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" + test_upload = true } diff --git a/conf/test_assembly_no_coverage_single_reads.config b/conf/test_assembly_no_coverage_single_reads.config index 814de31..ee269e9 100644 --- a/conf/test_assembly_no_coverage_single_reads.config +++ b/conf/test_assembly_no_coverage_single_reads.config @@ -30,4 +30,5 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" + test_upload = true } diff --git a/conf/test_assembly_no_study_complete_metadata.config b/conf/test_assembly_no_study_complete_metadata.config index b1c96d7..6831007 100644 --- a/conf/test_assembly_no_study_complete_metadata.config +++ b/conf/test_assembly_no_study_complete_metadata.config @@ -31,5 +31,4 @@ params { centre_name = "TEST_CENTER" test_upload = true - } diff --git a/conf/test_assembly_one_contig.config b/conf/test_assembly_one_contig.config index b27784e..ce2e86c 100644 --- a/conf/test_assembly_one_contig.config +++ b/conf/test_assembly_one_contig.config @@ -30,4 +30,5 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" + test_upload = true } From 100c7c0c832dfcf6f6343fc0de21f39db10ec048 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Wed, 13 May 2026 12:21:55 +0100 Subject: [PATCH 02/30] add partial co assembly support in ASSEMBLYSUBMIT Co-authored-by: Copilot --- ...esheet_assembly_coassembly_no_coverage.csv | 2 + ...heet_assembly_coassembly_with_coverage.csv | 2 + assets/schema_input_assembly.json | 18 +++--- .../create_assembly_metadata_csv/main.nf | 5 +- nextflow.config | 2 + tests/assembly_coassembly_no_coverage.nf.test | 39 ++++++++++++ .../assembly_coassembly_with_coverage.nf.test | 39 ++++++++++++ workflows/assemblysubmit.nf | 60 ++++++++++++++++--- 8 files changed, 146 insertions(+), 21 deletions(-) create mode 100644 assets/samplesheet_assembly_coassembly_no_coverage.csv create mode 100644 assets/samplesheet_assembly_coassembly_with_coverage.csv create mode 100644 tests/assembly_coassembly_no_coverage.nf.test create mode 100644 tests/assembly_coassembly_with_coverage.nf.test diff --git a/assets/samplesheet_assembly_coassembly_no_coverage.csv b/assets/samplesheet_assembly_coassembly_no_coverage.csv new file mode 100644 index 0000000..e0a4f22 --- /dev/null +++ b/assets/samplesheet_assembly_coassembly_no_coverage.csv @@ -0,0 +1,2 @@ +sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +no_coverage_paired_reads,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz,,ERR000001;ERR000002,SPAdes,3.15 diff --git a/assets/samplesheet_assembly_coassembly_with_coverage.csv b/assets/samplesheet_assembly_coassembly_with_coverage.csv new file mode 100644 index 0000000..7f51cca --- /dev/null +++ b/assets/samplesheet_assembly_coassembly_with_coverage.csv @@ -0,0 +1,2 @@ +sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,ERR000001;ERR000002,SPAdes,3.15 diff --git a/assets/schema_input_assembly.json b/assets/schema_input_assembly.json index 2b5fcea..929589a 100644 --- a/assets/schema_input_assembly.json +++ b/assets/schema_input_assembly.json @@ -25,33 +25,29 @@ "anyOf": [ { "type": "string", - "format": "file-path", - "exists": true, - "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$" + "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?(;\\S+\\.(fq|fastq)(\\.gz)?)*$" }, { "type": "string", "maxLength": 0 } ], - "errorMessage": "FASTQ file must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Forward reads if paired-end or single-end reads FASTQ file" + "errorMessage": "FASTQ file(s) must have extension '.fq' or '.fastq' (optionally gzipped); for co-assemblies use semicolon-separated paths", + "description": "Forward reads if paired-end or single-end reads FASTQ file. For co-assemblies, semicolon-separated paths (e.g. run1_R1.fq.gz;run2_R1.fq.gz)" }, "fastq_2": { "anyOf": [ { "type": "string", - "format": "file-path", - "exists": true, - "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?$" + "pattern": "^\\S+\\.(fq|fastq)(\\.gz)?(;\\S+\\.(fq|fastq)(\\.gz)?)*$" }, { "type": "string", "maxLength": 0 } ], - "errorMessage": "FASTQ file for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped)", - "description": "Reverse reads FASTQ file if paired-end. Leave empty for single-end reads" + "errorMessage": "FASTQ file(s) for reverse reads must have extension '.fq' or '.fastq' (optionally gzipped); for co-assemblies use semicolon-separated paths", + "description": "Reverse reads FASTQ file if paired-end. Leave empty for single-end reads. For co-assemblies, semicolon-separated paths (e.g. run1_R2.fq.gz;run2_R2.fq.gz)" }, "coverage": { "anyOf": [ @@ -71,7 +67,7 @@ "type": "string", "pattern": "^\\S+$", "errorMessage": "Accession must be provided and cannot contain spaces", - "description": "Accession of the run used to generate the assembly" + "description": "Accession(s) of the run(s) used to generate the assembly. For co-assemblies, use semicolon-separated values (e.g. ERR000001;ERR000002)" }, "assembler": { "type": "string", diff --git a/modules/local/create_assembly_metadata_csv/main.nf b/modules/local/create_assembly_metadata_csv/main.nf index b7d7151..f3f8312 100644 --- a/modules/local/create_assembly_metadata_csv/main.nf +++ b/modules/local/create_assembly_metadata_csv/main.nf @@ -19,8 +19,11 @@ process CREATE_ASSEMBLY_METADATA_CSV { script: def header = 'Runs,Coverage,Assembler,Version,Filepath,Sample' + // Format run accessions: wrap in quotes if it's a list (co-assembly) or single value + def runs_joined = meta.run_accession instanceof List ? meta.run_accession.join(',') : meta.run_accession + def runs = "\"${runs_joined}\"" def row = [ - meta.run_accession, + runs, meta.coverage, meta.assembler, meta.assembler_version, diff --git a/nextflow.config b/nextflow.config index 4f9e21a..5482eed 100644 --- a/nextflow.config +++ b/nextflow.config @@ -199,6 +199,8 @@ profiles { test_assembly_no_coverage_single_reads { includeConfig 'conf/test_assembly_no_coverage_single_reads.config' } test_assembly_no_coverage_paired_reads { includeConfig 'conf/test_assembly_no_coverage_paired_reads.config' } test_assembly_one_contig { includeConfig 'conf/test_assembly_one_contig.config' } + test_assembly_coassembly_with_coverage { includeConfig 'conf/test_assembly_coassembly_with_coverage.config' } + test_assembly_coassembly_no_coverage { includeConfig 'conf/test_assembly_coassembly_no_coverage.config' } } // Load nf-core custom profiles from different institutions diff --git a/tests/assembly_coassembly_no_coverage.nf.test b/tests/assembly_coassembly_no_coverage.nf.test new file mode 100644 index 0000000..a4eefd8 --- /dev/null +++ b/tests/assembly_coassembly_no_coverage.nf.test @@ -0,0 +1,39 @@ +nextflow_pipeline { + + name "Test assembly submission workflow - coassembly_no_coverage" + script "../main.nf" + tag "pipeline" + tag "mode_assembly" + tag "test_assembly_coassembly_no_coverage" + profile "test_assembly_coassembly_no_coverage" + + test("-profile test_assembly_coassembly_no_coverage") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Early failure no need to test the rest of snapshots + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() } + ) + } + } +} diff --git a/tests/assembly_coassembly_with_coverage.nf.test b/tests/assembly_coassembly_with_coverage.nf.test new file mode 100644 index 0000000..60f15e9 --- /dev/null +++ b/tests/assembly_coassembly_with_coverage.nf.test @@ -0,0 +1,39 @@ +nextflow_pipeline { + + name "Test assembly submission workflow - coassembly_with_coverage" + script "../main.nf" + tag "pipeline" + tag "mode_assembly" + tag "test_assembly_coassembly_with_coverage" + profile "test_assembly_coassembly_with_coverage" + + test("-profile test_assembly_coassembly_with_coverage") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Early failure no need to test the rest of snapshots + assert workflow.success + assertAll( + { assert snapshot( + // Number of successful tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_name, + // All files with stable contents + stable_path + ).match() } + ) + } + } +} diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 1f4dc7b..6d8aae4 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -49,11 +49,13 @@ workflow ASSEMBLYSUBMIT { // Create assembly channel with proper metadata structure assembly_fasta = ch_samplesheet .map { row -> + // support semicolon-separated values for co-assemblies (e.g. ERR000001;ERR000002) + def run_accessions = row[5].split(';').toList() def meta = [ id: row[0].id, single_end: row[3] ? false : true, coverage: row[4] ?: null, - run_accession: row[5], + run_accession: run_accessions.size() == 1 ? run_accessions[0] : run_accessions, assembler: row[6], assembler_version: row[7] ] @@ -63,21 +65,36 @@ workflow ASSEMBLYSUBMIT { reads_fastq = ch_samplesheet .filter { row -> row[2] && row[2] != "" } // Check if fastq_1 exists and is not empty .map { row -> + def run_accessions = row[5].split(';').toList() def meta = [ id: row[0].id, single_end: row[3] ? false : true, coverage: row[4] ?: null, - run_accession: row[5], + run_accession: run_accessions.size() == 1 ? run_accessions[0] : run_accessions, assembler: row[6], assembler_version: row[7] ] - if (row[3] && row[3] != "") { + def fastq1_list = row[2].split(';').toList() + def fastq2_list = (row[3] && row[3] != "") ? row[3].split(';').toList() : [] + // Validation: Check that number of read files matches number of accessions + if (fastq1_list.size() != run_accessions.size()) { + error "Sample ${meta.id}: Number of forward read files (${fastq1_list.size()}) does not match number of run accessions (${run_accessions.size()})" + } + + if (fastq2_list && fastq2_list.size() != run_accessions.size()) { + error "Sample ${meta.id}: Number of reverse read files (${fastq2_list.size()}) does not match number of run accessions (${run_accessions.size()})" + } + + // Convert paths to file objects + def fastq1_paths = fastq1_list.collect { path -> file(path) } + def fastq2_paths = fastq2_list ? fastq2_list.collect { path -> file(path) } : [] + if (fastq2_paths) { // If paired end reads - [meta, [file(row[2]), file(row[3])]] + [meta, fastq1_paths, fastq2_paths] } else { // If single end - [meta, file(row[2])] + [meta, fastq1_paths] } } @@ -94,11 +111,36 @@ workflow ASSEMBLYSUBMIT { } // For assemblies without coverage, calculate coverage with CoverM + // Transform reads into the format CoverM expects: list of all read files validated_fastas.filter { meta, _fasta -> meta.coverage == null } .join(reads_fastq) - .multiMap { meta, fasta, fastq -> + .map { tuple -> + def meta = tuple[0] + def fasta = tuple[1] + def reads_data = tuple[2..-1] + + // Transform reads into flat list for CoverM + def all_reads = [] + if (meta.single_end) { + // Single-end: just flatten the R1 list + def fastq1_list = reads_data[0] + all_reads = fastq1_list + } else { + // Paired-end: interleave R1 and R2 files + // Input format: [meta, [R1_1, R1_2, ...], [R2_1, R2_2, ...]] + def fastq1_list = reads_data[0] + def fastq2_list = reads_data[1] + + // Create list as: R1_1, R2_1, R1_2, R2_2, ... + // Use collectMany to flatten the pairs + all_reads = [fastq1_list, fastq2_list].transpose().collectMany { pair -> pair } + } + + [meta, fasta, all_reads] + } + .multiMap { meta, fasta, reads -> assembly: [ meta, fasta ] - reads: [ meta, fastq ] + reads: [ meta, reads ] } .set { coverm_input } @@ -232,8 +274,8 @@ workflow ASSEMBLYSUBMIT { : file("${projectDir}/assets/methods_description_template.yml", checkIfExists: true) def ch_methods_description = channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: true)) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{meta, file -> file}) - ch_multiqc_files = ch_multiqc_files.mix(CONCAT_METADATA.out.file_out.map{meta, file -> file}) + ch_multiqc_files = ch_multiqc_files.mix(CONCAT_ACCESSIONS.out.file_out.map{_meta, file -> file}) + ch_multiqc_files = ch_multiqc_files.mix(CONCAT_METADATA.out.file_out.map{_meta, file -> file}) MULTIQC( ch_multiqc_files.flatten().collect().map { files -> [ From a65c94c3b0f956f3896e522c78f057e8332ec433 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 14 May 2026 13:26:32 +0100 Subject: [PATCH 03/30] add REGISTER_COASSEMBLY_SAMPLE Co-authored-by: Copilot --- ...esheet_assembly_coassembly_no_coverage.csv | 2 +- ...heet_assembly_coassembly_with_coverage.csv | 2 +- bin/register_coassembly_sample.py | 484 ++++++++++++++++++ ...t_assembly_coassembly_with_coverage.config | 1 - .../create_assembly_metadata_csv/main.nf | 2 +- .../environment.yml | 8 + .../local/register_coassembly_sample/main.nf | 52 ++ .../local/register_coassembly_sample/meta.yml | 44 ++ .../tests/coassembly_metadata.csv | 11 + .../tests/main.nf.test | 36 ++ .../tests/nextflow.config | 6 + workflows/assemblysubmit.nf | 22 +- 12 files changed, 664 insertions(+), 6 deletions(-) create mode 100755 bin/register_coassembly_sample.py create mode 100644 modules/local/register_coassembly_sample/environment.yml create mode 100644 modules/local/register_coassembly_sample/main.nf create mode 100644 modules/local/register_coassembly_sample/meta.yml create mode 100644 modules/local/register_coassembly_sample/tests/coassembly_metadata.csv create mode 100644 modules/local/register_coassembly_sample/tests/main.nf.test create mode 100644 modules/local/register_coassembly_sample/tests/nextflow.config diff --git a/assets/samplesheet_assembly_coassembly_no_coverage.csv b/assets/samplesheet_assembly_coassembly_no_coverage.csv index e0a4f22..db29939 100644 --- a/assets/samplesheet_assembly_coassembly_no_coverage.csv +++ b/assets/samplesheet_assembly_coassembly_no_coverage.csv @@ -1,2 +1,2 @@ sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version -no_coverage_paired_reads,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz,,ERR000001;ERR000002,SPAdes,3.15 +no_coverage_paired_reads,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz,,SRR3183850;SRR3183951,SPAdes,3.15 diff --git a/assets/samplesheet_assembly_coassembly_with_coverage.csv b/assets/samplesheet_assembly_coassembly_with_coverage.csv index 7f51cca..cf9cab3 100644 --- a/assets/samplesheet_assembly_coassembly_with_coverage.csv +++ b/assets/samplesheet_assembly_coassembly_with_coverage.csv @@ -1,2 +1,2 @@ sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version -coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,ERR000001;ERR000002,SPAdes,3.15 +coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,SRR3183850;SRR3183951,SPAdes,3.15 diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py new file mode 100755 index 0000000..8238f50 --- /dev/null +++ b/bin/register_coassembly_sample.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import hashlib +import logging +import os +import xml.etree.ElementTree as ET +from collections import OrderedDict +import re + +import requests +from requests.auth import HTTPBasicAuth +from retry import retry + +logger = logging.getLogger("register_coassembly_sample") + + +RUN_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/{accession}" +SAMPLE_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/{accession}" +SAMPLE_PORTAL_API = "https://www.ebi.ac.uk/ena/portal/api/search" +CHECKLIST_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/ERC000011" + +TEST_SUBMIT_URL = "https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/" +PROD_SUBMIT_URL = "https://www.ebi.ac.uk/ena/submit/drop-box/submit/" + +INSDC_BIOSAMPLE_ACCESSION_REGEX = re.compile( + r"SAM[AG]?[0-9]+" +) +EXISTING_ACCESSION_IN_ERROR_REGEX = re.compile( + r'accession:\s*"([A-Z]+[0-9]+)"', + re.IGNORECASE, +) + + +class CoassemblyRegistrationError(RuntimeError): + """Raised for hard validation/submission errors.""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Register virtual ENA sample for co-assemblies in the metadata CSV") + parser.add_argument("--input", required=True, help="Input assembly metadata CSV") + parser.add_argument("--output", required=True, help="Output assembly metadata CSV") + parser.add_argument( + "--test", + action="store_true", + help="Register sample using the test submission endpoint", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable verbose debug logging", + ) + return parser.parse_args() + + +def setup_logging(debug: bool) -> None: + """Configure logging level and format based on CLI flags.""" + level = logging.DEBUG if debug else logging.INFO + logging.basicConfig(level=level, format="%(levelname)s: %(message)s", force=True) + if debug: + logging.getLogger("requests").setLevel(logging.INFO) + logging.getLogger("urllib3").setLevel(logging.INFO) + logging.getLogger("requests.packages.urllib3").setLevel(logging.INFO) + logger.debug("Debug logging enabled") + + +def get_credentials() -> tuple[str, str]: + username = os.environ.get("ENA_WEBIN", "").strip() + password = os.environ.get("ENA_WEBIN_PASSWORD", "").strip() + if not username or not password: + raise CoassemblyRegistrationError( + "Missing credentials. Set ENA_WEBIN/ENA_WEBIN_PASSWORD." + ) + return username, password + + +@retry(tries=3, delay=2, backoff=2) +def get_xml(url: str) -> ET.Element: + logger.debug(f"Requesting XML URL: {url}") + response = requests.get(url, timeout=60) + response.raise_for_status() + return ET.fromstring(response.content) + +@retry(tries=3, delay=2, backoff=2) +def get_json(url: str, params: dict[str, str]) -> list[dict[str, str]]: + prepared = requests.Request("GET", url, params=params).prepare() + logger.debug(f"Requesting JSON URL: {prepared.url}") + response = requests.get(url, params=params, timeout=60) + response.raise_for_status() + return response.json() + +def to_pretty_xml(root: ET.Element) -> str: + """Serialize an XML element to a pretty-printed UTF-8 XML string.""" + # Re-parse to avoid side effects from in-place indentation on caller-owned trees. + normalized_root = ET.fromstring(ET.tostring(root, encoding="utf-8")) + ET.indent(normalized_root, space=" ") + return ET.tostring(normalized_root, encoding="utf-8", xml_declaration=True).decode("utf-8") + +@retry(tries=3, delay=2, backoff=2) +def submit_sample_xml(auth: HTTPBasicAuth, submit_url: str, sample_xml: str) -> ET.Element: + submission_xml = """\n\n""" + files = { + "SUBMISSION": ("submission.xml", submission_xml, "application/xml"), + "SAMPLE": ("sample.xml", sample_xml, "application/xml"), + } + response = requests.post(submit_url, files=files, auth=auth, timeout=120) + response.raise_for_status() + return ET.fromstring(response.content) + +def merge_or_not_provided(values: list[str]) -> str: + unique = list(OrderedDict.fromkeys(values)) + logger.debug(f"Merging values: {unique}") + if any(not v for v in unique): + logger.info("Merge decision: 'not provided' (at least one value is empty)") + return "not provided" + if len(unique) != 1: + logger.info("Merge decision: 'not provided' (multiple conflicting values)") + return "not provided" + logger.info(f"Merge decision: using unique value '{unique[0]}'") + return unique[0] + +def get_run_sample_from_xml(run_accession: str) -> tuple[str, str | None]: + """ + Retrieve the sample accession from the RUN XML for a given run accession. + + Args: + run_accession (str): The run accession to query. + + Returns: + tuple[str, str | None]: The sample accession and an optional secondary sample accession. + + Raises: + CoassemblyRegistrationError: If the RUN XML does not contain a sample accession. + """ + run_root = get_xml(RUN_XML_API.format(accession=run_accession)) + sample_ref = run_root.find(".//RUN_LINKS/RUN_LINK/XREF_LINK[DB='ENA-SAMPLE']/ID") + + if sample_ref is not None and sample_ref.text: + sample_accession = sample_ref.text.strip() + else: + raise CoassemblyRegistrationError( + f"Run {run_accession} has no sample accession in RUN XML (ENA-SAMPLE link)" + ) + + return sample_accession + + +def get_sample_metadata(sample_accession: str) -> tuple[str, str, str, str]: + """ + Retrieve metadata for a given sample accession from the ENA portal. + + Args: + sample_accession (str): The sample accession to query. + + Returns: + tuple[str, str, str, str]: A tuple containing tax_id, scientific_name, location, and collection_date. + """ + if INSDC_BIOSAMPLE_ACCESSION_REGEX.match(sample_accession): + query = f'"sample_accession={sample_accession}"' + else: + query = f'"secondary_sample_accession={sample_accession}"' + + rows = get_json( + SAMPLE_PORTAL_API, + params={ + "result": "sample", + "query": query, + "fields": "country,collection_date,tax_id,scientific_name", + "format": "json", + }, + ) + + if not rows: + logger.warning(f"No portal metadata found for sample {sample_accession}") + return "", "", "", "" + + row = rows[0] + return ( + row.get("tax_id"), + row.get("scientific_name"), + row.get("country"), + row.get("collection_date"), + ) + +def get_allowed_countries() -> set[str]: + """ + Retrieve the set of allowed countries from the ENA checklist XML. + + Returns: + set[str]: A set of allowed country/sea values. + """ + root = get_xml(CHECKLIST_XML_API) + values = set() + field_nodes = root.findall(".//FIELD") + for field in field_nodes: + label = field.findtext("LABEL") + name = field.findtext("NAME") + if label == "geographic location (country and/or sea)" or name == "geographic_location_country_andor_sea": + for entry in field.findall(".//TEXT_VALUE/VALUE"): + if entry.text: + values.add(entry.text.strip()) + break + return values + +def build_sample_xml( + alias: str, + title: str, + description: str, + taxon_id: str, + scientific_name: str, + composed_of: str, + collection_date: str, + geographic_location: str, +) -> str: + """ + Build the XML representation of a sample for ENA submission. + + Args: + alias (str): The sample alias. + title (str): The sample title. + description (str): The sample description. + taxon_id (str): The taxon ID of the sample. + scientific_name (str): The scientific name of the sample. + composed_of (str): The list of source samples. + collection_date (str): The collection date of the sample. + geographic_location (str): The geographic location of the sample. + + Returns: + str: The XML string representation of the sample. + """ + root = ET.Element("SAMPLE_SET") + sample = ET.SubElement(root, "SAMPLE", {"alias": alias}) + + title_node = ET.SubElement(sample, "TITLE") + title_node.text = title + + sample_name = ET.SubElement(sample, "SAMPLE_NAME") + taxon = ET.SubElement(sample_name, "TAXON_ID") + taxon.text = taxon_id + sci = ET.SubElement(sample_name, "SCIENTIFIC_NAME") + sci.text = scientific_name + + description_node = ET.SubElement(sample, "DESCRIPTION") + description_node.text = description + + attrs = ET.SubElement(sample, "SAMPLE_ATTRIBUTES") + + def add_attr(tag: str, value: str) -> None: + item = ET.SubElement(attrs, "SAMPLE_ATTRIBUTE") + tag_node = ET.SubElement(item, "TAG") + tag_node.text = tag + value_node = ET.SubElement(item, "VALUE") + value_node.text = value + + add_attr("ENA-CHECKLIST", "ERC000011") + add_attr("organism", scientific_name) + add_attr("collection date", collection_date) + add_attr("composed of", composed_of) + add_attr("geographic location (country and/or sea)", geographic_location) + add_attr("scientific_name", scientific_name) + + return to_pretty_xml(root) + + +def build_virtual_sample_alias(source_samples: list[str], max_length: int = 50) -> str: + """Build deterministic alias from source samples with an md5 suffix.""" + sorted_samples = sorted(source_samples) + hash8 = hashlib.md5(",".join(sorted_samples).encode("utf-8")).hexdigest()[:8] + + first_two = sorted_samples[:2] + remaining = len(sorted_samples) - len(first_two) + + alias_core = f"coassembly_{'_'.join(first_two)}" + if remaining > 0: + alias_core = f"{alias_core}_{remaining}_others" + + alias = f"{alias_core}_{hash8}" + if len(alias) <= max_length: + return alias + + # Keep the hash suffix intact while trimming the front portion for ENA alias limits. + # suffix = f"_{hash8}" + # max_core_length = max_length - len(suffix) + # trimmed_core = alias_core[:max_core_length] + # trimmed_alias = f"{trimmed_core}{suffix}" + # logger.warning(f"Generated alias is too long, truncated to: {trimmed_alias}") + # return trimmed_alias + raise CoassemblyRegistrationError( + f"Hash suffix '{alias}' exceeds maximum alias length of {max_length}" + ) + + +def extract_accession_from_receipt(receipt: ET.Element) -> str: + """ + Extract the SAMEA accession from the ENA submission receipt XML. + + Args: + receipt (ET.Element): The root element of the receipt XML. + + Returns: + str: The SAMEA accession. + + Raises: + CoassemblyRegistrationError: If the receipt does not contain a SAMEA accession. + """ + success = receipt.attrib.get("success", "false").lower() == "true" + if not success: + messages = [m.text for m in receipt.findall(".//MESSAGES/*") if m.text] + message_text = "; ".join(messages) if messages else "no error details in receipt" + + # If this is a duplicate submission, ENA returns the existing accession. + existing_match = EXISTING_ACCESSION_IN_ERROR_REGEX.search(message_text) + if existing_match and "already exists" in message_text.lower(): + existing_accession = existing_match.group(1) + logger.info(f"Virtual sample already exists; reusing accession: {existing_accession}") + return existing_accession + + raise CoassemblyRegistrationError( + f"ENA sample submission failed: {message_text}" + ) + + biosample = receipt.find(".//SAMPLE/EXT_ID[@type='biosample']") + if biosample is not None and biosample.get("accession"): + return biosample.get("accession", "") + + external = receipt.find(".//SAMPLE/EXT_ID") + if external is not None and external.get("accession", "").startswith("SAMEA"): + return external.get("accession", "") + + for node in receipt.findall(".//SAMPLE"): + accession = node.get("accession", "") + if accession.startswith("SAMEA"): + return accession + + raise CoassemblyRegistrationError("Could not find SAMEA accession in ENA receipt XML") + +def register_virtual_sample(run_accessions: list[str], test: bool) -> str: + """ + Register a virtual sample for co-assemblies based on run accessions. + + Args: + run_accessions (list[str]): A list of run accessions for the co-assembly. + test (bool): Whether to use the test submission endpoint. + + Returns: + str: The SAMEA accession of the registered virtual sample. + + Raises: + CoassemblyRegistrationError: If validation or submission fails. + """ + if test: + submit_url = TEST_SUBMIT_URL + else: + submit_url = PROD_SUBMIT_URL + + sample_accessions = [] + taxon_ids = [] + scientific_names = [] + locations = [] + collection_dates = [] + + for run in run_accessions: + sample_acc = get_run_sample_from_xml(run) + logger.debug(f"Run {run} is linked to sample {sample_acc}") + sample_accessions.append(sample_acc) + + taxon_id, scientific_name, country, collection_date = get_sample_metadata(sample_acc) + logger.debug(f"Metadata for sample {sample_acc}: tax_id={taxon_id}, scientific_name={scientific_name}, country={country}, collection_date={collection_date}") + if not taxon_id: + raise CoassemblyRegistrationError(f"Sample {sample_acc} has no tax_id in ENA portal response") + if not scientific_name: + raise CoassemblyRegistrationError(f"Sample {sample_acc} has no scientific_name in ENA portal response") + taxon_ids.append(taxon_id) + scientific_names.append(scientific_name) + locations.append(country) + collection_dates.append(collection_date) + + unique_samples = list(OrderedDict.fromkeys(sample_accessions)) + if len(unique_samples) == 1: + logger.info(f"All source runs belong to one sample ({unique_samples[0]}); keeping original row unchanged") + return "" + + unique_taxa = list(OrderedDict.fromkeys([t for t in taxon_ids if t])) + if len(unique_taxa) != 1: + raise CoassemblyRegistrationError( + f"Co-assembly source samples have mixed taxa: {', '.join(unique_taxa)}" + ) + + unique_scientific_names = list(OrderedDict.fromkeys([s for s in scientific_names if s])) + if len(unique_scientific_names) != 1: + raise CoassemblyRegistrationError( + f"Co-assembly source samples have mixed scientific names: {', '.join(unique_scientific_names)}" + ) + + scientific_name = unique_scientific_names[0] + + merged_collection_date = merge_or_not_provided(collection_dates) + merged_location = merge_or_not_provided(locations) + + allowed_countries = get_allowed_countries() + if merged_location != "not provided" and merged_location not in allowed_countries: + logger.warning( + f"Location '{merged_location}' is not in ERC000011 allowed country/sea values" + ) + + sample_list = ",".join(unique_samples) + title = f"Combined sample from {sample_list}" + description = ( + "This sample is a virtual sample of co-assembled raw reads from multiple samples " + f"of {scientific_name}. Co-assembly was performed from runs of the samples {sample_list}" + ) + alias = build_virtual_sample_alias(unique_samples) + logger.info(f"Registering virtual sample with alias '{alias}' for co-assembly of samples: {sample_list}") + + sample_xml = build_sample_xml( + alias=alias, + title=title, + description=description, + taxon_id=unique_taxa[0], + scientific_name=scientific_name, + composed_of=sample_list, + collection_date=merged_collection_date, + geographic_location=merged_location, + ) + + with open(f"{alias}.xml", "w", encoding="utf-8") as sample_file: + sample_file.write(sample_xml) + + username, password = get_credentials() + receipt = submit_sample_xml(HTTPBasicAuth(username, password), submit_url, sample_xml) + samea = extract_accession_from_receipt(receipt) + logger.info(f"Registered virtual co-assembly sample: {samea}") + return samea + + +def main() -> int: + args = parse_args() + setup_logging(args.debug) + + logger.info(f"Starting coassembly registration script: input={args.input}, output={args.output}, test={args.test}") + + with open(args.input, newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + fieldnames = reader.fieldnames or [] + rows = list(reader) + + # The script may add/update the Sample value; ensure the output schema includes it + if "Sample" not in fieldnames: + fieldnames.append("Sample") + + updated_rows: list[dict[str, str]] = [] + for row in rows: + run_accessions = row["Runs"].split(",") + + logger.info(f"Processing assembly with runs: {run_accessions}") + + # Skip non-coassembly rows by design + if len(run_accessions) <= 1: + logger.info("Single run found; skipping coassembly sample registration for this row") + updated_rows.append(row) + continue + + virtual_sample = register_virtual_sample(run_accessions, args.test) + if virtual_sample or virtual_sample == "": + row["Sample"] = virtual_sample + updated_rows.append(row) + + logger.info(f"Writing updated assembly metadata to: {args.output}") + with open(args.output, "w", newline="", encoding="utf-8") as out_handle: + writer = csv.DictWriter(out_handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(updated_rows) + + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: # noqa: BLE001 + logger.error(str(exc)) + raise diff --git a/conf/test_assembly_coassembly_with_coverage.config b/conf/test_assembly_coassembly_with_coverage.config index 910231a..ba3d683 100644 --- a/conf/test_assembly_coassembly_with_coverage.config +++ b/conf/test_assembly_coassembly_with_coverage.config @@ -31,5 +31,4 @@ params { centre_name = "TEST_CENTER" test_upload = true - } diff --git a/modules/local/create_assembly_metadata_csv/main.nf b/modules/local/create_assembly_metadata_csv/main.nf index f3f8312..576f781 100644 --- a/modules/local/create_assembly_metadata_csv/main.nf +++ b/modules/local/create_assembly_metadata_csv/main.nf @@ -28,7 +28,7 @@ process CREATE_ASSEMBLY_METADATA_CSV { meta.assembler, meta.assembler_version, fasta.name, - '' // Sample column left empty because co-assemblies are not supported + '' // Sample column is filled later for co-assemblies that require virtual sample registration ].join(',') """ cat <<-END_CSV > ${meta.id}_assembly_metadata.csv diff --git a/modules/local/register_coassembly_sample/environment.yml b/modules/local/register_coassembly_sample/environment.yml new file mode 100644 index 0000000..66d3d46 --- /dev/null +++ b/modules/local/register_coassembly_sample/environment.yml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::python=3.12.11 + - conda-forge::requests=2.32.5 + - conda-forge::retry=0.9.2 diff --git a/modules/local/register_coassembly_sample/main.nf b/modules/local/register_coassembly_sample/main.nf new file mode 100644 index 0000000..e0d3d8e --- /dev/null +++ b/modules/local/register_coassembly_sample/main.nf @@ -0,0 +1,52 @@ +process REGISTER_COASSEMBLY_SAMPLE { + tag "${meta.id}" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'oras://community.wave.seqera.io/library/pip_requests_retry:d07e257656d938ab' : + 'community.wave.seqera.io/library/pip_requests_retry:eb16563fc2cb641f' }" + + input: + tuple val(meta), path(csv) + val(test) + + output: + tuple val(meta), path("*_updated.csv"), emit: csv + path "versions.yml", emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def test_arg = test ? "--test" : "" + """ + register_coassembly_sample.py \\ + --input ${csv} \\ + --output ${prefix}_updated.csv \\ + ${test_arg} \\ + ${args} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + python: \$(python --version | sed 's/Python //') + requests: \$(python -c "import requests; print(requests.__version__)") + retry: \$(python -c "import importlib.metadata as m; print(m.version('retry'))") + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + cp ${csv} ${prefix}_updated.csv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + python: \$(python --version | sed 's/Python //') + requests: \$(python -c "import requests; print(requests.__version__)") + retry: \$(python -c "import importlib.metadata as m; print(m.version('retry'))") + END_VERSIONS + """ +} diff --git a/modules/local/register_coassembly_sample/meta.yml b/modules/local/register_coassembly_sample/meta.yml new file mode 100644 index 0000000..977b239 --- /dev/null +++ b/modules/local/register_coassembly_sample/meta.yml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "register_coassembly_sample" +description: Register a virtual ENA sample for co-assemblies and write sample accession into assembly metadata CSV +keywords: + - ena + - sample + - metadata + - co-assembly +tools: + - python: + description: Python + homepage: https://www.python.org/ + - retry: + description: Retry decorator for resilient network calls + homepage: https://pypi.org/project/retry/ +input: + - - meta: + type: map + description: Groovy map containing sample metadata + - csv: + type: file + description: Assembly metadata CSV input for assembly_uploader tool + pattern: "*.csv" + - test: + type: string + description: Whether to submit to the ENA test server (`true`) instead of production. +output: + - csv: + - meta: + type: map + description: Same metadata map as input + - "*.csv": + type: file + description: Updated assembly metadata CSV with "Sample" column filled for co-assemblies + pattern: "*.csv" + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@ochkalova" +maintainers: + - "@ochkalova" diff --git a/modules/local/register_coassembly_sample/tests/coassembly_metadata.csv b/modules/local/register_coassembly_sample/tests/coassembly_metadata.csv new file mode 100644 index 0000000..689d97b --- /dev/null +++ b/modules/local/register_coassembly_sample/tests/coassembly_metadata.csv @@ -0,0 +1,11 @@ +Runs,Coverage,Assembler,Version,Filepath +"SRR3183853,SRR3183854,SRR3183855,SRR3183856,SRR3183857,SRR3183858,SRR3183859,SRR3183860,SRR3183861,SRR3183952,SRR3183953,SRR3183954,SRR3183955,SRR3183956,SRR3183957,SRR3183958,SRR3183959,SRR3183960",10.0,MEGAHIT,1.2.9,contigs1.fasta.gz +"SRR3183850,SRR3183851,SRR3183852,SRR3183949,SRR3183950,SRR3183951",12.0,MEGAHIT,1.2.9,contigs2.fasta.gz +"SRR1631070,SRR1631071,SRR1631072,SRR1631073,SRR1631074,SRR1631075,SRR1631076,SRR1631077,SRR1631078,SRR1631079,SRR1631080,SRR1631081,SRR1631082,SRR1631083,SRR3187740,SRR3187741,SRR3187742,SRR3187743,SRR3187744,SRR3187745,SRR3187746,SRR3187747,SRR3187748,SRR3188145,SRR3188146,SRR3188147,SRR3188148,SRR3188149,SRR3188150,SRR3188151,SRR3188152,SRR3188153",15.0,MEGAHIT,1.2.9,contigs3.fasta.gz +"SRR1622826,SRR3187094,SRR3187149",8.0,MEGAHIT,1.2.9,contigs4.fasta.gz +"SRR1622910,SRR1622911",7.0,MEGAHIT,1.2.9,contigs5.fasta.gz +"SRR1624846,SRR1624847,SRR1624848,SRR1624849,SRR1624850,SRR1624851,SRR1624852,SRR1624853,SRR1624854",9.0,MEGAHIT,1.2.9,contigs6.fasta.gz +"SRR1631360,SRR1631361,SRR1631362,SRR1631363,SRR1631364,SRR1631365",6.0,MEGAHIT,1.2.9,contigs9.fasta.gz +"SRR3183597,SRR3183598,SRR3183599,SRR3183600",3.0,MEGAHIT,1.2.9,contigs12.fasta.gz +"SRR3183638,SRR3183639,SRR3183640,SRR3183641,SRR3183642,SRR3183643,SRR3183644,SRR3183645,SRR3183646,SRR3183647",2.0,MEGAHIT,1.2.9,contigs13.fasta.gz +"SRR3183844,SRR3183845,SRR3183846",1.0,MEGAHIT,1.2.9,contigs14.fasta.gz diff --git a/modules/local/register_coassembly_sample/tests/main.nf.test b/modules/local/register_coassembly_sample/tests/main.nf.test new file mode 100644 index 0000000..1a93313 --- /dev/null +++ b/modules/local/register_coassembly_sample/tests/main.nf.test @@ -0,0 +1,36 @@ +nextflow_process { + name "Test Process REGISTER_COASSEMBLY_SAMPLE" + script "../main.nf" + config "./nextflow.config" + process "REGISTER_COASSEMBLY_SAMPLE" + + tag "modules" + tag "register_coassembly_sample" + + test("REGISTER_COASSEMBLY_SAMPLE completes with expected outputs") { + when { + process { + """ + input[0] = [ + [ id:'test' ], + file("${moduleDir}/tests/coassembly_metadata.csv", checkIfExists: true) + ] + input[1] = true + """ + } + } + + then { + assert process.success + assertAll( + { assert process.out.csv.size() == 1 }, + { assert process.out.csv[0][1].toString().endsWith("_updated.csv") }, + { + def csvContent = path(process.out.csv[0][1]).text + assert csvContent.contains("Sample") + assert csvContent.contains("SRR3183853") + } + ) + } + } +} diff --git a/modules/local/register_coassembly_sample/tests/nextflow.config b/modules/local/register_coassembly_sample/tests/nextflow.config new file mode 100644 index 0000000..58a5702 --- /dev/null +++ b/modules/local/register_coassembly_sample/tests/nextflow.config @@ -0,0 +1,6 @@ +process { + withName: 'REGISTER_COASSEMBLY_SAMPLE' { + ext.args = '--debug' + + } +} diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 6d8aae4..97d19d8 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -7,6 +7,7 @@ include { COVERM_CONTIG } from '../modules/nf-core/coverm/contig/main' include { FASTAVALIDATOR } from '../modules/nf-core/fastavalidator/main' include { CREATE_ASSEMBLY_METADATA_CSV } from '../modules/local/create_assembly_metadata_csv/main' +include { REGISTER_COASSEMBLY_SAMPLE } from '../modules/local/register_coassembly_sample/main' include { GENERATE_ASSEMBLY_MANIFEST } from '../modules/local/generate_assembly_manifest/main' include { REGISTERSTUDY } from '../modules/local/registerstudy/main' include { ENA_WEBIN_CLI_WRAPPER as SUBMIT } from '../modules/local/ena_webin_cli_wrapper' @@ -187,9 +188,26 @@ workflow ASSEMBLYSUBMIT { ) ch_versions = ch_versions.mix(CREATE_ASSEMBLY_METADATA_CSV.out.versions) + // For co-assemblies (multiple run accessions), register a virtual ENA sample and fill Sample column. + // Rows with a single run accession bypass this step unchanged. + assembly_metadata_by_type = CREATE_ASSEMBLY_METADATA_CSV.out.csv + .branch { meta, _csv -> + coassembly: meta.run_accession instanceof List && meta.run_accession.size() > 1 + standard: true + } + + REGISTER_COASSEMBLY_SAMPLE( + assembly_metadata_by_type.coassembly, + test_upload + ) + ch_versions = ch_versions.mix(REGISTER_COASSEMBLY_SAMPLE.out.versions) + + assembly_metadata_with_sample = assembly_metadata_by_type.standard + .mix(REGISTER_COASSEMBLY_SAMPLE.out.csv) + // Concatenate assembly metadata CSVs into single file to publish CONCAT_METADATA ( - CREATE_ASSEMBLY_METADATA_CSV.out.csv.map { _meta, file -> file }.collect().map { files -> [ [id: "assemblies_metadata"], files ] }, + assembly_metadata_with_sample.map { _meta, file -> file }.collect().map { files -> [ [id: "assemblies_metadata"], files ] }, 'true' // skip_header - we want to keep the header from the first file and skip it for the rest ) @@ -213,7 +231,7 @@ workflow ASSEMBLYSUBMIT { // Generate assembly manifest files and submit them to ENA GENERATE_ASSEMBLY_MANIFEST( - assemblies_with_coverage.join(CREATE_ASSEMBLY_METADATA_CSV.out.csv), + assemblies_with_coverage.join(assembly_metadata_with_sample), study_accession_ch.first(), upload_tpa, test_upload From b0689c69c3a856e36a57011078cd3d591586a533 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 14 May 2026 14:17:37 +0100 Subject: [PATCH 04/30] minor refactoring of register_coassembly_sample.py Co-authored-by: Copilot --- bin/register_coassembly_sample.py | 45 ++++++++++++++----------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index 8238f50..7a7bfdc 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -7,7 +7,6 @@ import logging import os import xml.etree.ElementTree as ET -from collections import OrderedDict import re import requests @@ -110,7 +109,7 @@ def submit_sample_xml(auth: HTTPBasicAuth, submit_url: str, sample_xml: str) -> return ET.fromstring(response.content) def merge_or_not_provided(values: list[str]) -> str: - unique = list(OrderedDict.fromkeys(values)) + unique = sorted(set(values)) logger.debug(f"Merging values: {unique}") if any(not v for v in unique): logger.info("Merge decision: 'not provided' (at least one value is empty)") @@ -155,7 +154,7 @@ def get_sample_metadata(sample_accession: str) -> tuple[str, str, str, str]: sample_accession (str): The sample accession to query. Returns: - tuple[str, str, str, str]: A tuple containing tax_id, scientific_name, location, and collection_date. + tuple[str, str, str, str]: A tuple containing tax_id, scientific_name, country, and collection_date. """ if INSDC_BIOSAMPLE_ACCESSION_REGEX.match(sample_accession): query = f'"sample_accession={sample_accession}"' @@ -345,7 +344,7 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: test (bool): Whether to use the test submission endpoint. Returns: - str: The SAMEA accession of the registered virtual sample. + str: The sample accession of the registered virtual sample. Raises: CoassemblyRegistrationError: If validation or submission fails. @@ -358,7 +357,7 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: sample_accessions = [] taxon_ids = [] scientific_names = [] - locations = [] + countries = [] collection_dates = [] for run in run_accessions: @@ -374,36 +373,36 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: raise CoassemblyRegistrationError(f"Sample {sample_acc} has no scientific_name in ENA portal response") taxon_ids.append(taxon_id) scientific_names.append(scientific_name) - locations.append(country) + countries.append(country) collection_dates.append(collection_date) - unique_samples = list(OrderedDict.fromkeys(sample_accessions)) + unique_samples = sorted(set(sample_accessions)) if len(unique_samples) == 1: logger.info(f"All source runs belong to one sample ({unique_samples[0]}); keeping original row unchanged") return "" - unique_taxa = list(OrderedDict.fromkeys([t for t in taxon_ids if t])) + unique_taxa = sorted(set([t for t in taxon_ids if t])) if len(unique_taxa) != 1: raise CoassemblyRegistrationError( f"Co-assembly source samples have mixed taxa: {', '.join(unique_taxa)}" ) - unique_scientific_names = list(OrderedDict.fromkeys([s for s in scientific_names if s])) + unique_scientific_names = sorted(set([s for s in scientific_names if s])) if len(unique_scientific_names) != 1: raise CoassemblyRegistrationError( f"Co-assembly source samples have mixed scientific names: {', '.join(unique_scientific_names)}" ) - scientific_name = unique_scientific_names[0] merged_collection_date = merge_or_not_provided(collection_dates) - merged_location = merge_or_not_provided(locations) + merged_country = merge_or_not_provided(countries) allowed_countries = get_allowed_countries() - if merged_location != "not provided" and merged_location not in allowed_countries: + if merged_country != "not provided" and merged_country not in allowed_countries: logger.warning( - f"Location '{merged_location}' is not in ERC000011 allowed country/sea values" + f"Country '{merged_country}' is not in ERC000011 allowed country/sea values; setting to 'not provided'" ) + merged_country = "not provided" sample_list = ",".join(unique_samples) title = f"Combined sample from {sample_list}" @@ -422,7 +421,7 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: scientific_name=scientific_name, composed_of=sample_list, collection_date=merged_collection_date, - geographic_location=merged_location, + geographic_location=merged_country, ) with open(f"{alias}.xml", "w", encoding="utf-8") as sample_file: @@ -430,20 +429,22 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: username, password = get_credentials() receipt = submit_sample_xml(HTTPBasicAuth(username, password), submit_url, sample_xml) - samea = extract_accession_from_receipt(receipt) - logger.info(f"Registered virtual co-assembly sample: {samea}") - return samea + virtual_sample = extract_accession_from_receipt(receipt) + logger.info(f"Registered virtual co-assembly sample: {virtual_sample}") + return virtual_sample def main() -> int: args = parse_args() setup_logging(args.debug) - logger.info(f"Starting coassembly registration script: input={args.input}, output={args.output}, test={args.test}") + logger.debug(f"Starting coassembly registration script: input={args.input}, output={args.output}, test={args.test}") with open(args.input, newline="", encoding="utf-8") as handle: reader = csv.DictReader(handle) fieldnames = reader.fieldnames or [] + if "Runs" not in fieldnames: + raise CoassemblyRegistrationError("Input CSV must have a 'Runs' column with ENA run accessions") rows = list(reader) # The script may add/update the Sample value; ensure the output schema includes it @@ -473,12 +474,6 @@ def main() -> int: writer.writeheader() writer.writerows(updated_rows) - return 0 - if __name__ == "__main__": - try: - raise SystemExit(main()) - except Exception as exc: # noqa: BLE001 - logger.error(str(exc)) - raise + main() From 33f3967a4322ce0789c56d5bee76b306a20895f6 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 14 May 2026 14:34:29 +0100 Subject: [PATCH 05/30] add default metadata values for register_coassembly_sample.py Co-authored-by: Copilot --- bin/register_coassembly_sample.py | 72 ++++++++++++++++++------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index 7a7bfdc..f98b0da 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -38,7 +38,7 @@ class CoassemblyRegistrationError(RuntimeError): def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Register virtual ENA sample for co-assemblies in the metadata CSV") + parser = argparse.ArgumentParser(description="Register co-assembly samples.") parser.add_argument("--input", required=True, help="Input assembly metadata CSV") parser.add_argument("--output", required=True, help="Output assembly metadata CSV") parser.add_argument( @@ -51,6 +51,10 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable verbose debug logging", ) + parser.add_argument("--default-country", type=str, help="Default country to use if values differ or are missing.") + parser.add_argument("--default-date", type=str, help="Default collection date to use if values differ or are missing.") + parser.add_argument("--default-taxid", type=str, help="Default taxon ID to use if values differ or are missing.") + parser.add_argument("--default-tax-name", type=str, help="Default taxon name to use if values differ or are missing.") return parser.parse_args() @@ -108,17 +112,19 @@ def submit_sample_xml(auth: HTTPBasicAuth, submit_url: str, sample_xml: str) -> response.raise_for_status() return ET.fromstring(response.content) -def merge_or_not_provided(values: list[str]) -> str: - unique = sorted(set(values)) - logger.debug(f"Merging values: {unique}") - if any(not v for v in unique): - logger.info("Merge decision: 'not provided' (at least one value is empty)") - return "not provided" - if len(unique) != 1: - logger.info("Merge decision: 'not provided' (multiple conflicting values)") +def merge_or_not_provided(values: list[str], default_value=None) -> str: + """Merge values or return default/not provided.""" + unique_values = sorted(set(values)) + logger.debug(f"Merging values: {unique_values}") + if len(unique_values) == 1 and unique_values[0]: + logger.info(f"Merge decision: using unique value '{unique_values[0]}'") + return unique_values[0] + elif default_value: + logger.info(f"Merge decision: using default value '{default_value}' due to multiple/conflicting values or missing data") + return default_value + else: + logger.info("Merge decision: 'not provided' (multiple conflicting values or some missing)") return "not provided" - logger.info(f"Merge decision: using unique value '{unique[0]}'") - return unique[0] def get_run_sample_from_xml(run_accession: str) -> tuple[str, str | None]: """ @@ -293,16 +299,16 @@ def build_virtual_sample_alias(source_samples: list[str], max_length: int = 50) def extract_accession_from_receipt(receipt: ET.Element) -> str: """ - Extract the SAMEA accession from the ENA submission receipt XML. + Extract the sample accession from the ENA submission receipt XML. Args: receipt (ET.Element): The root element of the receipt XML. Returns: - str: The SAMEA accession. + str: The sample accession. Raises: - CoassemblyRegistrationError: If the receipt does not contain a SAMEA accession. + CoassemblyRegistrationError: If the receipt does not contain a sample accession. """ success = receipt.attrib.get("success", "false").lower() == "true" if not success: @@ -333,9 +339,9 @@ def extract_accession_from_receipt(receipt: ET.Element) -> str: if accession.startswith("SAMEA"): return accession - raise CoassemblyRegistrationError("Could not find SAMEA accession in ENA receipt XML") + raise CoassemblyRegistrationError("Could not find sample accession in ENA receipt XML") -def register_virtual_sample(run_accessions: list[str], test: bool) -> str: +def register_virtual_sample(run_accessions: list[str], test: bool, default_country=None, default_date=None, default_taxid=None, default_tax_name=None) -> str: """ Register a virtual sample for co-assemblies based on run accessions. @@ -381,21 +387,20 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: logger.info(f"All source runs belong to one sample ({unique_samples[0]}); keeping original row unchanged") return "" - unique_taxa = sorted(set([t for t in taxon_ids if t])) - if len(unique_taxa) != 1: + merged_collection_date = merge_or_not_provided(collection_dates, default_date) + merged_country = merge_or_not_provided(countries, default_country) + merged_taxid = merge_or_not_provided(taxon_ids, default_taxid) + merged_tax_name = merge_or_not_provided(scientific_names, default_tax_name) + + if merged_taxid == "not provided": raise CoassemblyRegistrationError( - f"Co-assembly source samples have mixed taxa: {', '.join(unique_taxa)}" + f"Co-assembly source samples have mixed taxa: {', '.join(taxon_ids)}" ) - unique_scientific_names = sorted(set([s for s in scientific_names if s])) - if len(unique_scientific_names) != 1: + if merged_tax_name == "not provided": raise CoassemblyRegistrationError( - f"Co-assembly source samples have mixed scientific names: {', '.join(unique_scientific_names)}" + f"Co-assembly source samples have mixed scientific names: {', '.join(scientific_names)}" ) - scientific_name = unique_scientific_names[0] - - merged_collection_date = merge_or_not_provided(collection_dates) - merged_country = merge_or_not_provided(countries) allowed_countries = get_allowed_countries() if merged_country != "not provided" and merged_country not in allowed_countries: @@ -408,7 +413,7 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: title = f"Combined sample from {sample_list}" description = ( "This sample is a virtual sample of co-assembled raw reads from multiple samples " - f"of {scientific_name}. Co-assembly was performed from runs of the samples {sample_list}" + f"of {merged_tax_name}. Co-assembly was performed from runs of the samples {sample_list}" ) alias = build_virtual_sample_alias(unique_samples) logger.info(f"Registering virtual sample with alias '{alias}' for co-assembly of samples: {sample_list}") @@ -417,8 +422,8 @@ def register_virtual_sample(run_accessions: list[str], test: bool) -> str: alias=alias, title=title, description=description, - taxon_id=unique_taxa[0], - scientific_name=scientific_name, + taxon_id=merged_taxid, + scientific_name=merged_tax_name, composed_of=sample_list, collection_date=merged_collection_date, geographic_location=merged_country, @@ -463,7 +468,14 @@ def main() -> int: updated_rows.append(row) continue - virtual_sample = register_virtual_sample(run_accessions, args.test) + virtual_sample = register_virtual_sample( + run_accessions=run_accessions, + test=args.test, + default_country=args.default_country, + default_date=args.default_date, + default_taxid=args.default_taxid, + default_tax_name=args.default_tax_name + ) if virtual_sample or virtual_sample == "": row["Sample"] = virtual_sample updated_rows.append(row) From 1f738cbaee2cf13e9dbb138d8121b4ec1bacae1c Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 14 May 2026 14:50:57 +0100 Subject: [PATCH 06/30] minor refactoring of register_coassembly_sample.py Co-authored-by: Copilot --- bin/register_coassembly_sample.py | 44 ++++++++++++------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index f98b0da..f3cf212 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -32,6 +32,8 @@ re.IGNORECASE, ) +DEFAULT_NA_VALUE = "not provided" + class CoassemblyRegistrationError(RuntimeError): """Raised for hard validation/submission errors.""" @@ -113,7 +115,7 @@ def submit_sample_xml(auth: HTTPBasicAuth, submit_url: str, sample_xml: str) -> return ET.fromstring(response.content) def merge_or_not_provided(values: list[str], default_value=None) -> str: - """Merge values or return default/not provided.""" + """Merge values or return default_value or DEFAULT_NA_VALUE.""" unique_values = sorted(set(values)) logger.debug(f"Merging values: {unique_values}") if len(unique_values) == 1 and unique_values[0]: @@ -123,8 +125,8 @@ def merge_or_not_provided(values: list[str], default_value=None) -> str: logger.info(f"Merge decision: using default value '{default_value}' due to multiple/conflicting values or missing data") return default_value else: - logger.info("Merge decision: 'not provided' (multiple conflicting values or some missing)") - return "not provided" + logger.info(f"Merge decision: '{DEFAULT_NA_VALUE}' (multiple conflicting values or some missing)") + return DEFAULT_NA_VALUE def get_run_sample_from_xml(run_accession: str) -> tuple[str, str | None]: """ @@ -285,13 +287,6 @@ def build_virtual_sample_alias(source_samples: list[str], max_length: int = 50) if len(alias) <= max_length: return alias - # Keep the hash suffix intact while trimming the front portion for ENA alias limits. - # suffix = f"_{hash8}" - # max_core_length = max_length - len(suffix) - # trimmed_core = alias_core[:max_core_length] - # trimmed_alias = f"{trimmed_core}{suffix}" - # logger.warning(f"Generated alias is too long, truncated to: {trimmed_alias}") - # return trimmed_alias raise CoassemblyRegistrationError( f"Hash suffix '{alias}' exceeds maximum alias length of {max_length}" ) @@ -326,20 +321,14 @@ def extract_accession_from_receipt(receipt: ET.Element) -> str: f"ENA sample submission failed: {message_text}" ) - biosample = receipt.find(".//SAMPLE/EXT_ID[@type='biosample']") - if biosample is not None and biosample.get("accession"): - return biosample.get("accession", "") - - external = receipt.find(".//SAMPLE/EXT_ID") - if external is not None and external.get("accession", "").startswith("SAMEA"): - return external.get("accession", "") + sample_node = receipt.find(".//SAMPLE") + if sample_node is not None: + ena_accession = sample_node.get("accession", "") + if ena_accession: + return ena_accession - for node in receipt.findall(".//SAMPLE"): - accession = node.get("accession", "") - if accession.startswith("SAMEA"): - return accession + raise CoassemblyRegistrationError("ENA sample submission succeeded but no SAMPLE accession found in receipt XML") - raise CoassemblyRegistrationError("Could not find sample accession in ENA receipt XML") def register_virtual_sample(run_accessions: list[str], test: bool, default_country=None, default_date=None, default_taxid=None, default_tax_name=None) -> str: """ @@ -392,22 +381,22 @@ def register_virtual_sample(run_accessions: list[str], test: bool, default_count merged_taxid = merge_or_not_provided(taxon_ids, default_taxid) merged_tax_name = merge_or_not_provided(scientific_names, default_tax_name) - if merged_taxid == "not provided": + if merged_taxid == DEFAULT_NA_VALUE: raise CoassemblyRegistrationError( f"Co-assembly source samples have mixed taxa: {', '.join(taxon_ids)}" ) - if merged_tax_name == "not provided": + if merged_tax_name == DEFAULT_NA_VALUE: raise CoassemblyRegistrationError( f"Co-assembly source samples have mixed scientific names: {', '.join(scientific_names)}" ) allowed_countries = get_allowed_countries() - if merged_country != "not provided" and merged_country not in allowed_countries: + if merged_country != DEFAULT_NA_VALUE and merged_country not in allowed_countries: logger.warning( - f"Country '{merged_country}' is not in ERC000011 allowed country/sea values; setting to 'not provided'" + f"Country '{merged_country}' is not in ERC000011 allowed country/sea values; setting to '{DEFAULT_NA_VALUE}'" ) - merged_country = "not provided" + merged_country = DEFAULT_NA_VALUE sample_list = ",".join(unique_samples) title = f"Combined sample from {sample_list}" @@ -434,6 +423,7 @@ def register_virtual_sample(run_accessions: list[str], test: bool, default_count username, password = get_credentials() receipt = submit_sample_xml(HTTPBasicAuth(username, password), submit_url, sample_xml) + logger.debug(f"ENA receipt XML:\n{ET.tostring(receipt, encoding='unicode')}") virtual_sample = extract_accession_from_receipt(receipt) logger.info(f"Registered virtual co-assembly sample: {virtual_sample}") return virtual_sample From 25d01719bf24cf6fa079eb48726006431b14d002 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 14 May 2026 15:01:41 +0100 Subject: [PATCH 07/30] disable publishDir for REGISTER_COASSEMBLY_SAMPLE --- conf/modules.config | 2 +- modules/local/register_coassembly_sample/tests/nextflow.config | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 931f57e..3a16be0 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -176,7 +176,7 @@ process { ] } - withName: 'REGISTERSTUDY|GENERATE_ASSEMBLY_MANIFEST' { + withName: 'REGISTERSTUDY|GENERATE_ASSEMBLY_MANIFEST|REGISTER_COASSEMBLY_SAMPLE' { publishDir = [ enabled: false ] diff --git a/modules/local/register_coassembly_sample/tests/nextflow.config b/modules/local/register_coassembly_sample/tests/nextflow.config index 58a5702..ea49989 100644 --- a/modules/local/register_coassembly_sample/tests/nextflow.config +++ b/modules/local/register_coassembly_sample/tests/nextflow.config @@ -1,6 +1,5 @@ process { withName: 'REGISTER_COASSEMBLY_SAMPLE' { ext.args = '--debug' - } } From 29a6b8c273a2f90ae50c1174f037f14c19923a59 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Mon, 10 Aug 2026 13:31:13 +0100 Subject: [PATCH 08/30] update samplesheet header --- assets/samplesheet_assembly_coassembly_no_coverage.csv | 2 +- assets/samplesheet_assembly_coassembly_with_coverage.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/samplesheet_assembly_coassembly_no_coverage.csv b/assets/samplesheet_assembly_coassembly_no_coverage.csv index db29939..8c7cd3f 100644 --- a/assets/samplesheet_assembly_coassembly_no_coverage.csv +++ b/assets/samplesheet_assembly_coassembly_no_coverage.csv @@ -1,2 +1,2 @@ -sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version no_coverage_paired_reads,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz,,SRR3183850;SRR3183951,SPAdes,3.15 diff --git a/assets/samplesheet_assembly_coassembly_with_coverage.csv b/assets/samplesheet_assembly_coassembly_with_coverage.csv index cf9cab3..1a459d0 100644 --- a/assets/samplesheet_assembly_coassembly_with_coverage.csv +++ b/assets/samplesheet_assembly_coassembly_with_coverage.csv @@ -1,2 +1,2 @@ -sample,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,SRR3183850;SRR3183951,SPAdes,3.15 From afd7ae187849d51efa505b782524d873c911a224 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Mon, 10 Aug 2026 13:58:44 +0100 Subject: [PATCH 09/30] append hash to co assembly sample if run in --test mode --- bin/register_coassembly_sample.py | 32 ++++++++++++++++++------------- workflows/assemblysubmit.nf | 1 - 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index f3cf212..1fbf071 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -6,6 +6,7 @@ import hashlib import logging import os +import time import xml.etree.ElementTree as ET import re @@ -117,7 +118,7 @@ def submit_sample_xml(auth: HTTPBasicAuth, submit_url: str, sample_xml: str) -> def merge_or_not_provided(values: list[str], default_value=None) -> str: """Merge values or return default_value or DEFAULT_NA_VALUE.""" unique_values = sorted(set(values)) - logger.debug(f"Merging values: {unique_values}") + logger.info(f"Merging values: {unique_values}") if len(unique_values) == 1 and unique_values[0]: logger.info(f"Merge decision: using unique value '{unique_values[0]}'") return unique_values[0] @@ -271,19 +272,21 @@ def add_attr(tag: str, value: str) -> None: return to_pretty_xml(root) -def build_virtual_sample_alias(source_samples: list[str], max_length: int = 50) -> str: - """Build deterministic alias from source samples with an md5 suffix.""" +def build_virtual_sample_alias(source_samples: list[str], max_length: int = 50, test: bool = False) -> str: + """Build deterministic alias from source samples with an md5 suffix. + + When `test` is set, a short timestamp-based hash is appended so repeated + test submissions don't collide on alias uniqueness. + """ sorted_samples = sorted(source_samples) - hash8 = hashlib.md5(",".join(sorted_samples).encode("utf-8")).hexdigest()[:8] + samples_hash = hashlib.md5(",".join(sorted_samples).encode("utf-8")).hexdigest()[:8] - first_two = sorted_samples[:2] - remaining = len(sorted_samples) - len(first_two) + alias = f"coassembly_{len(sorted_samples)}_samples_{samples_hash}" - alias_core = f"coassembly_{'_'.join(first_two)}" - if remaining > 0: - alias_core = f"{alias_core}_{remaining}_others" + if test: + timestamp_hash = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()[:6] + alias = f"{alias}_{timestamp_hash}" - alias = f"{alias_core}_{hash8}" if len(alias) <= max_length: return alias @@ -357,11 +360,11 @@ def register_virtual_sample(run_accessions: list[str], test: bool, default_count for run in run_accessions: sample_acc = get_run_sample_from_xml(run) - logger.debug(f"Run {run} is linked to sample {sample_acc}") + logger.info(f"Run {run} is linked to sample {sample_acc}") sample_accessions.append(sample_acc) taxon_id, scientific_name, country, collection_date = get_sample_metadata(sample_acc) - logger.debug(f"Metadata for sample {sample_acc}: tax_id={taxon_id}, scientific_name={scientific_name}, country={country}, collection_date={collection_date}") + logger.info(f"Metadata for sample {sample_acc}: tax_id={taxon_id}, scientific_name={scientific_name}, country={country}, collection_date={collection_date}") if not taxon_id: raise CoassemblyRegistrationError(f"Sample {sample_acc} has no tax_id in ENA portal response") if not scientific_name: @@ -404,7 +407,7 @@ def register_virtual_sample(run_accessions: list[str], test: bool, default_count "This sample is a virtual sample of co-assembled raw reads from multiple samples " f"of {merged_tax_name}. Co-assembly was performed from runs of the samples {sample_list}" ) - alias = build_virtual_sample_alias(unique_samples) + alias = build_virtual_sample_alias(unique_samples, test=test) logger.info(f"Registering virtual sample with alias '{alias}' for co-assembly of samples: {sample_list}") sample_xml = build_sample_xml( @@ -435,6 +438,9 @@ def main() -> int: logger.debug(f"Starting coassembly registration script: input={args.input}, output={args.output}, test={args.test}") + if args.test: + logger.info("Running in test mode: submitting to the ENA test server. Submission will be discarded after 24 hours.") + with open(args.input, newline="", encoding="utf-8") as handle: reader = csv.DictReader(handle) fieldnames = reader.fieldnames or [] diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 57bcc30..8728957 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -188,7 +188,6 @@ workflow ASSEMBLYSUBMIT { assembly_metadata_by_type.coassembly, test_upload ) - ch_versions = ch_versions.mix(REGISTER_COASSEMBLY_SAMPLE.out.versions) assembly_metadata_with_sample = assembly_metadata_by_type.standard .mix(REGISTER_COASSEMBLY_SAMPLE.out.csv) From a9155e98f52f9915f6240ec91020efab3b37527b Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Mon, 10 Aug 2026 16:17:16 +0100 Subject: [PATCH 10/30] add saveguard against duplicated accession in run_accession column --- workflows/assemblysubmit.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 8728957..c32733f 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -53,7 +53,7 @@ workflow ASSEMBLYSUBMIT { assembly_fasta = ch_samplesheet .map { meta, fasta, reads_1, reads_2 -> // support semicolon-separated values for co-assemblies (e.g. ERR000001;ERR000002) - def run_accessions = meta.run_accession.split(';')?.toList() ?: [] + def run_accessions = meta.run_accession.split(';')?.toList()?.unique() ?: [] def new_meta = meta + [ single_end: !reads_2, run_accession: run_accessions.size() == 1 ? run_accessions[0] : run_accessions @@ -66,7 +66,7 @@ workflow ASSEMBLYSUBMIT { .filter { row -> row[2] && row[2] != "" } // Check if fastq_1 exists and is not empty .map { meta, fasta, reads_1, reads_2 -> // support semicolon-separated values for co-assemblies (e.g. ERR000001;ERR000002) - def run_accessions = meta.run_accession.split(';')?.toList() ?: [] + def run_accessions = meta.run_accession.split(';')?.toList()?.unique() ?: [] def new_meta = meta + [ single_end: !reads_2, run_accession: run_accessions.size() == 1 ? run_accessions[0] : run_accessions From a8e81a22a480e27bdf53d50f60d4372c8fe34604 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 15 Oct 2026 14:46:15 +0200 Subject: [PATCH 11/30] add co assembly support for coverage calculation --- ..._assembly_big_coassembly_with_coverage.csv | 2 + conf/modules.config | 1 + docs/methods.md | 12 ++- .../utils_nfcore_seqsubmit_pipeline/main.nf | 28 +++++++ .../fixtures/coverm_header_only.depth.tsv | 1 + .../tests/fixtures/coverm_no_length.depth.tsv | 2 + .../fixtures/coverm_single_run.depth.tsv | 3 + .../fixtures/coverm_three_runs.depth.tsv | 4 + .../tests/main.function.nf.test | 84 +++++++++++++++++++ workflows/assemblysubmit.nf | 13 +-- 10 files changed, 137 insertions(+), 13 deletions(-) create mode 100644 assets/samplesheet_assembly_big_coassembly_with_coverage.csv create mode 100644 subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_header_only.depth.tsv create mode 100644 subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_no_length.depth.tsv create mode 100644 subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_single_run.depth.tsv create mode 100644 subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_three_runs.depth.tsv create mode 100644 subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test diff --git a/assets/samplesheet_assembly_big_coassembly_with_coverage.csv b/assets/samplesheet_assembly_big_coassembly_with_coverage.csv new file mode 100644 index 0000000..254dced --- /dev/null +++ b/assets/samplesheet_assembly_big_coassembly_with_coverage.csv @@ -0,0 +1,2 @@ +id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version +big_coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,SRR3187093;SRR3187094;SRR3187095;SRR3187096;SRR3187097;SRR3187098;SRR3187099;SRR3187100;SRR3187101;SRR3187102;SRR3187103;SRR3187104;SRR3187105;SRR3187106;SRR3187107;SRR3187108;SRR3187109;SRR3187110;SRR3187111;SRR3187112;SRR3187113;SRR3187114;SRR3187115;SRR3187116;SRR3187117;SRR3187118;SRR3187119;SRR3187120;SRR3187121;SRR3187122;SRR3187123;SRR3187124;SRR3187125;SRR3187126;SRR3187127;SRR3187128;SRR3187129;SRR3187130;SRR3187131;SRR3187132;SRR3187133;SRR3187134;SRR3187135;SRR3187136;SRR3187137;SRR3187138;SRR3187139;SRR3187140;SRR3187141;SRR3187142;SRR3187143,SPAdes,3.15 diff --git a/conf/modules.config b/conf/modules.config index 0c6cf5b..ed8b196 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -133,6 +133,7 @@ process { } withName: 'COVERM_CONTIG' { + ext.args = '--methods mean length' publishDir = [ path: { "${params.outdir}/${params.mode}/${meta.id}/coverage" }, mode: params.publish_dir_mode, diff --git a/docs/methods.md b/docs/methods.md index 1504fa1..019a720 100644 --- a/docs/methods.md +++ b/docs/methods.md @@ -24,14 +24,18 @@ Only assemblies with successful validation are forwarded to coverage estimation, If the `coverage` column is already populated in the samplesheet, that value is used directly. -If `coverage` is missing, the workflow calculates coverage with `coverm contig`. +If `coverage` is missing, the workflow maps the reads of all runs listed for the assembly with `coverm contig`, which reports the mean depth of each contig for each run. -`coverm contig` outputs per-contig depth. The workflow then reads this file and calculates the arithmetic mean across all contigs. +Because a co-assembly is built from the pooled reads of all runs, the depth of contig $c$ is the sum of its depths across the $N$ runs: -If the per-contig coverage values are $c_1, c_2, \ldots, c_n$, the workflow currently computes assembly coverage as an unweighted mean across contigs: +$$ +D_c = \sum_{i=1}^{N} D_{c,i} +$$ + +The assembly coverage is the mean of these per-contig depths, weighted by contig length $L_c$: $$ -\bar{c} = \frac{1}{n} \sum_{i=1}^{n} c_i +D_{\text{assembly}} = \frac{\sum_{c} L_c \cdot D_c}{\sum_{c} L_c} $$ ## `GENOMESUBMIT` diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf index c6dea44..95b20c6 100644 --- a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf @@ -191,6 +191,34 @@ def validateInputSamplesheet(input) { return [ metas[0], fastqs ] } +// +// Calculate co-assembly coverage from a `coverm contig --methods mean length` table. +// Rows are contigs; each read set contributes one ` Mean` and one ` Length` column. +// Per-contig depth is the sum of the `Mean` columns (reads from all runs were pooled for the +// co-assembly). Assembly depth is the length-weighted mean of the per-contig depths: +// sum_c(L_c * sum_i(D_c,i)) / sum_c(L_c) +// +def calculateCoassemblyCoverage(coverage_file) { + double weighted_depth = 0d + double total_length = 0d + + coverage_file.splitCsv(sep: '\t', header: true).each { row -> + def mean_columns = row.findAll { key, _value -> key.endsWith(' Mean') } + def length_column = row.find { key, _value -> key.endsWith(' Length') } + if (!mean_columns || !length_column) { + error("CoverM output ${coverage_file.name} must contain ' Mean' and ' Length' columns. Run 'coverm contig' with '--methods mean length'.") + } + + def depth = mean_columns.values().sum { value -> value as Double } + def length = length_column.value as Double + + weighted_depth += depth * length + total_length += length + } + + return total_length > 0 ? weighted_depth / total_length : 0d +} + // // Generate methods description for MultiQC // diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_header_only.depth.tsv b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_header_only.depth.tsv new file mode 100644 index 0000000..4daac9d --- /dev/null +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_header_only.depth.tsv @@ -0,0 +1 @@ +Contig contigs.fasta/run1_1.fastq.gz Mean contigs.fasta/run1_1.fastq.gz Length diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_no_length.depth.tsv b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_no_length.depth.tsv new file mode 100644 index 0000000..b140cc0 --- /dev/null +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_no_length.depth.tsv @@ -0,0 +1,2 @@ +Contig contigs.fasta/run1_1.fastq.gz Mean +contig_1 10 diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_single_run.depth.tsv b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_single_run.depth.tsv new file mode 100644 index 0000000..e791392 --- /dev/null +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_single_run.depth.tsv @@ -0,0 +1,3 @@ +Contig contigs.fasta/run1_1.fastq.gz Mean contigs.fasta/run1_1.fastq.gz Length +contig_1 10 1000 +contig_2 2 100 diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_three_runs.depth.tsv b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_three_runs.depth.tsv new file mode 100644 index 0000000..659598b --- /dev/null +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_three_runs.depth.tsv @@ -0,0 +1,4 @@ +Contig contigs.fasta/run1_1.fastq.gz Mean contigs.fasta/run1_1.fastq.gz Length contigs.fasta/run2_1.fastq.gz Mean contigs.fasta/run2_1.fastq.gz Length contigs.fasta/run3_1.fastq.gz Mean contigs.fasta/run3_1.fastq.gz Length +contig_1 4 1000 3 1000 3 1000 +contig_2 1 3000 1 3000 0 3000 +contig_3 0 500 0 500 0 500 diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..9299089 --- /dev/null +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test @@ -0,0 +1,84 @@ +nextflow_function { + + name "Test calculateCoassemblyCoverage" + script "../main.nf" + function "calculateCoassemblyCoverage" + tag "functions" + tag "calculateCoassemblyCoverage" + tag "mode_assembly" + + test("Single read set - length-weighted mean of contig depths") { + + when { + function { + """ + input[0] = file("${moduleDir}/tests/fixtures/coverm_single_run.depth.tsv") + """ + } + } + + then { + // (10 * 1000 + 2 * 100) / (1000 + 100) + assertAll( + { assert function.success }, + { assert Math.abs(function.result - 10200d / 1100d) < 1e-9 } + ) + } + } + + test("Several read sets - depths summed per contig, zero-depth contigs kept") { + + when { + function { + """ + input[0] = file("${moduleDir}/tests/fixtures/coverm_three_runs.depth.tsv") + """ + } + } + + then { + // contig_1: 4 + 3 + 3 = 10, contig_2: 1 + 1 + 0 = 2, contig_3: 0 + // (10 * 1000 + 2 * 3000 + 0 * 500) / (1000 + 3000 + 500) + assertAll( + { assert function.success }, + { assert Math.abs(function.result - 16000d / 4500d) < 1e-9 } + ) + } + } + + test("No contigs in CoverM output - coverage is 0") { + + when { + function { + """ + input[0] = file("${moduleDir}/tests/fixtures/coverm_header_only.depth.tsv") + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == 0d } + ) + } + } + + test("CoverM run without '--methods mean length' - fails with a clear error") { + + when { + function { + """ + input[0] = file("${moduleDir}/tests/fixtures/coverm_no_length.depth.tsv") + """ + } + } + + then { + assertAll( + { assert function.failed }, + { assert function.stdout.any { it.contains("--methods mean length") } } + ) + } + } +} diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index c32733f..9ad6031 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -21,6 +21,7 @@ include { softwareVersionsToYAML } from '../subworkflows/nf-core/ include { FASTA_VALIDATION } from '../subworkflows/local/fasta_validation/main' include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_seqsubmit_pipeline' +include { calculateCoassemblyCoverage } from '../subworkflows/local/utils_nfcore_seqsubmit_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -143,16 +144,10 @@ workflow ASSEMBLYSUBMIT { false // enable_bam_output ) - // Calculate average coverage using splitCsv operator + // Length-weighted co-assembly coverage (see docs/methods.md) average_coverage_ch = COVERM_CONTIG.out.coverage - .splitCsv(sep: '\t', skip: 1) - .map { meta, row -> - [meta, row[1] as Double] - } - .groupTuple() - .map { meta, coverages -> - def average = coverages.sum() / coverages.size() - [meta, average] + .map { meta, coverage_tsv -> + [ meta, calculateCoassemblyCoverage(coverage_tsv) ] } // Update metadata with calculated coverage From 9210a17d13b96b75a01d8c84d892c4cae4ec7938 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 15 Oct 2026 15:48:33 +0200 Subject: [PATCH 12/30] add snapshot --- ...sembly_coassembly_no_coverage.nf.test.snap | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/assembly_coassembly_no_coverage.nf.test.snap diff --git a/tests/assembly_coassembly_no_coverage.nf.test.snap b/tests/assembly_coassembly_no_coverage.nf.test.snap new file mode 100644 index 0000000..31342e2 --- /dev/null +++ b/tests/assembly_coassembly_no_coverage.nf.test.snap @@ -0,0 +1,78 @@ +{ + "-profile test_assembly_coassembly_no_coverage": { + "content": [ + 9, + { + "CONCAT_ACCESSIONS": { + "coreutils": 9.4, + "find": "4.6.0", + "pigz": 2.8 + }, + "CONCAT_METADATA": { + "coreutils": 9.4, + "find": "4.6.0", + "pigz": 2.8 + }, + "COVERM_CONTIG": { + "coverm": "0.7.0" + }, + "CREATE_ASSEMBLY_METADATA_CSV": { + "bash": "5.2.37(1)-release" + }, + "FALINT": { + "falint": "1.2.0" + }, + "GENERATE_ASSEMBLY_MANIFEST": { + "assembly_uploader": "1.3.5" + }, + "SUBMIT": { + "ena-webin-cli": "9.0.3", + "mgnify-pipelines-toolkit": "1.5.1", + "python": "3.12.13" + }, + "Workflow": { + "nf-core/seqsubmit": "v1.0.0" + } + }, + [ + "metagenomic_assemblies", + "metagenomic_assemblies/no_coverage_paired_reads", + "metagenomic_assemblies/no_coverage_paired_reads/coverage", + "metagenomic_assemblies/no_coverage_paired_reads/coverage/no_coverage_paired_reads.depth.tsv", + "metagenomic_assemblies/no_coverage_paired_reads/upload", + "metagenomic_assemblies/no_coverage_paired_reads/upload/webin_cli", + "metagenomic_assemblies/no_coverage_paired_reads/upload/webin_cli/no_coverage_paired_reads_accessions.tsv", + "metagenomic_assemblies/upload", + "metagenomic_assemblies/upload/assemblies_accessions.tsv", + "metagenomic_assemblies/upload/assemblies_metadata.csv", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_Assemblies assigned accessions.txt", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" + ], + [ + "no_coverage_paired_reads.depth.tsv:md5,6e61f10c995d45b15f33d572d773e80e", + "assemblies_metadata.csv:md5,22f1108ab2a35170ee1fe86954e7fd3d", + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-15T15:17:34.623978", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.1" + } + } +} \ No newline at end of file From 4461b2dd89c5558f13c0d7ee0d9209cd89ebc776 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Fri, 16 Oct 2026 11:55:24 +0200 Subject: [PATCH 13/30] add private data support for cooassembly sample registration --- bin/register_coassembly_sample.py | 99 +++++++++++++------ .../local/register_coassembly_sample/main.nf | 3 + .../tests/main.nf.test | 1 + workflows/assemblysubmit.nf | 3 +- 4 files changed, 75 insertions(+), 31 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index 1fbf071..37c9cc0 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -17,9 +17,7 @@ logger = logging.getLogger("register_coassembly_sample") -RUN_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/{accession}" -SAMPLE_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/{accession}" -SAMPLE_PORTAL_API = "https://www.ebi.ac.uk/ena/portal/api/search" +PORTAL_SEARCH_API = "https://www.ebi.ac.uk/ena/portal/api/search" CHECKLIST_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/ERC000011" TEST_SUBMIT_URL = "https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/" @@ -49,6 +47,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Register sample using the test submission endpoint", ) + parser.add_argument( + "--private", + action="store_true", + help="Source runs/samples are private: query the ENA Portal API with Webin auth (ENA_WEBIN/ENA_WEBIN_PASSWORD)", + ) parser.add_argument( "--debug", action="store_true", @@ -90,12 +93,39 @@ def get_xml(url: str) -> ET.Element: return ET.fromstring(response.content) @retry(tries=3, delay=2, backoff=2) -def get_json(url: str, params: dict[str, str]) -> list[dict[str, str]]: +def get_json(url: str, params: dict[str, str], auth: HTTPBasicAuth | None = None) -> list[dict[str, str]]: prepared = requests.Request("GET", url, params=params).prepare() - logger.debug(f"Requesting JSON URL: {prepared.url}") - response = requests.get(url, params=params, timeout=60) + logger.debug(f"Requesting JSON URL ({'with' if auth else 'without'} Webin auth): {prepared.url}") + response = requests.get(url, params=params, auth=auth, timeout=60) response.raise_for_status() - return response.json() + if not response.text.strip(): + return [] + result = response.json() + if isinstance(result, dict) and "message" in result: + raise CoassemblyRegistrationError(f"ENA Portal API error: {result['message']}") + return result + + +def search_portal(result: str, query: str, fields: str, private: bool = False) -> list[dict[str, str]]: + """ + Query the ENA Portal API. + + When `private` is set, the request is authenticated with the Webin account + (ENA_WEBIN/ENA_WEBIN_PASSWORD), which also returns private records owned by it. + """ + params = {"result": result, "query": query, "fields": fields, "format": "json"} + if not private: + return get_json(PORTAL_SEARCH_API, params=params) + + username, password = get_credentials() + try: + return get_json(PORTAL_SEARCH_API, params=params, auth=HTTPBasicAuth(username, password)) + except requests.HTTPError as exc: + # ENA returns a 500 (not 401) for invalid Webin credentials, so a 500 is ambiguous + raise CoassemblyRegistrationError( + f"Authenticated ENA Portal API search failed ({exc}). This may be a temporary ENA API " + "issue, or invalid Webin credentials - check ENA_WEBIN/ENA_WEBIN_PASSWORD." + ) from exc def to_pretty_xml(root: ET.Element) -> str: """Serialize an XML element to a pretty-printed UTF-8 XML string.""" @@ -129,38 +159,48 @@ def merge_or_not_provided(values: list[str], default_value=None) -> str: logger.info(f"Merge decision: '{DEFAULT_NA_VALUE}' (multiple conflicting values or some missing)") return DEFAULT_NA_VALUE -def get_run_sample_from_xml(run_accession: str) -> tuple[str, str | None]: +def get_run_sample(run_accession: str, private: bool = False) -> str: """ - Retrieve the sample accession from the RUN XML for a given run accession. + Retrieve the sample accession linked to a run from the ENA Portal API. + Use Webin auth to retrieve private records. Args: run_accession (str): The run accession to query. + private (bool): Query with Webin auth to access private data. Returns: - tuple[str, str | None]: The sample accession and an optional secondary sample accession. + str: The secondary sample accession (e.g. ERS/SRS), or the BioSample + accession if no secondary accession is available. Raises: - CoassemblyRegistrationError: If the RUN XML does not contain a sample accession. + CoassemblyRegistrationError: If the run is not found or has no linked sample. """ - run_root = get_xml(RUN_XML_API.format(accession=run_accession)) - sample_ref = run_root.find(".//RUN_LINKS/RUN_LINK/XREF_LINK[DB='ENA-SAMPLE']/ID") - - if sample_ref is not None and sample_ref.text: - sample_accession = sample_ref.text.strip() - else: + rows = search_portal( + result="read_run", + query=f'"run_accession={run_accession}"', + fields="sample_accession,secondary_sample_accession", + private=private, + ) + if not rows: raise CoassemblyRegistrationError( - f"Run {run_accession} has no sample accession in RUN XML (ENA-SAMPLE link)" + f"Run {run_accession} not found in ENA" + ("" if private else " (use --private for private data)") ) + row = rows[0] + sample_accession = (row.get("secondary_sample_accession") or row.get("sample_accession") or "").strip() + if not sample_accession: + raise CoassemblyRegistrationError(f"Run {run_accession} has no linked sample accession in ENA") + return sample_accession -def get_sample_metadata(sample_accession: str) -> tuple[str, str, str, str]: +def get_sample_metadata(sample_accession: str, private: bool = False) -> tuple[str, str, str, str]: """ Retrieve metadata for a given sample accession from the ENA portal. Args: sample_accession (str): The sample accession to query. + private (bool): Query with Webin auth to access private data. Returns: tuple[str, str, str, str]: A tuple containing tax_id, scientific_name, country, and collection_date. @@ -170,14 +210,11 @@ def get_sample_metadata(sample_accession: str) -> tuple[str, str, str, str]: else: query = f'"secondary_sample_accession={sample_accession}"' - rows = get_json( - SAMPLE_PORTAL_API, - params={ - "result": "sample", - "query": query, - "fields": "country,collection_date,tax_id,scientific_name", - "format": "json", - }, + rows = search_portal( + result="sample", + query=query, + fields="country,collection_date,tax_id,scientific_name", + private=private, ) if not rows: @@ -333,13 +370,14 @@ def extract_accession_from_receipt(receipt: ET.Element) -> str: raise CoassemblyRegistrationError("ENA sample submission succeeded but no SAMPLE accession found in receipt XML") -def register_virtual_sample(run_accessions: list[str], test: bool, default_country=None, default_date=None, default_taxid=None, default_tax_name=None) -> str: +def register_virtual_sample(run_accessions: list[str], test: bool, private: bool = False, default_country=None, default_date=None, default_taxid=None, default_tax_name=None) -> str: """ Register a virtual sample for co-assemblies based on run accessions. Args: run_accessions (list[str]): A list of run accessions for the co-assembly. test (bool): Whether to use the test submission endpoint. + private (bool): Whether source runs/samples are private (query ENA with Webin auth). Returns: str: The sample accession of the registered virtual sample. @@ -359,11 +397,11 @@ def register_virtual_sample(run_accessions: list[str], test: bool, default_count collection_dates = [] for run in run_accessions: - sample_acc = get_run_sample_from_xml(run) + sample_acc = get_run_sample(run, private=private) logger.info(f"Run {run} is linked to sample {sample_acc}") sample_accessions.append(sample_acc) - taxon_id, scientific_name, country, collection_date = get_sample_metadata(sample_acc) + taxon_id, scientific_name, country, collection_date = get_sample_metadata(sample_acc, private=private) logger.info(f"Metadata for sample {sample_acc}: tax_id={taxon_id}, scientific_name={scientific_name}, country={country}, collection_date={collection_date}") if not taxon_id: raise CoassemblyRegistrationError(f"Sample {sample_acc} has no tax_id in ENA portal response") @@ -467,6 +505,7 @@ def main() -> int: virtual_sample = register_virtual_sample( run_accessions=run_accessions, test=args.test, + private=args.private, default_country=args.default_country, default_date=args.default_date, default_taxid=args.default_taxid, diff --git a/modules/local/register_coassembly_sample/main.nf b/modules/local/register_coassembly_sample/main.nf index e0d3d8e..f86b3a6 100644 --- a/modules/local/register_coassembly_sample/main.nf +++ b/modules/local/register_coassembly_sample/main.nf @@ -10,6 +10,7 @@ process REGISTER_COASSEMBLY_SAMPLE { input: tuple val(meta), path(csv) val(test) + val(is_private) output: tuple val(meta), path("*_updated.csv"), emit: csv @@ -22,11 +23,13 @@ process REGISTER_COASSEMBLY_SAMPLE { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def test_arg = test ? "--test" : "" + def private_arg = is_private ? "--private" : "" """ register_coassembly_sample.py \\ --input ${csv} \\ --output ${prefix}_updated.csv \\ ${test_arg} \\ + ${private_arg} \\ ${args} cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/register_coassembly_sample/tests/main.nf.test b/modules/local/register_coassembly_sample/tests/main.nf.test index 1a93313..51df911 100644 --- a/modules/local/register_coassembly_sample/tests/main.nf.test +++ b/modules/local/register_coassembly_sample/tests/main.nf.test @@ -16,6 +16,7 @@ nextflow_process { file("${moduleDir}/tests/coassembly_metadata.csv", checkIfExists: true) ] input[1] = true + input[2] = false """ } } diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 9ad6031..4fb5b1f 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -181,7 +181,8 @@ workflow ASSEMBLYSUBMIT { REGISTER_COASSEMBLY_SAMPLE( assembly_metadata_by_type.coassembly, - test_upload + test_upload, + is_private ) assembly_metadata_with_sample = assembly_metadata_by_type.standard From 06c5ffcc84f204977ebda1e719ae629246b11954 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Fri, 16 Oct 2026 12:01:29 +0200 Subject: [PATCH 14/30] revert test_upload changes in assembly test configs Co-Authored-By: Claude Opus 5.5 --- conf/test_assembly_complete_metadata.config | 1 - conf/test_assembly_no_coverage_paired_reads.config | 1 - conf/test_assembly_no_coverage_single_reads.config | 1 - conf/test_assembly_no_study_complete_metadata.config | 1 + conf/test_assembly_one_contig.config | 1 - 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/conf/test_assembly_complete_metadata.config b/conf/test_assembly_complete_metadata.config index ebcdd40..801cab7 100644 --- a/conf/test_assembly_complete_metadata.config +++ b/conf/test_assembly_complete_metadata.config @@ -30,5 +30,4 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" - test_upload = true } diff --git a/conf/test_assembly_no_coverage_paired_reads.config b/conf/test_assembly_no_coverage_paired_reads.config index 4571a1c..65c73e4 100644 --- a/conf/test_assembly_no_coverage_paired_reads.config +++ b/conf/test_assembly_no_coverage_paired_reads.config @@ -30,5 +30,4 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" - test_upload = true } diff --git a/conf/test_assembly_no_coverage_single_reads.config b/conf/test_assembly_no_coverage_single_reads.config index ee269e9..814de31 100644 --- a/conf/test_assembly_no_coverage_single_reads.config +++ b/conf/test_assembly_no_coverage_single_reads.config @@ -30,5 +30,4 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" - test_upload = true } diff --git a/conf/test_assembly_no_study_complete_metadata.config b/conf/test_assembly_no_study_complete_metadata.config index ecb958e..ac83a22 100644 --- a/conf/test_assembly_no_study_complete_metadata.config +++ b/conf/test_assembly_no_study_complete_metadata.config @@ -31,4 +31,5 @@ params { centre_name = "TEST_CENTER" test_upload = true + } diff --git a/conf/test_assembly_one_contig.config b/conf/test_assembly_one_contig.config index ce2e86c..b27784e 100644 --- a/conf/test_assembly_one_contig.config +++ b/conf/test_assembly_one_contig.config @@ -30,5 +30,4 @@ params { submission_study = "PRJEB98843" centre_name = "TEST_CENTER" - test_upload = true } From 8eb09b5cda7c0cb9a1f2c47b7b42345883daf307 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Fri, 16 Oct 2026 13:04:54 +0200 Subject: [PATCH 15/30] move samplesheets to test datasets folder --- assets/samplesheet_assembly_big_coassembly_with_coverage.csv | 2 -- assets/samplesheet_assembly_coassembly_no_coverage.csv | 2 -- assets/samplesheet_assembly_coassembly_with_coverage.csv | 2 -- conf/test_assembly_coassembly_no_coverage.config | 2 +- conf/test_assembly_coassembly_with_coverage.config | 2 +- 5 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 assets/samplesheet_assembly_big_coassembly_with_coverage.csv delete mode 100644 assets/samplesheet_assembly_coassembly_no_coverage.csv delete mode 100644 assets/samplesheet_assembly_coassembly_with_coverage.csv diff --git a/assets/samplesheet_assembly_big_coassembly_with_coverage.csv b/assets/samplesheet_assembly_big_coassembly_with_coverage.csv deleted file mode 100644 index 254dced..0000000 --- a/assets/samplesheet_assembly_big_coassembly_with_coverage.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version -big_coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,SRR3187093;SRR3187094;SRR3187095;SRR3187096;SRR3187097;SRR3187098;SRR3187099;SRR3187100;SRR3187101;SRR3187102;SRR3187103;SRR3187104;SRR3187105;SRR3187106;SRR3187107;SRR3187108;SRR3187109;SRR3187110;SRR3187111;SRR3187112;SRR3187113;SRR3187114;SRR3187115;SRR3187116;SRR3187117;SRR3187118;SRR3187119;SRR3187120;SRR3187121;SRR3187122;SRR3187123;SRR3187124;SRR3187125;SRR3187126;SRR3187127;SRR3187128;SRR3187129;SRR3187130;SRR3187131;SRR3187132;SRR3187133;SRR3187134;SRR3187135;SRR3187136;SRR3187137;SRR3187138;SRR3187139;SRR3187140;SRR3187141;SRR3187142;SRR3187143,SPAdes,3.15 diff --git a/assets/samplesheet_assembly_coassembly_no_coverage.csv b/assets/samplesheet_assembly_coassembly_no_coverage.csv deleted file mode 100644 index 8c7cd3f..0000000 --- a/assets/samplesheet_assembly_coassembly_no_coverage.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version -no_coverage_paired_reads,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz;https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz,,SRR3183850;SRR3183951,SPAdes,3.15 diff --git a/assets/samplesheet_assembly_coassembly_with_coverage.csv b/assets/samplesheet_assembly_coassembly_with_coverage.csv deleted file mode 100644 index 1a459d0..0000000 --- a/assets/samplesheet_assembly_coassembly_with_coverage.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version -coassembly_sample,https://github.com/nf-core/test-datasets/raw/modules/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz,,,45.5,SRR3183850;SRR3183951,SPAdes,3.15 diff --git a/conf/test_assembly_coassembly_no_coverage.config b/conf/test_assembly_coassembly_no_coverage.config index eda3ca5..834afcd 100644 --- a/conf/test_assembly_coassembly_no_coverage.config +++ b/conf/test_assembly_coassembly_no_coverage.config @@ -23,7 +23,7 @@ params { config_profile_description = 'Co-assembly test with missing coverage and paired-end reads from two runs' // Input data - input = "${projectDir}/assets/samplesheet_assembly_coassembly_no_coverage.csv" + input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_assembly_coassembly_no_coverage.csv' outdir = 'test_output' mode = "metagenomic_assemblies" diff --git a/conf/test_assembly_coassembly_with_coverage.config b/conf/test_assembly_coassembly_with_coverage.config index ba3d683..89a8307 100644 --- a/conf/test_assembly_coassembly_with_coverage.config +++ b/conf/test_assembly_coassembly_with_coverage.config @@ -23,7 +23,7 @@ params { config_profile_description = 'Co-assembly test with pre-provided coverage and multiple run accessions' // Input data - input = "${projectDir}/assets/samplesheet_assembly_coassembly_with_coverage.csv" + input = params.pipelines_testdata_base_path + 'seqsubmit/samplesheets/samplesheet_assembly_coassembly_with_coverage.csv' outdir = 'test_output' mode = "metagenomic_assemblies" From f0cf4f2814eb7b4615c7372f5747d67315babcd3 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Fri, 16 Oct 2026 13:08:37 +0200 Subject: [PATCH 16/30] do not use hardcoded submission study in tests --- conf/test_assembly_coassembly_no_coverage.config | 3 ++- conf/test_assembly_coassembly_with_coverage.config | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/test_assembly_coassembly_no_coverage.config b/conf/test_assembly_coassembly_no_coverage.config index 834afcd..41d1e46 100644 --- a/conf/test_assembly_coassembly_no_coverage.config +++ b/conf/test_assembly_coassembly_no_coverage.config @@ -27,7 +27,8 @@ params { outdir = 'test_output' mode = "metagenomic_assemblies" - submission_study = "PRJEB98843" + submission_study = null + study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json' centre_name = "TEST_CENTER" test_upload = true diff --git a/conf/test_assembly_coassembly_with_coverage.config b/conf/test_assembly_coassembly_with_coverage.config index 89a8307..f688999 100644 --- a/conf/test_assembly_coassembly_with_coverage.config +++ b/conf/test_assembly_coassembly_with_coverage.config @@ -27,7 +27,8 @@ params { outdir = 'test_output' mode = "metagenomic_assemblies" - submission_study = "PRJEB98843" + submission_study = null + study_metadata = params.pipelines_testdata_base_path + 'seqsubmit/test_data/study_metadata/study_metadata.json' centre_name = "TEST_CENTER" test_upload = true From d97e1cef4b2a25b6c6a94dd84edff7a335340f8f Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Fri, 16 Oct 2026 15:03:55 +0200 Subject: [PATCH 17/30] update tests and snapshots --- tests/assembly_coassembly_no_coverage.nf.test | 3 +- ...sembly_coassembly_no_coverage.nf.test.snap | 11 ++- .../assembly_coassembly_with_coverage.nf.test | 3 +- ...mbly_coassembly_with_coverage.nf.test.snap | 77 +++++++++++++++++++ 4 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 tests/assembly_coassembly_with_coverage.nf.test.snap diff --git a/tests/assembly_coassembly_no_coverage.nf.test b/tests/assembly_coassembly_no_coverage.nf.test index a4eefd8..fd50466 100644 --- a/tests/assembly_coassembly_no_coverage.nf.test +++ b/tests/assembly_coassembly_no_coverage.nf.test @@ -19,7 +19,8 @@ nextflow_pipeline { // stable_name: All files + folders in ${params.outdir}/ with a stable name def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) // stable_path: All files in ${params.outdir}/ with stable content - def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // assemblies_metadata.csv holds the co-assembly sample accession, newly registered on every --test run + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore', ignore: ['**/upload/assemblies_metadata.csv']) // Early failure no need to test the rest of snapshots assert workflow.success assertAll( diff --git a/tests/assembly_coassembly_no_coverage.nf.test.snap b/tests/assembly_coassembly_no_coverage.nf.test.snap index 31342e2..5785326 100644 --- a/tests/assembly_coassembly_no_coverage.nf.test.snap +++ b/tests/assembly_coassembly_no_coverage.nf.test.snap @@ -1,7 +1,7 @@ { "-profile test_assembly_coassembly_no_coverage": { "content": [ - 9, + 10, { "CONCAT_ACCESSIONS": { "coreutils": 9.4, @@ -25,6 +25,9 @@ "GENERATE_ASSEMBLY_MANIFEST": { "assembly_uploader": "1.3.5" }, + "REGISTERSTUDY": { + "mgnify-pipelines-toolkit": "1.5.1" + }, "SUBMIT": { "ena-webin-cli": "9.0.3", "mgnify-pipelines-toolkit": "1.5.1", @@ -57,19 +60,21 @@ "multiqc/multiqc_data/multiqc_sources.txt", "multiqc/multiqc_plots", "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/Assemblies assigned accessions.pdf", "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/Assemblies assigned accessions.png", "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/Assemblies assigned accessions.svg", "multiqc/multiqc_report.html", "pipeline_info", "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ "no_coverage_paired_reads.depth.tsv:md5,6e61f10c995d45b15f33d572d773e80e", - "assemblies_metadata.csv:md5,22f1108ab2a35170ee1fe86954e7fd3d", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-10-15T15:17:34.623978", + "timestamp": "2026-10-16T15:01:52.422755", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" diff --git a/tests/assembly_coassembly_with_coverage.nf.test b/tests/assembly_coassembly_with_coverage.nf.test index 60f15e9..44629f7 100644 --- a/tests/assembly_coassembly_with_coverage.nf.test +++ b/tests/assembly_coassembly_with_coverage.nf.test @@ -19,7 +19,8 @@ nextflow_pipeline { // stable_name: All files + folders in ${params.outdir}/ with a stable name def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) // stable_path: All files in ${params.outdir}/ with stable content - def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // assemblies_metadata.csv holds the co-assembly sample accession, newly registered on every --test run + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore', ignore: ['**/upload/assemblies_metadata.csv']) // Early failure no need to test the rest of snapshots assert workflow.success assertAll( diff --git a/tests/assembly_coassembly_with_coverage.nf.test.snap b/tests/assembly_coassembly_with_coverage.nf.test.snap new file mode 100644 index 0000000..3c54f4f --- /dev/null +++ b/tests/assembly_coassembly_with_coverage.nf.test.snap @@ -0,0 +1,77 @@ +{ + "-profile test_assembly_coassembly_with_coverage": { + "content": [ + 9, + { + "CONCAT_ACCESSIONS": { + "coreutils": 9.4, + "find": "4.6.0", + "pigz": 2.8 + }, + "CONCAT_METADATA": { + "coreutils": 9.4, + "find": "4.6.0", + "pigz": 2.8 + }, + "CREATE_ASSEMBLY_METADATA_CSV": { + "bash": "5.2.37(1)-release" + }, + "FALINT": { + "falint": "1.2.0" + }, + "GENERATE_ASSEMBLY_MANIFEST": { + "assembly_uploader": "1.3.5" + }, + "REGISTERSTUDY": { + "mgnify-pipelines-toolkit": "1.5.1" + }, + "SUBMIT": { + "ena-webin-cli": "9.0.3", + "mgnify-pipelines-toolkit": "1.5.1", + "python": "3.12.13" + }, + "Workflow": { + "nf-core/seqsubmit": "v1.0.0" + } + }, + [ + "metagenomic_assemblies", + "metagenomic_assemblies/coassembly_sample", + "metagenomic_assemblies/coassembly_sample/upload", + "metagenomic_assemblies/coassembly_sample/upload/webin_cli", + "metagenomic_assemblies/coassembly_sample/upload/webin_cli/coassembly_sample_accessions.tsv", + "metagenomic_assemblies/upload", + "metagenomic_assemblies/upload/assemblies_accessions.tsv", + "metagenomic_assemblies/upload/assemblies_metadata.csv", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_Assemblies assigned accessions.txt", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/Assemblies assigned accessions.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/Assemblies assigned accessions.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/Assemblies assigned accessions.svg", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" + ], + [ + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" + ] + ], + "timestamp": "2026-10-16T15:03:42.484719", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.1" + } + } +} \ No newline at end of file From 4401ad237f6e19167e87f6210d576049d2d74be6 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Fri, 16 Oct 2026 15:32:57 +0200 Subject: [PATCH 18/30] update docs --- README.md | 3 +++ docs/methods.md | 7 +++++-- docs/usage.md | 34 ++++++++++++++++++++++------------ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ecf1b66..522de92 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,11 @@ Example: id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version assembly_1,data/contigs_1.fasta.gz,data/reads_1.fastq.gz,data/reads_2.fastq.gz,,ERR011322,SPAdes,3.15.5 assembly_2,data/contigs_2.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9 +coassembly_1,data/contigs_3.fasta.gz,data/run1_R1.fastq.gz;data/run2_R1.fastq.gz,data/run1_R2.fastq.gz;data/run2_R2.fastq.gz,,ERR011324;ERR011325,MEGAHIT,1.2.9 ``` +Co-assemblies are supported: list all source runs in `run_accession` and their read files in `fastq_1`/`fastq_2`, separated by semicolons. + See the [`metagenomic_assemblies` mode section](docs/usage.md#samplesheet-input-1) of the usage documentation for more details. ### `mags` and `bins` modes diff --git a/docs/methods.md b/docs/methods.md index 019a720..38c79e8 100644 --- a/docs/methods.md +++ b/docs/methods.md @@ -12,7 +12,8 @@ The `ASSEMBLYSUBMIT` workflow: 2. Validates assembly FASTA files. 3. Reuses provided or calculates missing `coverage` values. 4. Builds the metadata table used for manifest generation. -5. Generates assembly manifests and submits the assemblies to ENA. +5. Registers a virtual ENA sample for co-assemblies whose runs come from different samples. +6. Generates assembly manifests and submits the assemblies to ENA. ### Assembly FASTA validation @@ -24,7 +25,9 @@ Only assemblies with successful validation are forwarded to coverage estimation, If the `coverage` column is already populated in the samplesheet, that value is used directly. -If `coverage` is missing, the workflow maps the reads of all runs listed for the assembly with `coverm contig`, which reports the mean depth of each contig for each run. +If `coverage` is missing, the workflow maps the reads of all runs listed for the assembly with `coverm contig`, which reports the mean depth and the length of each contig for each run. + +The same calculation is applied to every assembly. For an assembly from a single run ($N = 1$), it reduces to the length-weighted mean of the per-contig depths. Because a co-assembly is built from the pooled reads of all runs, the depth of contig $c$ is the sum of its depths across the $N$ runs: diff --git a/docs/usage.md b/docs/usage.md index 14c38e2..820326e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -217,21 +217,33 @@ Example: id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version assembly_001,data/assembly_001.fasta.gz,data/assembly_001_R1.fastq.gz,data/assembly_001_R2.fastq.gz,,ERR011322,SPAdes,3.15.5 assembly_002,data/assembly_002.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9 +coassembly_001,data/coassembly_001.fasta.gz,data/run1_R1.fastq.gz;data/run2_R1.fastq.gz,data/run1_R2.fastq.gz;data/run2_R2.fastq.gz,,ERR011324;ERR011325,MEGAHIT,1.2.9 ``` -| Column | Required | Description | -| ------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | Yes | A unique identifier for this data entry. Must be globally unique within the input dataset. | -| `fasta` | Yes | Path to assembly contigs in FASTA format compressed with `gzip`. | -| `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required if `coverage` is not provided. | -| `fastq_2` | No | Path to the second read file in FASTQ format for paired-end data used to generate the source metagenomic assembly. Leave empty for single-end reads. | -| `coverage` | Conditional | Estimated sequencing depth of the assembly. If this value is missing, it is computed automatically during pipeline execution when reads are provided. | -| `run_accession` | Yes | ENA run accession for the reads used to generate the metagenomic assembly. Reads must already be submitted to ENA. | -| `assembler` | Yes | Name of the assembler software used to generate the assembly, e.g. MEGAHIT, SPAdes, metaSPAdes, Flye, etc. | -| `assembler_version` | Yes | Version of the assembler software used to generate the assembly, e.g. 1.0.0. | +| Column | Required | Description | +| ------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Yes | A unique identifier for this data entry. Must be globally unique within the input dataset. | +| `fasta` | Yes | Path to assembly contigs in FASTA format compressed with `gzip`. | +| `fastq_1` | Conditional | Path to single-end/paired-end forward reads in FASTQ format (optionally gzipped). Required if `coverage` is not provided. For co-assemblies, semicolon-separated paths, one per run, in the same order as `run_accession` (e.g. `run1_R1.fastq.gz;run2_R1.fastq.gz`). | +| `fastq_2` | No | Path to the second read file in FASTQ format for paired-end data used to generate the source metagenomic assembly. Leave empty for single-end reads. For co-assemblies, semicolon-separated paths, one per run, in the same order as `run_accession`. | +| `coverage` | Conditional | Estimated sequencing depth of the assembly. If this value is missing, it is computed automatically during pipeline execution when reads are provided. | +| `run_accession` | Yes | ENA run accession(s) for the reads used to generate the metagenomic assembly. Reads must already be submitted to ENA. For co-assemblies, semicolon-separated accessions (e.g. `ERR011324;ERR011325`). | +| `assembler` | Yes | Name of the assembler software used to generate the assembly, e.g. MEGAHIT, SPAdes, metaSPAdes, Flye, etc. | +| `assembler_version` | Yes | Version of the assembler software used to generate the assembly, e.g. 1.0.0. | Provide either read files (`fastq_1`, optionally `fastq_2`) or a `coverage` value for each row. If `coverage` is missing and reads are provided, the workflow calculates average coverage with CoverM. See the [Methods documentation](docs/methods.md) for more information on how coverage is calculated. +### Co-assemblies + +An assembly built from the pooled reads of several runs (a co-assembly) is described by a single samplesheet row listing all source runs in `run_accession`, separated by semicolons. If coverage has to be calculated, provide the read files of every run in `fastq_1` (and `fastq_2` for paired-end data) in the same order as the accessions; the pipeline fails if the number of read files does not match the number of run accessions. + +ENA requires each assembly to reference a single sample. The pipeline handles this as follows: + +- If all source runs belong to the same ENA sample, the assembly is submitted against that sample. +- If the runs belong to different samples, the pipeline registers a virtual ENA sample (checklist `ERC000011`) with a `composed of` attribute listing the source samples, and submits the assembly against it. Its metadata is merged from the source samples: + - `collection date` and `geographic location (country and/or sea)` are kept if all source samples share the same value, otherwise they are set to `not provided`. A country that is not in the `ERC000011` list of allowed values is also set to `not provided`. + - The taxon and scientific name must be the same for all source samples; otherwise the pipeline stops with an error. + ### `metagenomic_assemblies` parameters and example In addition to the [common parameters](#running-the-pipeline): @@ -270,8 +282,6 @@ nextflow run nf-core/seqsubmit \ `metagenomic_assemblies` mode does not yet support the following: -- **Co-assemblies**: Metagenomic co-assemblies are not supported. The pipeline currently expects each assembly to be derived from a single run. Tracked in [issue #61](https://github.com/nf-core/seqsubmit/issues/61), addressed in [PR #66](https://github.com/nf-core/seqsubmit/pull/66). - - **Submission of an assembly without submitting the raw reads**: Assemblies that have no associated reads accession (`run_accession` column in the samplesheet) are not supported, since the pipeline relies on that lineage to build the required metadata. Tracked in [issue #29](https://github.com/nf-core/seqsubmit/issues/29). - **Single-contig assemblies**: Assemblies that consist of only one contig cannot be uploaded through this pipeline. ENA classifies these as "chromosomal assemblies", which follow a different submission procedure with different metadata requirements. If you have a single-contig assembly to submit, please contact [ENA support](https://www.ebi.ac.uk/ena/browser/support) for guidance. From 0c92b6d7bc049acf4f62ca1e3a787088d02df5e8 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Mon, 19 Oct 2026 16:30:07 +0100 Subject: [PATCH 19/30] update REGISTER_COASSEMBLY_SAMPLE versions to topics --- .../local/register_coassembly_sample/main.nf | 18 +++--------------- .../local/register_coassembly_sample/meta.yml | 5 ----- ...ssembly_coassembly_no_coverage.nf.test.snap | 7 ++++++- ...embly_coassembly_with_coverage.nf.test.snap | 7 ++++++- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/modules/local/register_coassembly_sample/main.nf b/modules/local/register_coassembly_sample/main.nf index f86b3a6..6d7e132 100644 --- a/modules/local/register_coassembly_sample/main.nf +++ b/modules/local/register_coassembly_sample/main.nf @@ -14,7 +14,9 @@ process REGISTER_COASSEMBLY_SAMPLE { output: tuple val(meta), path("*_updated.csv"), emit: csv - path "versions.yml", emit: versions + tuple val("${task.process}"), val('python'), eval('python --version 2>&1 | sed "s/Python //g"'), topic: versions + tuple val("${task.process}"), val('requests'), eval('python -c "import requests; print(requests.__version__)"'), topic: versions + tuple val("${task.process}"), val('retry'), eval('python -c "import importlib.metadata; print(importlib.metadata.version(\'retry\'))"'), topic: versions when: task.ext.when == null || task.ext.when @@ -31,25 +33,11 @@ process REGISTER_COASSEMBLY_SAMPLE { ${test_arg} \\ ${private_arg} \\ ${args} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //') - requests: \$(python -c "import requests; print(requests.__version__)") - retry: \$(python -c "import importlib.metadata as m; print(m.version('retry'))") - END_VERSIONS """ stub: def prefix = task.ext.prefix ?: "${meta.id}" """ cp ${csv} ${prefix}_updated.csv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //') - requests: \$(python -c "import requests; print(requests.__version__)") - retry: \$(python -c "import importlib.metadata as m; print(m.version('retry'))") - END_VERSIONS """ } diff --git a/modules/local/register_coassembly_sample/meta.yml b/modules/local/register_coassembly_sample/meta.yml index 977b239..6f02c99 100644 --- a/modules/local/register_coassembly_sample/meta.yml +++ b/modules/local/register_coassembly_sample/meta.yml @@ -33,11 +33,6 @@ output: type: file description: Updated assembly metadata CSV with "Sample" column filled for co-assemblies pattern: "*.csv" - - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" authors: - "@ochkalova" maintainers: diff --git a/tests/assembly_coassembly_no_coverage.nf.test.snap b/tests/assembly_coassembly_no_coverage.nf.test.snap index 5785326..26b7163 100644 --- a/tests/assembly_coassembly_no_coverage.nf.test.snap +++ b/tests/assembly_coassembly_no_coverage.nf.test.snap @@ -28,6 +28,11 @@ "REGISTERSTUDY": { "mgnify-pipelines-toolkit": "1.5.1" }, + "REGISTER_COASSEMBLY_SAMPLE": { + "python": "3.14.4", + "requests": "2.34.1", + "retry": "0.9.2" + }, "SUBMIT": { "ena-webin-cli": "9.0.3", "mgnify-pipelines-toolkit": "1.5.1", @@ -74,7 +79,7 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-10-16T15:01:52.422755", + "timestamp": "2026-10-19T16:26:50.911456", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" diff --git a/tests/assembly_coassembly_with_coverage.nf.test.snap b/tests/assembly_coassembly_with_coverage.nf.test.snap index 3c54f4f..7cadef4 100644 --- a/tests/assembly_coassembly_with_coverage.nf.test.snap +++ b/tests/assembly_coassembly_with_coverage.nf.test.snap @@ -25,6 +25,11 @@ "REGISTERSTUDY": { "mgnify-pipelines-toolkit": "1.5.1" }, + "REGISTER_COASSEMBLY_SAMPLE": { + "python": "3.14.4", + "requests": "2.34.1", + "retry": "0.9.2" + }, "SUBMIT": { "ena-webin-cli": "9.0.3", "mgnify-pipelines-toolkit": "1.5.1", @@ -68,7 +73,7 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], - "timestamp": "2026-10-16T15:03:42.484719", + "timestamp": "2026-10-19T16:27:45.228133", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.1" From 90bd67115470373f791e6df9973a70a1c7f5d379 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Mon, 19 Oct 2026 16:30:51 +0100 Subject: [PATCH 20/30] update BioSample regex --- bin/register_coassembly_sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index 37c9cc0..2386727 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -24,7 +24,7 @@ PROD_SUBMIT_URL = "https://www.ebi.ac.uk/ena/submit/drop-box/submit/" INSDC_BIOSAMPLE_ACCESSION_REGEX = re.compile( - r"SAM[AG]?[0-9]+" + r"^SAM(N|D|E|EA)[0-9]{7,8}$" ) EXISTING_ACCESSION_IN_ERROR_REGEX = re.compile( r'accession:\s*"([A-Z]+[0-9]+)"', From 29fbe914b35429e399405aec46b6666065af7470 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Tue, 20 Oct 2026 16:15:14 +0100 Subject: [PATCH 21/30] make sure read files exist with checkIfExists: true --- workflows/assemblysubmit.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index f03bf1f..7cdb1d7 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -87,8 +87,8 @@ workflow ASSEMBLYSUBMIT { } // Convert paths to file objects - def fastq1_paths = fastq1_list.collect { path -> file(path.trim()) } - def fastq2_paths = fastq2_list ? fastq2_list.collect { path -> file(path.trim()) } : [] + def fastq1_paths = fastq1_list.collect { path -> file(path.trim(), checkIfExists: true) } + def fastq2_paths = fastq2_list ? fastq2_list.collect { path -> file(path.trim(), checkIfExists: true) } : [] if (fastq2_paths) { [new_meta, fastq1_paths, fastq2_paths] From 0181af90aafd745b7453cc2096b52a53e6411ec5 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Tue, 20 Oct 2026 16:15:53 +0100 Subject: [PATCH 22/30] add guard against empty files in calculateCoassemblyCoverage --- .../utils_nfcore_seqsubmit_pipeline/main.nf | 9 ++++++- .../fixtures/coverm_zero_depth.depth.tsv | 3 +++ .../tests/main.function.nf.test | 24 ++++++++++++++++--- 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_zero_depth.depth.tsv diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf index 168b0cb..0dcba33 100644 --- a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/main.nf @@ -216,7 +216,14 @@ def calculateCoassemblyCoverage(coverage_file) { total_length += length } - return total_length > 0 ? weighted_depth / total_length : 0d + if (total_length == 0) { + error("CoverM output ${coverage_file.name} has no contigs, so assembly coverage cannot be calculated.") + } + def coverage = weighted_depth / total_length + if (coverage == 0) { + error("Calculated coverage for ${coverage_file.name} is 0: no reads mapped to the assembly. Check that the reads in 'fastq_1'/'fastq_2' belong to this assembly.") + } + return coverage } // diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_zero_depth.depth.tsv b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_zero_depth.depth.tsv new file mode 100644 index 0000000..4ca64eb --- /dev/null +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/fixtures/coverm_zero_depth.depth.tsv @@ -0,0 +1,3 @@ +Contig contigs.fasta/run1_1.fastq.gz Mean contigs.fasta/run1_1.fastq.gz Length +contig_1 0 1000 +contig_2 0 100 diff --git a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test index 9299089..937b342 100644 --- a/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test +++ b/subworkflows/local/utils_nfcore_seqsubmit_pipeline/tests/main.function.nf.test @@ -46,7 +46,7 @@ nextflow_function { } } - test("No contigs in CoverM output - coverage is 0") { + test("No contigs in CoverM output - fails with a clear error") { when { function { @@ -58,8 +58,26 @@ nextflow_function { then { assertAll( - { assert function.success }, - { assert function.result == 0d } + { assert function.failed }, + { assert function.stdout.any { it.contains("has no contigs") } } + ) + } + } + + test("No reads mapped - coverage 0 fails with a clear error") { + + when { + function { + """ + input[0] = file("${moduleDir}/tests/fixtures/coverm_zero_depth.depth.tsv") + """ + } + } + + then { + assertAll( + { assert function.failed }, + { assert function.stdout.any { it.contains("no reads mapped") } } ) } } From 3e72efa2adfb5062b574456d4fe221e6f3c5cb07 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Tue, 20 Oct 2026 16:17:20 +0100 Subject: [PATCH 23/30] update snapshot --- tests/assembly_coassembly_no_coverage.nf.test.snap | 2 +- tests/assembly_coassembly_with_coverage.nf.test.snap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/assembly_coassembly_no_coverage.nf.test.snap b/tests/assembly_coassembly_no_coverage.nf.test.snap index 26b7163..a833d10 100644 --- a/tests/assembly_coassembly_no_coverage.nf.test.snap +++ b/tests/assembly_coassembly_no_coverage.nf.test.snap @@ -23,7 +23,7 @@ "falint": "1.2.0" }, "GENERATE_ASSEMBLY_MANIFEST": { - "assembly_uploader": "1.3.5" + "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { "mgnify-pipelines-toolkit": "1.5.1" diff --git a/tests/assembly_coassembly_with_coverage.nf.test.snap b/tests/assembly_coassembly_with_coverage.nf.test.snap index 7cadef4..fdf2a5d 100644 --- a/tests/assembly_coassembly_with_coverage.nf.test.snap +++ b/tests/assembly_coassembly_with_coverage.nf.test.snap @@ -20,7 +20,7 @@ "falint": "1.2.0" }, "GENERATE_ASSEMBLY_MANIFEST": { - "assembly_uploader": "1.3.5" + "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { "mgnify-pipelines-toolkit": "1.5.1" From 0d6d64e99e54bce90ca9b2f1059ec9f29e6a3172 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 12:18:24 +0000 Subject: [PATCH 24/30] Address review comments in register_coassembly_sample.py - Move ERC000011 checklist and ENA Portal field lists to global constants - Validate secondary sample accessions with a regex - Report tax_id/scientific_name of all source samples in one error - Reword virtual sample description Co-Authored-By: Claude Opus 5.5 --- bin/register_coassembly_sample.py | 44 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index 2386727..7ac4437 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -18,7 +18,13 @@ PORTAL_SEARCH_API = "https://www.ebi.ac.uk/ena/portal/api/search" -CHECKLIST_XML_API = "https://www.ebi.ac.uk/ena/browser/api/xml/ERC000011" + +CHECKLIST = "ERC000011" +CHECKLIST_XML_API = f"https://www.ebi.ac.uk/ena/browser/api/xml/{CHECKLIST}" + +# ENA Portal API fields retrieved for runs and samples +RUN_SAMPLE_FIELDS = ["sample_accession", "secondary_sample_accession"] +SAMPLE_METADATA_FIELDS = ["country", "collection_date", "tax_id", "scientific_name"] TEST_SUBMIT_URL = "https://wwwdev.ebi.ac.uk/ena/submit/drop-box/submit/" PROD_SUBMIT_URL = "https://www.ebi.ac.uk/ena/submit/drop-box/submit/" @@ -26,6 +32,9 @@ INSDC_BIOSAMPLE_ACCESSION_REGEX = re.compile( r"^SAM(N|D|E|EA)[0-9]{7,8}$" ) +INSDC_SECONDARY_SAMPLE_ACCESSION_REGEX = re.compile( + r"^(E|D|S)RS[0-9]{6,}$" +) EXISTING_ACCESSION_IN_ERROR_REGEX = re.compile( r'accession:\s*"([A-Z]+[0-9]+)"', re.IGNORECASE, @@ -106,14 +115,14 @@ def get_json(url: str, params: dict[str, str], auth: HTTPBasicAuth | None = None return result -def search_portal(result: str, query: str, fields: str, private: bool = False) -> list[dict[str, str]]: +def search_portal(result: str, query: str, fields: list[str], private: bool = False) -> list[dict[str, str]]: """ Query the ENA Portal API. When `private` is set, the request is authenticated with the Webin account (ENA_WEBIN/ENA_WEBIN_PASSWORD), which also returns private records owned by it. """ - params = {"result": result, "query": query, "fields": fields, "format": "json"} + params = {"result": result, "query": query, "fields": ",".join(fields), "format": "json"} if not private: return get_json(PORTAL_SEARCH_API, params=params) @@ -178,7 +187,7 @@ def get_run_sample(run_accession: str, private: bool = False) -> str: rows = search_portal( result="read_run", query=f'"run_accession={run_accession}"', - fields="sample_accession,secondary_sample_accession", + fields=RUN_SAMPLE_FIELDS, private=private, ) if not rows: @@ -207,13 +216,15 @@ def get_sample_metadata(sample_accession: str, private: bool = False) -> tuple[s """ if INSDC_BIOSAMPLE_ACCESSION_REGEX.match(sample_accession): query = f'"sample_accession={sample_accession}"' - else: + elif INSDC_SECONDARY_SAMPLE_ACCESSION_REGEX.match(sample_accession): query = f'"secondary_sample_accession={sample_accession}"' + else: + raise CoassemblyRegistrationError(f"Unrecognised sample accession format: {sample_accession}") rows = search_portal( result="sample", query=query, - fields="country,collection_date,tax_id,scientific_name", + fields=SAMPLE_METADATA_FIELDS, private=private, ) @@ -299,7 +310,7 @@ def add_attr(tag: str, value: str) -> None: value_node = ET.SubElement(item, "VALUE") value_node.text = value - add_attr("ENA-CHECKLIST", "ERC000011") + add_attr("ENA-CHECKLIST", CHECKLIST) add_attr("organism", scientific_name) add_attr("collection date", collection_date) add_attr("composed of", composed_of) @@ -422,28 +433,29 @@ def register_virtual_sample(run_accessions: list[str], test: bool, private: bool merged_taxid = merge_or_not_provided(taxon_ids, default_taxid) merged_tax_name = merge_or_not_provided(scientific_names, default_tax_name) - if merged_taxid == DEFAULT_NA_VALUE: - raise CoassemblyRegistrationError( - f"Co-assembly source samples have mixed taxa: {', '.join(taxon_ids)}" + if merged_taxid == DEFAULT_NA_VALUE or merged_tax_name == DEFAULT_NA_VALUE: + sample_taxa = dict(zip(sample_accessions, zip(taxon_ids, scientific_names))) + details = "\n".join( + f" {sample}: tax_id={taxid}, scientific_name={name}" + for sample, (taxid, name) in sample_taxa.items() ) - - if merged_tax_name == DEFAULT_NA_VALUE: raise CoassemblyRegistrationError( - f"Co-assembly source samples have mixed scientific names: {', '.join(scientific_names)}" + "Co-assembly source samples must share the same tax_id and scientific_name:\n" + f"{details}" ) allowed_countries = get_allowed_countries() if merged_country != DEFAULT_NA_VALUE and merged_country not in allowed_countries: logger.warning( - f"Country '{merged_country}' is not in ERC000011 allowed country/sea values; setting to '{DEFAULT_NA_VALUE}'" + f"Country '{merged_country}' is not in {CHECKLIST} allowed country/sea values; setting to '{DEFAULT_NA_VALUE}'" ) merged_country = DEFAULT_NA_VALUE sample_list = ",".join(unique_samples) title = f"Combined sample from {sample_list}" description = ( - "This sample is a virtual sample of co-assembled raw reads from multiple samples " - f"of {merged_tax_name}. Co-assembly was performed from runs of the samples {sample_list}" + f"This is a virtual sample representing a co-assembly of raw reads from multiple " + f"{merged_tax_name} samples. Co-assembly was performed from runs of {len(unique_samples)} samples {sample_list}." ) alias = build_virtual_sample_alias(unique_samples, test=test) logger.info(f"Registering virtual sample with alias '{alias}' for co-assembly of samples: {sample_list}") From 3f4d4245557f104eda6fe345c5428f79f35d6fd1 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 12:24:44 +0000 Subject: [PATCH 25/30] Update CHANGELOG and co-assembly docs after review --- CHANGELOG.md | 6 ++++++ docs/usage.md | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33b9d14..d47a5e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## v1.0.1dev - [date] +### `Added` + +- [#66](https://github.com/nf-core/seqsubmit/pull/66) + - Added co-assembly submission to `ASSEMBLYSUBMIT`: a samplesheet row can list several semicolon-separated run accessions (and matching read files for coverage calculation) + - When the source runs belong to different ENA samples, a virtual sample with a `composed of` attribute is registered and the co-assembly is submitted under it + ### `Fixed` - [#116](https://github.com/nf-core/seqsubmit/pull/116) diff --git a/docs/usage.md b/docs/usage.md index 3670ad8..ebbb154 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -240,10 +240,10 @@ An assembly built from the pooled reads of several runs (a co-assembly) is descr ENA requires each assembly to reference a single sample. The pipeline handles this as follows: -- If all source runs belong to the same ENA sample, the assembly is submitted against that sample. -- If the runs belong to different samples, the pipeline registers a virtual ENA sample (checklist `ERC000011`) with a `composed of` attribute listing the source samples, and submits the assembly against it. Its metadata is merged from the source samples: +- If all source runs belong to the same ENA sample, the assembly is submitted under that sample. +- If the runs belong to different samples, the pipeline registers a virtual ENA sample with a `composed of` attribute listing the source samples, and submits the assembly under it. The virtual sample uses the [ENA default sample checklist (`ERC000011`)](https://www.ebi.ac.uk/ena/browser/view/ERC000011), the minimal checklist, because its mandatory fields (`collection date` and `geographic location (country and/or sea)`) can be derived from any source samples, whichever checklists they were originally registered with. Its metadata is merged from the source samples: - `collection date` and `geographic location (country and/or sea)` are kept if all source samples share the same value, otherwise they are set to `not provided`. A country that is not in the `ERC000011` list of allowed values is also set to `not provided`. - - The taxon and scientific name must be the same for all source samples; otherwise the pipeline stops with an error. + - `tax_id` and `scientific_name` must be the same for all source samples; otherwise the pipeline stops with an error. ### `metagenomic_assemblies` parameters and example From 07abba49112d1e2e0328ec644f00dde8cf271100 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 14:51:02 +0000 Subject: [PATCH 26/30] add more assertions for co assembly tests --- tests/assembly_coassembly_no_coverage.nf.test | 5 +++++ tests/assembly_coassembly_with_coverage.nf.test | 3 +++ 2 files changed, 8 insertions(+) diff --git a/tests/assembly_coassembly_no_coverage.nf.test b/tests/assembly_coassembly_no_coverage.nf.test index fd50466..0846965 100644 --- a/tests/assembly_coassembly_no_coverage.nf.test +++ b/tests/assembly_coassembly_no_coverage.nf.test @@ -23,7 +23,12 @@ nextflow_pipeline { def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore', ignore: ['**/upload/assemblies_metadata.csv']) // Early failure no need to test the rest of snapshots assert workflow.success + def succeeded_tasks = workflow.trace.succeeded()*.name assertAll( + // Coverage is computed with CoverM and the runs are registered as a virtual co-assembly sample + { assert succeeded_tasks.any { it.contains(':COVERM_CONTIG') } }, + { assert succeeded_tasks.any { it.contains(':REGISTER_COASSEMBLY_SAMPLE') } }, + { assert path("$outputDir/metagenomic_assemblies/upload/assemblies_metadata.csv").readLines()[1].contains(',1.263635324935214,') }, { assert snapshot( // Number of successful tasks workflow.trace.succeeded().size(), diff --git a/tests/assembly_coassembly_with_coverage.nf.test b/tests/assembly_coassembly_with_coverage.nf.test index 44629f7..5b606e4 100644 --- a/tests/assembly_coassembly_with_coverage.nf.test +++ b/tests/assembly_coassembly_with_coverage.nf.test @@ -23,7 +23,10 @@ nextflow_pipeline { def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore', ignore: ['**/upload/assemblies_metadata.csv']) // Early failure no need to test the rest of snapshots assert workflow.success + def all_tasks = workflow.trace.tasks()*.name assertAll( + // Coverage is provided in the samplesheet, so CoverM must not run + { assert !all_tasks.any { it.contains(':COVERM_CONTIG') } }, { assert snapshot( // Number of successful tasks workflow.trace.succeeded().size(), From 38a69dba47beb6ce5901e24ad15c84f2a710d1c4 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 15:29:08 +0000 Subject: [PATCH 27/30] update snapshots --- .../create_assembly_metadata_csv/tests/main.nf.test.snap | 8 ++++---- tests/assembly_coassembly_no_coverage.nf.test.snap | 6 +++--- tests/assembly_coassembly_with_coverage.nf.test.snap | 6 +++--- tests/assembly_complete_metadata.nf.test.snap | 2 +- ...assembly_complete_metadata_existing_study.nf.test.snap | 2 +- tests/assembly_no_coverage_paired_reads.nf.test.snap | 6 +++--- tests/assembly_no_coverage_single_reads.nf.test.snap | 6 +++--- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/modules/local/create_assembly_metadata_csv/tests/main.nf.test.snap b/modules/local/create_assembly_metadata_csv/tests/main.nf.test.snap index af589fe..455ddaa 100644 --- a/modules/local/create_assembly_metadata_csv/tests/main.nf.test.snap +++ b/modules/local/create_assembly_metadata_csv/tests/main.nf.test.snap @@ -12,7 +12,7 @@ "assembler_version": "3.15.5", "assembly_type": "metagenome" }, - "test_sample_assembly_metadata.csv:md5,5627f0b0bc83720dd269e2ea8ac16eff" + "test_sample_assembly_metadata.csv:md5,000897a94b9f0873867fa463790583f2" ] ], "1": [ @@ -32,15 +32,15 @@ "assembler_version": "3.15.5", "assembly_type": "metagenome" }, - "test_sample_assembly_metadata.csv:md5,5627f0b0bc83720dd269e2ea8ac16eff" + "test_sample_assembly_metadata.csv:md5,000897a94b9f0873867fa463790583f2" ] ] } ], - "timestamp": "2026-10-15T11:45:01.123968", + "timestamp": "2026-10-29T15:18:32.564715", "meta": { "nf-test": "0.9.5", - "nextflow": "26.04.0" + "nextflow": "26.04.1" } }, "test_create_assembly_metadata_csv - stub": { diff --git a/tests/assembly_coassembly_no_coverage.nf.test.snap b/tests/assembly_coassembly_no_coverage.nf.test.snap index a833d10..063c1be 100644 --- a/tests/assembly_coassembly_no_coverage.nf.test.snap +++ b/tests/assembly_coassembly_no_coverage.nf.test.snap @@ -26,7 +26,7 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "REGISTER_COASSEMBLY_SAMPLE": { "python": "3.14.4", @@ -35,8 +35,8 @@ }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_coassembly_with_coverage.nf.test.snap b/tests/assembly_coassembly_with_coverage.nf.test.snap index fdf2a5d..19c51e2 100644 --- a/tests/assembly_coassembly_with_coverage.nf.test.snap +++ b/tests/assembly_coassembly_with_coverage.nf.test.snap @@ -23,7 +23,7 @@ "assembly_uploader": "1.3.6" }, "REGISTERSTUDY": { - "mgnify-pipelines-toolkit": "1.5.1" + "mgnify-pipelines-toolkit": "1.5.5" }, "REGISTER_COASSEMBLY_SAMPLE": { "python": "3.14.4", @@ -32,8 +32,8 @@ }, "SUBMIT": { "ena-webin-cli": "9.0.3", - "mgnify-pipelines-toolkit": "1.5.1", - "python": "3.12.13" + "mgnify-pipelines-toolkit": "1.5.5", + "python": "3.12.16" }, "Workflow": { "nf-core/seqsubmit": "v1.0.0" diff --git a/tests/assembly_complete_metadata.nf.test.snap b/tests/assembly_complete_metadata.nf.test.snap index 45ea705..daf4de4 100644 --- a/tests/assembly_complete_metadata.nf.test.snap +++ b/tests/assembly_complete_metadata.nf.test.snap @@ -69,7 +69,7 @@ "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ - "assemblies_metadata.csv:md5,4a171d2d13d999e391fd6b460f5927ec", + "assemblies_metadata.csv:md5,98b1419c74815101cfacd3daf92a435a", "multiqc_assembly_metadata.txt:md5,3187b2ddb3922a060b58f330c7609d0d", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] diff --git a/tests/assembly_complete_metadata_existing_study.nf.test.snap b/tests/assembly_complete_metadata_existing_study.nf.test.snap index 9e251d5..848c6db 100644 --- a/tests/assembly_complete_metadata_existing_study.nf.test.snap +++ b/tests/assembly_complete_metadata_existing_study.nf.test.snap @@ -66,7 +66,7 @@ "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ - "assemblies_metadata.csv:md5,4a171d2d13d999e391fd6b460f5927ec", + "assemblies_metadata.csv:md5,98b1419c74815101cfacd3daf92a435a", "multiqc_assembly_metadata.txt:md5,3187b2ddb3922a060b58f330c7609d0d", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] diff --git a/tests/assembly_no_coverage_paired_reads.nf.test.snap b/tests/assembly_no_coverage_paired_reads.nf.test.snap index 21cec5d..9edf9d4 100644 --- a/tests/assembly_no_coverage_paired_reads.nf.test.snap +++ b/tests/assembly_no_coverage_paired_reads.nf.test.snap @@ -74,9 +74,9 @@ "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ - "no_coverage_paired_reads.depth.tsv:md5,bb5f99e74d21df3c73e0ae0f388bcbcb", - "assemblies_metadata.csv:md5,199890227285973c56ea126e7f556017", - "multiqc_assembly_metadata.txt:md5,07b7d9fc2d6b6eaa9e779af80eb8fdde", + "no_coverage_paired_reads.depth.tsv:md5,ed807966913ebeec8b3face8e30317ed", + "assemblies_metadata.csv:md5,5a2a691d2e2ecf059a74cfa56bef5f19", + "multiqc_assembly_metadata.txt:md5,d830fad9de3ef5531c2170db07341e3a", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], diff --git a/tests/assembly_no_coverage_single_reads.nf.test.snap b/tests/assembly_no_coverage_single_reads.nf.test.snap index 7dbf494..7fad7d9 100644 --- a/tests/assembly_no_coverage_single_reads.nf.test.snap +++ b/tests/assembly_no_coverage_single_reads.nf.test.snap @@ -74,9 +74,9 @@ "pipeline_info/nf_core_seqsubmit_software_mqc_versions.yml" ], [ - "no_coverage_single_reads.depth.tsv:md5,bd88282b17e851377b1dd223839be150", - "assemblies_metadata.csv:md5,f7506e0c624ad2909827966bf2526dfd", - "multiqc_assembly_metadata.txt:md5,e7fd323a6691296e89dc20174fcd8083", + "no_coverage_single_reads.depth.tsv:md5,7d71e53c7424f930cee76eee9fa91b04", + "assemblies_metadata.csv:md5,cfd0b7cab10c2f93d9dcff3435cff8ec", + "multiqc_assembly_metadata.txt:md5,4d08d8ca7b7d37c5db8fedeabfca82fe", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f" ] ], From ca5205702d6e3ed12f81a84db28b09b0a584fc12 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 15:45:34 +0000 Subject: [PATCH 28/30] keep only tax_id input arg instead of tax_id and scientific_name, because name is defined by the tax_id --- bin/register_coassembly_sample.py | 48 ++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/bin/register_coassembly_sample.py b/bin/register_coassembly_sample.py index 7ac4437..436a4e6 100755 --- a/bin/register_coassembly_sample.py +++ b/bin/register_coassembly_sample.py @@ -21,6 +21,7 @@ CHECKLIST = "ERC000011" CHECKLIST_XML_API = f"https://www.ebi.ac.uk/ena/browser/api/xml/{CHECKLIST}" +TAXONOMY_API = "https://www.ebi.ac.uk/ena/taxonomy/rest/tax-id" # ENA Portal API fields retrieved for runs and samples RUN_SAMPLE_FIELDS = ["sample_accession", "secondary_sample_accession"] @@ -68,8 +69,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--default-country", type=str, help="Default country to use if values differ or are missing.") parser.add_argument("--default-date", type=str, help="Default collection date to use if values differ or are missing.") - parser.add_argument("--default-taxid", type=str, help="Default taxon ID to use if values differ or are missing.") - parser.add_argument("--default-tax-name", type=str, help="Default taxon name to use if values differ or are missing.") + parser.add_argument("--default-taxid", type=str, help="Default taxon ID to use if values differ or are missing. The scientific name is fetched from ENA taxonomy.") return parser.parse_args() @@ -162,7 +162,7 @@ def merge_or_not_provided(values: list[str], default_value=None) -> str: logger.info(f"Merge decision: using unique value '{unique_values[0]}'") return unique_values[0] elif default_value: - logger.info(f"Merge decision: using default value '{default_value}' due to multiple/conflicting values or missing data") + logger.info(f"Merge decision: using default value '{default_value}' to overwrite samples metadata") return default_value else: logger.info(f"Merge decision: '{DEFAULT_NA_VALUE}' (multiple conflicting values or some missing)") @@ -240,6 +240,31 @@ def get_sample_metadata(sample_accession: str, private: bool = False) -> tuple[s row.get("collection_date"), ) +@retry(requests.RequestException, tries=3, delay=2, backoff=2) +def get_scientific_name(tax_id: str) -> str: + """ + Retrieve the scientific name for a taxon ID from the ENA taxonomy API. + + Args: + tax_id (str): The NCBI taxon ID. + + Returns: + str: The scientific name of the taxon. + + Raises: + CoassemblyRegistrationError: If the taxon is not found or cannot be used for submission. + """ + url = f"{TAXONOMY_API}/{tax_id}" + logger.debug(f"Requesting taxonomy URL: {url}") + response = requests.get(url, timeout=60) + if response.status_code == 404: + raise CoassemblyRegistrationError(f"Taxon ID {tax_id} not found in ENA taxonomy") + response.raise_for_status() + taxon = response.json() + if taxon.get("submittable") != "true": + raise CoassemblyRegistrationError(f"Taxon ID {tax_id} ({taxon.get('scientificName')}) is not submittable to ENA") + return taxon["scientificName"] + def get_allowed_countries() -> set[str]: """ Retrieve the set of allowed countries from the ENA checklist XML. @@ -381,7 +406,7 @@ def extract_accession_from_receipt(receipt: ET.Element) -> str: raise CoassemblyRegistrationError("ENA sample submission succeeded but no SAMPLE accession found in receipt XML") -def register_virtual_sample(run_accessions: list[str], test: bool, private: bool = False, default_country=None, default_date=None, default_taxid=None, default_tax_name=None) -> str: +def register_virtual_sample(run_accessions: list[str], test: bool, private: bool = False, default_country=None, default_date=None, default_taxid=None) -> str: """ Register a virtual sample for co-assemblies based on run accessions. @@ -431,19 +456,23 @@ def register_virtual_sample(run_accessions: list[str], test: bool, private: bool merged_collection_date = merge_or_not_provided(collection_dates, default_date) merged_country = merge_or_not_provided(countries, default_country) merged_taxid = merge_or_not_provided(taxon_ids, default_taxid) - merged_tax_name = merge_or_not_provided(scientific_names, default_tax_name) - if merged_taxid == DEFAULT_NA_VALUE or merged_tax_name == DEFAULT_NA_VALUE: + if merged_taxid == DEFAULT_NA_VALUE: sample_taxa = dict(zip(sample_accessions, zip(taxon_ids, scientific_names))) details = "\n".join( - f" {sample}: tax_id={taxid}, scientific_name={name}" + f" {sample}: tax_id={taxid} ({name})" for sample, (taxid, name) in sample_taxa.items() ) raise CoassemblyRegistrationError( - "Co-assembly source samples must share the same tax_id and scientific_name:\n" - f"{details}" + "Co-assembly source samples must share the same tax_id:\n" + f"{details}\n" + "Set the tax_id to register the co-assembly sample under with --default-taxid " + "(the 'tax_id' column of the samplesheet when running the pipeline)." ) + merged_tax_name = get_scientific_name(merged_taxid) + logger.info(f"Scientific name for tax_id {merged_taxid}: {merged_tax_name}") + allowed_countries = get_allowed_countries() if merged_country != DEFAULT_NA_VALUE and merged_country not in allowed_countries: logger.warning( @@ -521,7 +550,6 @@ def main() -> int: default_country=args.default_country, default_date=args.default_date, default_taxid=args.default_taxid, - default_tax_name=args.default_tax_name ) if virtual_sample or virtual_sample == "": row["Sample"] = virtual_sample From 1851faa841f7215d98c5a89ec221c23943e26ff8 Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 15:56:37 +0000 Subject: [PATCH 29/30] add tax_id column for co assembly submssion --- CHANGELOG.md | 1 + assets/schema_input_assembly.json | 16 ++++++++++++++++ docs/usage.md | 12 +++++++----- modules/local/register_coassembly_sample/main.nf | 2 ++ workflows/assemblysubmit.nf | 8 ++++---- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b78eae6..5bf3ad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#66](https://github.com/nf-core/seqsubmit/pull/66) - Added co-assembly submission to `ASSEMBLYSUBMIT`: a samplesheet row can list several semicolon-separated run accessions (and matching read files for coverage calculation) - When the source runs belong to different ENA samples, a virtual sample with a `composed of` attribute is registered and the co-assembly is submitted under it + - Optional `tax_id` samplesheet column to set the taxon of the virtual co-assembly sample when the source samples have different taxa ### `Fixed` diff --git a/assets/schema_input_assembly.json b/assets/schema_input_assembly.json index 4702d03..4bf1ae7 100644 --- a/assets/schema_input_assembly.json +++ b/assets/schema_input_assembly.json @@ -96,6 +96,22 @@ "description": "Assembly type: metagenome or metatranscriptome. Written to the ENA manifest as ASSEMBLY_TYPE 'primary metagenome' or 'metatranscriptome'. Default: metagenome", "default": "metagenome", "meta": ["assembly_type"] + }, + "tax_id": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0 + }, + { + "type": "string", + "maxLength": 0 + } + ], + "default": null, + "errorMessage": "Taxon ID must be a positive integer or empty", + "description": "NCBI taxon ID for the virtual sample registered for a co-assembly (optional). Provide when the source samples have different taxon IDs. Ignored for assemblies that are not co-assemblies of several samples", + "meta": ["tax_id"] } }, "required": ["id", "fasta", "run_accession", "assembler", "assembler_version"], diff --git a/docs/usage.md b/docs/usage.md index e573d8a..61691bb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -217,10 +217,11 @@ You will need to create a samplesheet with information about the metagenomic ass Example: ```csv title="samplesheet_assembly.csv" -id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version,assembly_type -assembly_001,data/assembly_001.fasta.gz,data/assembly_001_R1.fastq.gz,data/assembly_001_R2.fastq.gz,,ERR011322,SPAdes,3.15.5,metagenome -assembly_002,data/assembly_002.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9,metatranscriptome -coassembly_001,data/coassembly_001.fasta.gz,data/run1_R1.fastq.gz;data/run2_R1.fastq.gz,data/run1_R2.fastq.gz;data/run2_R2.fastq.gz,,ERR011324;ERR011325,MEGAHIT,1.2.9, +id,fasta,fastq_1,fastq_2,coverage,run_accession,assembler,assembler_version,assembly_type,tax_id +assembly_001,data/assembly_001.fasta.gz,data/assembly_001_R1.fastq.gz,data/assembly_001_R2.fastq.gz,,ERR011322,SPAdes,3.15.5,metagenome, +assembly_002,data/assembly_002.fasta.gz,,,42.7,ERR011323,MEGAHIT,1.2.9,metatranscriptome, +coassembly_001,data/coassembly_001.fasta.gz,data/run1_R1.fastq.gz;data/run2_R1.fastq.gz,data/run1_R2.fastq.gz;data/run2_R2.fastq.gz,,ERR011324;ERR011325,MEGAHIT,1.2.9,, +coassembly_002,data/coassembly_002.fasta.gz,,,35.2,ERR011326;ERR011327,MEGAHIT,1.2.9,metagenome,408172 ``` | Column | Required | Description | @@ -234,6 +235,7 @@ coassembly_001,data/coassembly_001.fasta.gz,data/run1_R1.fastq.gz;data/run2_R1.f | `assembler` | Yes | Name of the assembler software used to generate the assembly, e.g. MEGAHIT, SPAdes, metaSPAdes, Flye, etc. | | `assembler_version` | Yes | Version of the assembler software used to generate the assembly, e.g. 1.0.0. | | `assembly_type` | No | Sequencing library source: `metagenome` for metagenome assembly (would be used as _'primary metagenome'_ in manifest file following [ENA docs](https://ena-docs.readthedocs.io/en/latest/submit/assembly/metagenome/primary.html)) or `metatranscriptome` for metatranscriptome assembly (woule be used _'metatranscriptome'_ in assembly manifest following [ENA docs](https://ena-docs.readthedocs.io/en/latest/submit/assembly/metatranscriptome.html)). Default: `metagenome` | +| `tax_id` | No | NCBI taxon ID for the virtual sample registered for a co-assembly, e.g. `408172` (marine metagenome). Used only when the source samples have different taxon IDs; see [Co-assemblies](#co-assemblies). | Provide either read files (`fastq_1`, optionally `fastq_2`) or a `coverage` value for each row. If `coverage` is missing and reads are provided, the workflow calculates average coverage with CoverM. See the [Methods documentation](docs/methods.md) for more information on how coverage is calculated. @@ -246,7 +248,7 @@ ENA requires each assembly to reference a single sample. The pipeline handles th - If all source runs belong to the same ENA sample, the assembly is submitted under that sample. - If the runs belong to different samples, the pipeline registers a virtual ENA sample with a `composed of` attribute listing the source samples, and submits the assembly under it. The virtual sample uses the [ENA default sample checklist (`ERC000011`)](https://www.ebi.ac.uk/ena/browser/view/ERC000011), the minimal checklist, because its mandatory fields (`collection date` and `geographic location (country and/or sea)`) can be derived from any source samples, whichever checklists they were originally registered with. Its metadata is merged from the source samples: - `collection date` and `geographic location (country and/or sea)` are kept if all source samples share the same value, otherwise they are set to `not provided`. A country that is not in the `ERC000011` list of allowed values is also set to `not provided`. - - `tax_id` and `scientific_name` must be the same for all source samples; otherwise the pipeline stops with an error. + - `tax_id` must be the same for all source samples. If it differs, set the `tax_id` column for that samplesheet row to the taxon the co-assembly should be registered under (e.g. `408172` for marine metagenome when combining marine and marine sediment samples); otherwise the submission fails. The scientific name is fetched from the [ENA taxonomy](https://ena-docs.readthedocs.io/en/latest/faq/taxonomy.html) for the final `tax_id`. ### `metagenomic_assemblies` parameters and example diff --git a/modules/local/register_coassembly_sample/main.nf b/modules/local/register_coassembly_sample/main.nf index 6d7e132..ba5f971 100644 --- a/modules/local/register_coassembly_sample/main.nf +++ b/modules/local/register_coassembly_sample/main.nf @@ -26,12 +26,14 @@ process REGISTER_COASSEMBLY_SAMPLE { def prefix = task.ext.prefix ?: "${meta.id}" def test_arg = test ? "--test" : "" def private_arg = is_private ? "--private" : "" + def taxid_arg = meta.tax_id ? "--default-taxid ${meta.tax_id}" : "" """ register_coassembly_sample.py \\ --input ${csv} \\ --output ${prefix}_updated.csv \\ ${test_arg} \\ ${private_arg} \\ + ${taxid_arg} \\ ${args} """ diff --git a/workflows/assemblysubmit.nf b/workflows/assemblysubmit.nf index 7cdb1d7..31a0503 100644 --- a/workflows/assemblysubmit.nf +++ b/workflows/assemblysubmit.nf @@ -52,7 +52,7 @@ workflow ASSEMBLYSUBMIT { // --------- Create assembly channel with proper metadata structure assembly_fasta = ch_samplesheet - .map { meta, fasta, reads_1, reads_2 -> + .map { meta, fasta, _reads_1, reads_2 -> // support semicolon-separated values for co-assemblies (e.g. ERR000001;ERR000002) def run_accessions = meta.run_accession.split(';')?.toList()?.unique() ?: [] def new_meta = meta + [ @@ -65,7 +65,7 @@ workflow ASSEMBLYSUBMIT { // --------- Create reads channel with proper metadata structure reads_fastq = ch_samplesheet .filter { row -> row[2] && row[2] != "" } // Check if fastq_1 exists and is not empty - .map { meta, fasta, reads_1, reads_2 -> + .map { meta, _fasta, reads_1, reads_2 -> // support semicolon-separated values for co-assemblies (e.g. ERR000001;ERR000002) def run_accessions = meta.run_accession.split(';')?.toList()?.unique() ?: [] def new_meta = meta + [ @@ -176,7 +176,7 @@ workflow ASSEMBLYSUBMIT { assembly_metadata_by_type = CREATE_ASSEMBLY_METADATA_CSV.out.csv .branch { meta, _csv -> coassembly: meta.run_accession instanceof List && meta.run_accession.size() > 1 - standard: true + individual_assembly: true } REGISTER_COASSEMBLY_SAMPLE( @@ -185,7 +185,7 @@ workflow ASSEMBLYSUBMIT { is_private ) - assembly_metadata_with_sample = assembly_metadata_by_type.standard + assembly_metadata_with_sample = assembly_metadata_by_type.individual_assembly .mix(REGISTER_COASSEMBLY_SAMPLE.out.csv) // --------- Concatenate assembly metadata CSVs into single file to publish From 9d2a06ad80382f0be19205fe546c4eeeb7e77cae Mon Sep 17 00:00:00 2001 From: Sofia Ochkalova Date: Thu, 29 Oct 2026 16:13:33 +0000 Subject: [PATCH 30/30] fix errorStrategy definition in GENERATE_ASSEMBLY_MANIFEST --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 7775dad..c0ee3be 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -190,7 +190,7 @@ process { } withName: 'GENERATE_ASSEMBLY_MANIFEST' { - errorStrategy = { task.exitStatus in (1 + (130..145) + 104 + (175..177)) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + 1 + (175..177)) ? 'retry' : 'finish' } maxRetries = 3 publishDir = [ enabled: false