From 59c2216e21d717aec67adba7ab390b40f2ae9172 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 18 Sep 2026 11:38:19 +0200 Subject: [PATCH 01/36] make passing variants the only form pushed through to downstream analysis in Deepvariant by default (toggleable) --- CHANGELOG.md | 1 + conf/modules.config | 22 +++++++++ docs/usage.md | 18 ++++++++ nextflow.config | 2 + nextflow_schema.json | 14 ++++++ .../local/paired/paired_smallvar_germline.nf | 19 ++++++++ .../local/paired/paired_smallvar_somatic.nf | 19 ++++++++ .../local/tumor_only/tumoronly_smallvar.nf | 38 +++++++++++++++ tests/consensus.nf.test.snap | 42 +++++++++-------- tests/deep_only.nf.test.snap | 26 +++++++---- tests/union.nf.test.snap | 46 +++++++++++-------- 11 files changed, 199 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41af5f5d..42204985 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - DeepVariant and DeepSomatic output is now restricted to `PASS` records before it is used downstream, controlled by the new `--deepvariant_filter_pass` and `--deepsomatic_filter_pass` parameters (both `true` by default). Both callers emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. ClairS/ClairS-TO calls are already `PASS`-only downstream (`VCFSPLIT`), so with the default `*_var_combine = 'all'` the union was "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP, signature fitting and the report see far fewer variants. Set `--deepsomatic_filter_pass false` or `--deepvariant_filter_pass false` for the previous behaviour. The raw per-caller VCFs published under `variants/deepsomatic` and `variants/deepvariant` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). - [#186](https://github.com/IntGenomicsLab/lrsomatic/pull/186) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.0, which adds tickbox dropdown filters on the categorical columns of both variant tables and turns the report's gene panel selector into checkboxes (@ljwharbers). - [#176](https://github.com/IntGenomicsLab/lrsomatic/pull/176) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.2: facet dropdown counts follow the active filters, opening a facet menu no longer resets the table's horizontal scroll, a flatter clinical theme, and inline code comments trimmed to one line (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 9a46a239..b360097b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -650,6 +650,28 @@ process { ] } + // DeepVariant and DeepSomatic emit every evaluated site, not just calls. These + // filtered copies are what the consensus, phasing, VEP, signature and report steps + // consume; the raw per-caller VCFs under variants/deepvariant and variants/deepsomatic + // are published from the POSTPROCESSVARIANTS processes and are left untouched. + // --write-index is required because the module's index output is optional, and the + // downstream join() would otherwise drop every sample. + withName : '.*:DEEPVARIANT_PASS_FILTER' { + ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } + ext.prefix = { "${meta.id}_germline_pass" } + publishDir = [ + enabled: false + ] + } + + withName : '.*:DEEPSOMATIC_PASS_FILTER' { + ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } + ext.prefix = { "${meta.id}_somatic_pass" } + publishDir = [ + enabled: false + ] + } + withName : '.*:SIGPROFILER_MATRIXGENERATOR' { ext.args = { params.sigprofiler_matrix_args ?: '' } publishDir = [ diff --git a/docs/usage.md b/docs/usage.md index 33dcf18e..8b29b096 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -361,6 +361,24 @@ These options control how variants from multiple callers are filtered and merged | `--somatic_var_combine` | Strategy for combining somatic variant caller outputs (e.g. union, intersection). Default = `null` | | `--prioritize_caller_germline` | Comma-separated caller priority order used when combining germline calls. Default = `null` | | `--prioritize_caller_somatic` | Comma-separated caller priority order used when combining somatic calls. Default = `null` | +| `--deepvariant_filter_pass` | Keep only PASS records from DeepVariant for downstream use. Default = `true` | +| `--deepsomatic_filter_pass` | Keep only PASS records from DeepSomatic for downstream use. Default = `true` | + +DeepVariant and DeepSomatic emit a record for every site they evaluate, not only +for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds +around 13.7 M records of which roughly 50 k are `PASS`, the rest being `RefCall`, +`GERMLINE` or `PON`. ClairS/ClairS-TO output is already restricted to `PASS` +before it is used downstream, so with `*_var_combine = 'all'` the union would +otherwise be "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at", +which inflates the phased VCFs by three orders of magnitude and produces a +meaningless mutation burden. + +`--deepvariant_filter_pass` and `--deepsomatic_filter_pass` (both `true` by +default) restrict the copy handed to the caller consensus, phasing, VEP, signature +fitting and the report. Set either to `false` to restore the previous unfiltered +behaviour. The raw per-caller VCFs published under +`//variants/deepvariant` and `//variants/deepsomatic` +are never filtered, so no calls are lost from the results directory. #### PON Options diff --git a/nextflow.config b/nextflow.config index ada7c82a..f64eef4a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -20,6 +20,8 @@ params { somatic_var_combine = 'all' prioritize_caller_germline = 'clair' prioritize_caller_somatic = 'clair' + deepvariant_filter_pass = true + deepsomatic_filter_pass = true generate_gvcf = false // Longphase options diff --git a/nextflow_schema.json b/nextflow_schema.json index d5e93d85..e1e17365 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -101,6 +101,20 @@ "default": "clair", "enum": ["deepsomatic", "clair"] }, + "deepvariant_filter_pass": { + "type": "boolean", + "default": true, + "description": "Keep only PASS records from DeepVariant for downstream use.", + "help_text": "DeepVariant emits every evaluated site, so most records are RefCall rather than calls. Those records otherwise flow into the caller consensus, phasing, VEP and the report. Set to false to consume the unfiltered DeepVariant output. The raw VCF published to `//variants/deepvariant` is unaffected either way.", + "fa_icon": "fas fa-filter" + }, + "deepsomatic_filter_pass": { + "type": "boolean", + "default": true, + "description": "Keep only PASS records from DeepSomatic for downstream use.", + "help_text": "DeepSomatic emits every evaluated site, so most records are RefCall, GERMLINE or PON rather than somatic calls. Those records otherwise flow into the caller consensus, phasing, VEP, signature fitting and the report. Set to false to consume the unfiltered DeepSomatic output. The raw VCF published to `//variants/deepsomatic` is unaffected either way.", + "fa_icon": "fas fa-filter" + }, "generate_gvcf": { "type": "boolean" } diff --git a/subworkflows/local/paired/paired_smallvar_germline.nf b/subworkflows/local/paired/paired_smallvar_germline.nf index 3b473006..0ae9a23c 100644 --- a/subworkflows/local/paired/paired_smallvar_germline.nf +++ b/subworkflows/local/paired/paired_smallvar_germline.nf @@ -1,5 +1,6 @@ // IMPORT MODULES include { CLAIR3 } from '../../../modules/local/clair3/main.nf' +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' @@ -121,6 +122,24 @@ workflow PAIRED_SMALLVAR_GERMLINE { DEEPVARIANT.out.vcf .join(DEEPVARIANT.out.vcf_index) + .set{deepvariant_raw_ch} + + // DeepVariant emits every evaluated site, not just calls, so most records are + // RefCall. Clair3 is already PASS-filtered downstream, so without this the + // union is "PASS Clair3 + every site DeepVariant evaluated". + // The raw VCF published under variants/deepvariant/ is unaffected. + if (params.deepvariant_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) + DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index) + .set{deepvariant_pass_ch} + } + else { + deepvariant_raw_ch + .set{deepvariant_pass_ch} + } + + deepvariant_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/paired/paired_smallvar_somatic.nf b/subworkflows/local/paired/paired_smallvar_somatic.nf index cf5a749d..d6ac19c4 100644 --- a/subworkflows/local/paired/paired_smallvar_somatic.nf +++ b/subworkflows/local/paired/paired_smallvar_somatic.nf @@ -2,6 +2,7 @@ include { CLAIRS } from '../../../modules/local/clairs/main.nf' include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat' include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' @@ -111,6 +112,24 @@ workflow PAIRED_SMALLVAR_SOMATIC { DEEPSOMATIC.out.vcf .join(DEEPSOMATIC.out.vcf_index) + .set{deepsomatic_raw_ch} + + // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. + // ClairS is already PASS-filtered downstream, so without this the union is + // "PASS ClairS + every site DeepSomatic evaluated". + // The raw VCF published under variants/deepsomatic/ is unaffected. + if (params.deepsomatic_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) + DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index) + .set{deepsomatic_pass_ch} + } + else { + deepsomatic_raw_ch + .set{deepsomatic_pass_ch} + } + + deepsomatic_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 10f18b59..796bc0ab 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -1,6 +1,8 @@ // IMPORT MODULES include { CLAIRSTO } from '../../../modules/local/clairsto/main.nf' include { VCFSPLIT } from '../../../modules/local/vcfsplit/main.nf' +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' @@ -126,6 +128,24 @@ workflow TUMORONLY_SMALLVAR { DEEPVARIANT.out.vcf .join(DEEPVARIANT.out.vcf_index) + .set{deepvariant_raw_ch} + + // DeepVariant emits every evaluated site, not just calls, so most records are + // RefCall. ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without + // this the union is "PASS ClairS-TO + every site DeepVariant evaluated". + // The raw VCF published under variants/deepvariant/ is unaffected. + if (params.deepvariant_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) + DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index) + .set{deepvariant_pass_ch} + } + else { + deepvariant_raw_ch + .set{deepvariant_pass_ch} + } + + deepvariant_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] @@ -194,6 +214,24 @@ workflow TUMORONLY_SMALLVAR { ) DEEPSOMATIC.out.vcf .join(DEEPSOMATIC.out.vcf_index) + .set{deepsomatic_raw_ch} + + // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. + // ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without this the + // union is "PASS ClairS-TO + every site DeepSomatic evaluated". + // The raw VCF published under variants/deepsomatic/ is unaffected. + if (params.deepsomatic_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) + DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index) + .set{deepsomatic_pass_ch} + } + else { + deepsomatic_raw_ch + .set{deepsomatic_pass_ch} + } + + deepsomatic_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index 5123680d..d0a4150e 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -47,6 +47,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -56,6 +59,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -575,38 +581,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,93dd8ac8b67eb4eb4bf27e09c8f5f99b", - "sample1_normal.bam.bai:md5,75402ef1cc35229cc131155d9ec973e0", - "sample1_tumor.bam:md5,69cba03cad51bcc1d1ee8c48da042527", - "sample1_tumor.bam.bai:md5,5d633ed05021ad81ce24b1f18cbf38b4", + "sample1_normal.bam:md5,28ad28688852dfe5c2527c88fe52d2c7", + "sample1_normal.bam.bai:md5,3f1a1e308eb5f7b6a3805e76ff7034a3", + "sample1_tumor.bam:md5,e25dec2664b709de329909c5f1be74f2", + "sample1_tumor.bam.bai:md5,04d8b757427e0ff00b19aa338c681a54", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,9f09f9ad1a788384cb8e46a933f77b3b", - "sample1_whatshap_stats.log:md5,20135b4e9965a31d3f9bb0df7d2cec90", - "sample1_whatshap_stats.tsv:md5,264d2d76a9b8d34ea4933aee325ce36e", + "sample1_whatshap_stats.gtf:md5,19483ff8a25ef082cde1d2c7d5faac6f", + "sample1_whatshap_stats.log:md5,ede92982bd2c122b30c5625ff1627c46", + "sample1_whatshap_stats.tsv:md5,4c347cec00fefec30623219a865e3eb3", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,f2ba30d007c521d479c6158e1e22367a", - "sample2_normal.bam.bai:md5,c3096f52115ec1e24c46fedc41f1f3d3", - "sample2_tumor.bam:md5,1c0287d24fa5b25b86e48024f2f55031", - "sample2_tumor.bam.bai:md5,62849cea5a005e3d8dbe8f9edcefaf60", + "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", + "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", + "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,f15fb43f0af73d02fc73b66fdc12d5d8", - "sample2_whatshap_stats.log:md5,ca87088fc2f11665eca3fb9c80489085", - "sample2_whatshap_stats.tsv:md5,ca53f81e39bf5d46aa4f604216add1f6", + "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", + "sample2_whatshap_stats.log:md5,f9628717e0c4944888687c13098c62eb", + "sample2_whatshap_stats.tsv:md5,4fd295d10a8c461fd2c91851027503a4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -629,10 +635,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:55:50.117252839", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T11:10:43.315178155" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 6a48eeaa..4c20adca 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -23,6 +23,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -32,6 +35,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -531,8 +537,8 @@ "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", "sample1_whatshap_stats.gtf:md5,e1d0e87353a5f9aed8a9ac4bf7973427", - "sample1_whatshap_stats.log:md5,bd6b83a062e22cd3201523dc4c2c13e7", - "sample1_whatshap_stats.tsv:md5,7a1508751cb1daa841a577ae25f55586", + "sample1_whatshap_stats.log:md5,43811a1aa4726c2ef62646ff22d1fad0", + "sample1_whatshap_stats.tsv:md5,a821c5d0e3451f82327645fee89e68eb", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", @@ -550,8 +556,8 @@ "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", "sample2_whatshap_stats.gtf:md5,af33281699a1d0da83fbe7eaff198d03", - "sample2_whatshap_stats.log:md5,bbd9ab2ce07a009d9348a1d78bc6fc70", - "sample2_whatshap_stats.tsv:md5,c65436f930c23ddbfd568532d07dce70", + "sample2_whatshap_stats.log:md5,0cd536df69e244a5271e9c5440ea0f3f", + "sample2_whatshap_stats.tsv:md5,c7cc47024ef622a72e7f4f0d5fa119eb", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -564,8 +570,8 @@ "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,4f7648e763004ab764143cb4f8b6499e", - "sample3_whatshap_stats.tsv:md5,4cb58bb3b663aaba23da004d69adab3e", + "sample3_whatshap_stats.log:md5,061ecb622d83678229f5d0a7b0e95d75", + "sample3_whatshap_stats.tsv:md5,7c7453a5eed17e9ccc405844f3bd511a", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -574,10 +580,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:51:30.278291697", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T11:19:11.135286604" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 08d81773..2e110695 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -44,6 +44,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -53,6 +56,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -575,38 +581,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,cbfc940a38c74cbe8435c18b9da5dd32", - "sample1_normal.bam.bai:md5,c1498328929d45b2898fa2265b0d617c", - "sample1_tumor.bam:md5,b78866edf991393806d37505d16f7e3d", - "sample1_tumor.bam.bai:md5,f613de14ab19fc3a85403661a4f6188c", + "sample1_normal.bam:md5,cd28bdb7edefa810b86a12029783e1b6", + "sample1_normal.bam.bai:md5,6c19fcce6bd9050fb03a00e66ba1b866", + "sample1_tumor.bam:md5,dcb89119e272efbe0168d1737f714a8a", + "sample1_tumor.bam.bai:md5,9577c39e6f59056421a8192b0ba43acc", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,9ae556e13516dd47d4108acf2104bddb", - "sample1_whatshap_stats.log:md5,eaddcf6a1666d4a3c1ad3316dac24139", - "sample1_whatshap_stats.tsv:md5,c2773e011c2781160fd9a7741b10546b", + "sample1_whatshap_stats.gtf:md5,43773d064cd353d1dbbae45adb3be334", + "sample1_whatshap_stats.log:md5,13f4530236189960576b5bc45e63bc1a", + "sample1_whatshap_stats.tsv:md5,bf14b5a1fd9d95797f36a48e7e16953d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,f2ba30d007c521d479c6158e1e22367a", - "sample2_normal.bam.bai:md5,c3096f52115ec1e24c46fedc41f1f3d3", - "sample2_tumor.bam:md5,1c0287d24fa5b25b86e48024f2f55031", - "sample2_tumor.bam.bai:md5,62849cea5a005e3d8dbe8f9edcefaf60", + "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", + "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", + "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,f15fb43f0af73d02fc73b66fdc12d5d8", - "sample2_whatshap_stats.log:md5,a6767b3490cafdcbaf3b7114644028de", - "sample2_whatshap_stats.tsv:md5,570796e5e291229e8872733425e0b133", + "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", + "sample2_whatshap_stats.log:md5,af84d397bf163be8de77fc3c7401493b", + "sample2_whatshap_stats.tsv:md5,fb38bde18fbfc61e52947b68d4260829", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -619,8 +625,8 @@ "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,679dcfa209888a9e69a07e4c4e4b049e", - "sample3_whatshap_stats.tsv:md5,035d5aa0425ba3fc32d65268b793b424", + "sample3_whatshap_stats.log:md5,496a531c3c19585603fa7fded0c958ba", + "sample3_whatshap_stats.tsv:md5,fba638c8da0d7a9fdacb11734e219b16", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -629,10 +635,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:55:27.445799687", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T11:35:45.671299133" } } \ No newline at end of file From c596e1b92f66b73286378cf87000ed672de9e2bf Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 18 Sep 2026 18:04:28 +0200 Subject: [PATCH 02/36] fixes to the filtering --- CHANGELOG.md | 2 +- conf/modules.config | 38 ++++++++---- docs/usage.md | 26 ++++----- nextflow.config | 3 +- nextflow_schema.json | 13 +---- .../local/paired/paired_smallvar_germline.nf | 39 ++++++------- .../local/paired/paired_smallvar_somatic.nf | 39 ++++++------- .../local/tumor_only/tumoronly_smallvar.nf | 58 ++++++------------- subworkflows/local/vcf_pass_filter.nf | 40 +++++++++++++ tests/clair_only.nf.test.snap | 43 +++++++------- tests/consensus.nf.test.snap | 35 +++++------ tests/deep_only.nf.test.snap | 11 ++-- tests/default.nf.test.snap | 39 +++++++------ tests/union.nf.test | 42 ++++++++++++++ tests/union.nf.test.snap | 35 +++++------ 15 files changed, 259 insertions(+), 204 deletions(-) create mode 100644 subworkflows/local/vcf_pass_filter.nf diff --git a/CHANGELOG.md b/CHANGELOG.md index 42204985..e316511a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - DeepVariant and DeepSomatic output is now restricted to `PASS` records before it is used downstream, controlled by the new `--deepvariant_filter_pass` and `--deepsomatic_filter_pass` parameters (both `true` by default). Both callers emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. ClairS/ClairS-TO calls are already `PASS`-only downstream (`VCFSPLIT`), so with the default `*_var_combine = 'all'` the union was "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP, signature fitting and the report see far fewer variants. Set `--deepsomatic_filter_pass false` or `--deepvariant_filter_pass false` for the previous behaviour. The raw per-caller VCFs published under `variants/deepsomatic` and `variants/deepvariant` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. Clair3 and ClairS are far less extreme but still kept their `LowQual` and `NonSomatic` records, so with the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`, and signature fitting is unchanged because `SIGNATURES_BCFTOOLS_VIEW` already applied its own `PASS` filter. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP and the report see far fewer variants. Set `--smallvar_filter_pass false` for the previous behaviour. The per-caller VCFs published under `variants/deepvariant`, `variants/deepsomatic`, `variants/clair3` and `variants/clairs` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). - [#186](https://github.com/IntGenomicsLab/lrsomatic/pull/186) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.0, which adds tickbox dropdown filters on the categorical columns of both variant tables and turns the report's gene panel selector into checkboxes (@ljwharbers). - [#176](https://github.com/IntGenomicsLab/lrsomatic/pull/176) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.2: facet dropdown counts follow the active filters, opening a facet menu no longer resets the table's horizontal scroll, a flatter clinical theme, and inline code comments trimmed to one line (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index b360097b..078a1db1 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -650,23 +650,41 @@ process { ] } - // DeepVariant and DeepSomatic emit every evaluated site, not just calls. These - // filtered copies are what the consensus, phasing, VEP, signature and report steps - // consume; the raw per-caller VCFs under variants/deepvariant and variants/deepsomatic - // are published from the POSTPROCESSVARIANTS processes and are left untouched. + // VCF_PASS_FILTER copies, one alias per small variant caller. DeepVariant and + // DeepSomatic emit every evaluated site rather than only their calls, and Clair3/ClairS + // keep their LowQual and NonSomatic records; these filtered copies are what the + // consensus, phasing, VEP and report steps consume. The per-caller VCFs under + // variants// are published from the caller processes and are left untouched. + // ClairS-TO has no entry here: VCFSPLIT already restricts it to PASS. // --write-index is required because the module's index output is optional, and the // downstream join() would otherwise drop every sample. - withName : '.*:DEEPVARIANT_PASS_FILTER' { - ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } - ext.prefix = { "${meta.id}_germline_pass" } + withName: '.*:DEEPVARIANT_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_deepvariant_pass" } publishDir = [ enabled: false ] } - withName : '.*:DEEPSOMATIC_PASS_FILTER' { - ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } - ext.prefix = { "${meta.id}_somatic_pass" } + withName: '.*:DEEPSOMATIC_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_deepsomatic_pass" } + publishDir = [ + enabled: false + ] + } + + withName: '.*:CLAIR3_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_clair3_pass" } + publishDir = [ + enabled: false + ] + } + + withName: '.*:CLAIRS_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_clairs_pass" } publishDir = [ enabled: false ] diff --git a/docs/usage.md b/docs/usage.md index 8b29b096..488fd69f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -361,24 +361,22 @@ These options control how variants from multiple callers are filtered and merged | `--somatic_var_combine` | Strategy for combining somatic variant caller outputs (e.g. union, intersection). Default = `null` | | `--prioritize_caller_germline` | Comma-separated caller priority order used when combining germline calls. Default = `null` | | `--prioritize_caller_somatic` | Comma-separated caller priority order used when combining somatic calls. Default = `null` | -| `--deepvariant_filter_pass` | Keep only PASS records from DeepVariant for downstream use. Default = `true` | -| `--deepsomatic_filter_pass` | Keep only PASS records from DeepSomatic for downstream use. Default = `true` | +| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds around 13.7 M records of which roughly 50 k are `PASS`, the rest being `RefCall`, -`GERMLINE` or `PON`. ClairS/ClairS-TO output is already restricted to `PASS` -before it is used downstream, so with `*_var_combine = 'all'` the union would -otherwise be "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at", -which inflates the phased VCFs by three orders of magnitude and produces a -meaningless mutation burden. - -`--deepvariant_filter_pass` and `--deepsomatic_filter_pass` (both `true` by -default) restrict the copy handed to the caller consensus, phasing, VEP, signature -fitting and the report. Set either to `false` to restore the previous unfiltered -behaviour. The raw per-caller VCFs published under -`//variants/deepvariant` and `//variants/deepsomatic` -are never filtered, so no calls are lost from the results directory. +`GERMLINE` or `PON`. Clair3 and ClairS are far less extreme but still keep their +`LowQual` and `NonSomatic` records. With `*_var_combine = 'all'` the union would +otherwise be "every site every caller looked at", which inflates the phased VCFs +by three orders of magnitude and produces a meaningless mutation burden. + +`--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's +VCF that is handed to the caller consensus, phasing, VEP and the report. Set it to +`false` to restore the previous unfiltered behaviour. In tumor-only mode ClairS-TO +is unaffected by the setting: `VCFSPLIT` already restricts it to `PASS`. The +per-caller VCFs published under `//variants/` are never +filtered, so no calls are lost from the results directory. #### PON Options diff --git a/nextflow.config b/nextflow.config index f64eef4a..3e213473 100644 --- a/nextflow.config +++ b/nextflow.config @@ -20,8 +20,7 @@ params { somatic_var_combine = 'all' prioritize_caller_germline = 'clair' prioritize_caller_somatic = 'clair' - deepvariant_filter_pass = true - deepsomatic_filter_pass = true + smallvar_filter_pass = true generate_gvcf = false // Longphase options diff --git a/nextflow_schema.json b/nextflow_schema.json index e1e17365..4dc48241 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -101,18 +101,11 @@ "default": "clair", "enum": ["deepsomatic", "clair"] }, - "deepvariant_filter_pass": { + "smallvar_filter_pass": { "type": "boolean", "default": true, - "description": "Keep only PASS records from DeepVariant for downstream use.", - "help_text": "DeepVariant emits every evaluated site, so most records are RefCall rather than calls. Those records otherwise flow into the caller consensus, phasing, VEP and the report. Set to false to consume the unfiltered DeepVariant output. The raw VCF published to `//variants/deepvariant` is unaffected either way.", - "fa_icon": "fas fa-filter" - }, - "deepsomatic_filter_pass": { - "type": "boolean", - "default": true, - "description": "Keep only PASS records from DeepSomatic for downstream use.", - "help_text": "DeepSomatic emits every evaluated site, so most records are RefCall, GERMLINE or PON rather than somatic calls. Those records otherwise flow into the caller consensus, phasing, VEP, signature fitting and the report. Set to false to consume the unfiltered DeepSomatic output. The raw VCF published to `//variants/deepsomatic` is unaffected either way.", + "description": "Keep only PASS records from each small variant caller for downstream use.", + "help_text": "DeepVariant and DeepSomatic emit a record for every site they evaluate, so most records are RefCall (or GERMLINE/PON) rather than calls, and Clair3/ClairS keep their LowQual and NonSomatic calls. Those records otherwise flow into the caller consensus, phasing, VEP and the report. Set to false to consume each caller's unfiltered output. The per-caller VCFs published under `//variants/` are unaffected either way.", "fa_icon": "fas fa-filter" }, "generate_gvcf": { diff --git a/subworkflows/local/paired/paired_smallvar_germline.nf b/subworkflows/local/paired/paired_smallvar_germline.nf index 0ae9a23c..798e5b3f 100644 --- a/subworkflows/local/paired/paired_smallvar_germline.nf +++ b/subworkflows/local/paired/paired_smallvar_germline.nf @@ -1,10 +1,11 @@ // IMPORT MODULES include { CLAIR3 } from '../../../modules/local/clair3/main.nf' -include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' +include { VCF_PASS_FILTER as CLAIR3_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_GERMLINE { @@ -74,8 +75,14 @@ workflow PAIRED_SMALLVAR_GERMLINE { fai ) - CLAIR3.out.vcf - .join(CLAIR3.out.tbi) + // Clair3's merge_output.vcf.gz keeps LowQual and RefCall records; restrict them the + // same way DeepVariant's output is restricted so both callers enter the union on + // equal terms. The VCF published under variants/clair3/ is unaffected. + CLAIR3_PASS_FILTER ( + CLAIR3.out.vcf.join(CLAIR3.out.tbi) + ) + + CLAIR3_PASS_FILTER.out.vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clair3'] return [new_meta, vcf, tbi] @@ -120,26 +127,14 @@ workflow PAIRED_SMALLVAR_GERMLINE { [[:],[]] // GFF annotation (not used) ) - DEEPVARIANT.out.vcf - .join(DEEPVARIANT.out.vcf_index) - .set{deepvariant_raw_ch} - - // DeepVariant emits every evaluated site, not just calls, so most records are - // RefCall. Clair3 is already PASS-filtered downstream, so without this the - // union is "PASS Clair3 + every site DeepVariant evaluated". - // The raw VCF published under variants/deepvariant/ is unaffected. - if (params.deepvariant_filter_pass) { - DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) - DEEPVARIANT_PASS_FILTER.out.vcf - .join(DEEPVARIANT_PASS_FILTER.out.index) - .set{deepvariant_pass_ch} - } - else { - deepvariant_raw_ch - .set{deepvariant_pass_ch} - } + // DeepVariant emits a record for every site it evaluates, not just its calls, so + // most records are RefCall. Without this the union is "every site DeepVariant + // looked at". The VCF published under variants/deepvariant/ is unaffected. + DEEPVARIANT_PASS_FILTER ( + DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + ) - deepvariant_pass_ch + DEEPVARIANT_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/paired/paired_smallvar_somatic.nf b/subworkflows/local/paired/paired_smallvar_somatic.nf index d6ac19c4..953ed01d 100644 --- a/subworkflows/local/paired/paired_smallvar_somatic.nf +++ b/subworkflows/local/paired/paired_smallvar_somatic.nf @@ -2,11 +2,12 @@ include { CLAIRS } from '../../../modules/local/clairs/main.nf' include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat' include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort' -include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' +include { VCF_PASS_FILTER as CLAIRS_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_SOMATIC { @@ -72,8 +73,14 @@ workflow PAIRED_SMALLVAR_SOMATIC { BCFTOOLS_CONCAT.out.vcf ) - BCFTOOLS_SORT.out.vcf - .join(BCFTOOLS_SORT.out.tbi) + // The concatenated ClairS VCF still carries its LowQual and NonSomatic calls; + // restrict them the same way DeepSomatic's output is restricted so both callers + // enter the union on equal terms. variants/clairs/ is unaffected. + CLAIRS_PASS_FILTER ( + BCFTOOLS_SORT.out.vcf.join(BCFTOOLS_SORT.out.tbi) + ) + + CLAIRS_PASS_FILTER.out.vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clairs'] return [new_meta, vcf, tbi] @@ -110,26 +117,14 @@ workflow PAIRED_SMALLVAR_SOMATIC { ds_pon_channel ) - DEEPSOMATIC.out.vcf - .join(DEEPSOMATIC.out.vcf_index) - .set{deepsomatic_raw_ch} - - // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. - // ClairS is already PASS-filtered downstream, so without this the union is - // "PASS ClairS + every site DeepSomatic evaluated". - // The raw VCF published under variants/deepsomatic/ is unaffected. - if (params.deepsomatic_filter_pass) { - DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) - DEEPSOMATIC_PASS_FILTER.out.vcf - .join(DEEPSOMATIC_PASS_FILTER.out.index) - .set{deepsomatic_pass_ch} - } - else { - deepsomatic_raw_ch - .set{deepsomatic_pass_ch} - } + // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), + // not just its calls. Without this the union is "every site DeepSomatic looked + // at". The VCF published under variants/deepsomatic/ is unaffected. + DEEPSOMATIC_PASS_FILTER ( + DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + ) - deepsomatic_pass_ch + DEEPSOMATIC_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 796bc0ab..299bc0c7 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -1,14 +1,14 @@ // IMPORT MODULES include { CLAIRSTO } from '../../../modules/local/clairsto/main.nf' include { VCFSPLIT } from '../../../modules/local/vcfsplit/main.nf' -include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' -include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' +include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow TUMORONLY_SMALLVAR { @@ -126,26 +126,15 @@ workflow TUMORONLY_SMALLVAR { [[:],[]] // GFF annotation (not used) ) - DEEPVARIANT.out.vcf - .join(DEEPVARIANT.out.vcf_index) - .set{deepvariant_raw_ch} - - // DeepVariant emits every evaluated site, not just calls, so most records are - // RefCall. ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without - // this the union is "PASS ClairS-TO + every site DeepVariant evaluated". - // The raw VCF published under variants/deepvariant/ is unaffected. - if (params.deepvariant_filter_pass) { - DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) - DEEPVARIANT_PASS_FILTER.out.vcf - .join(DEEPVARIANT_PASS_FILTER.out.index) - .set{deepvariant_pass_ch} - } - else { - deepvariant_raw_ch - .set{deepvariant_pass_ch} - } + // DeepVariant emits a record for every site it evaluates, not just its calls, so + // most records are RefCall. ClairS-TO needs no equivalent step here because + // VCFSPLIT already restricts it to PASS. The VCF published under + // variants/deepvariant/ is unaffected. + DEEPVARIANT_PASS_FILTER ( + DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + ) - deepvariant_pass_ch + DEEPVARIANT_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] @@ -212,26 +201,15 @@ workflow TUMORONLY_SMALLVAR { [[:],[]], // GZI (empty if FASTA is uncompressed) ds_pon_channel ) - DEEPSOMATIC.out.vcf - .join(DEEPSOMATIC.out.vcf_index) - .set{deepsomatic_raw_ch} - - // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. - // ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without this the - // union is "PASS ClairS-TO + every site DeepSomatic evaluated". - // The raw VCF published under variants/deepsomatic/ is unaffected. - if (params.deepsomatic_filter_pass) { - DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) - DEEPSOMATIC_PASS_FILTER.out.vcf - .join(DEEPSOMATIC_PASS_FILTER.out.index) - .set{deepsomatic_pass_ch} - } - else { - deepsomatic_raw_ch - .set{deepsomatic_pass_ch} - } + // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), + // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT + // already restricts it to PASS. The VCF published under variants/deepsomatic/ is + // unaffected. + DEEPSOMATIC_PASS_FILTER ( + DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + ) - deepsomatic_pass_ch + DEEPSOMATIC_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/vcf_pass_filter.nf b/subworkflows/local/vcf_pass_filter.nf new file mode 100644 index 00000000..51f0833a --- /dev/null +++ b/subworkflows/local/vcf_pass_filter.nf @@ -0,0 +1,40 @@ +// IMPORT MODULES +// Aliased so the software-versions report keeps this separate from the local +// BCFTOOLS_VIEW used by PHASING_HAPLOTYPING, which is a different bcftools build. +include { BCFTOOLS_VIEW as PASS_FILTER } from '../../modules/nf-core/bcftools/view/main' + +// +// SUBWORKFLOW: VCF_PASS_FILTER +// Restrict a per-caller VCF to its PASS records before it is handed to the caller +// consensus, phasing, VEP and the report. DeepVariant and DeepSomatic emit a record for +// every site they evaluate (RefCall/GERMLINE/PON), and Clair3/ClairS keep their LowQual +// and NonSomatic calls, so without this the union is "every site every caller looked at". +// The per-caller VCFs published under variants// are produced elsewhere and are +// never filtered, so no calls are lost from the results directory. +// +// Include once per caller under an alias so conf/modules.config can give each one its own +// prefix, e.g. include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER }. +// +workflow VCF_PASS_FILTER { + + take: + vcfs // [meta, vcf, tbi] + + main: + if (params.smallvar_filter_pass) { + // --write-index is set in conf/modules.config because the module's index output is + // optional, and the join() below would otherwise drop every sample. + PASS_FILTER ( vcfs, [], [], [] ) + + PASS_FILTER.out.vcf + .join(PASS_FILTER.out.index) + .set{ filtered } + } + else { + vcfs + .set{ filtered } + } + + emit: + vcf = filtered // [meta, vcf, tbi] +} diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index e3db716b..1a13f798 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -3,11 +3,11 @@ "content": [ "88c8d3cf9cb49fdbc53372b2275d5f3e" ], - "timestamp": "2026-09-02T10:24:37.511974063", "meta": { "nf-test": "0.9.4", "nextflow": "26.04.3" - } + }, + "timestamp": "2026-09-02T10:24:37.511974063" }, "-profile test, clair only, extended samplesheet": { "content": [ @@ -74,6 +74,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -717,38 +720,38 @@ "sample5/vep/somatic/sample5_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,772e41f7cd86c03a22afbe5ec0592a6b", - "sample1_normal.bam.bai:md5,1b501f6a11efe5d2e6f47b7f1523220b", - "sample1_tumor.bam:md5,c8315c80dc92dfb5d874aef3f5dd46fb", - "sample1_tumor.bam.bai:md5,bc35f807be4b93fc795a14d701469367", + "sample1_normal.bam:md5,9a73c3f90bc4d9a140bcf8f652b0f269", + "sample1_normal.bam.bai:md5,37a866c569f24ed2b38f093f9475b9d6", + "sample1_tumor.bam:md5,dc15bf0e9ff1d401491347c15c408b6d", + "sample1_tumor.bam.bai:md5,fa57db4206692079d9b2084a866bf411", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,eff050a68e36e778b06e0ec19435c569", - "sample1_whatshap_stats.log:md5,76b73731f74fe32ef2d11f6bb0a0f71a", - "sample1_whatshap_stats.tsv:md5,f566ae25b3c5a8f7e94b3d6c1b0417f8", + "sample1_whatshap_stats.gtf:md5,36bda647c08358df0eac0be321c24b20", + "sample1_whatshap_stats.log:md5,938f792fd22bb658a2c7cb1b7653035f", + "sample1_whatshap_stats.tsv:md5,cf8917be389dfef1344eeb6b7e99c7c5", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,3157bd11ba095a884c7951aafcfcfb1c", - "sample2_normal.bam.bai:md5,edebda44c4383173caea728acde4ac43", - "sample2_tumor.bam:md5,47b2c5f86e0493ba94ff72cea77eeae3", - "sample2_tumor.bam.bai:md5,abf2c290c815f54c2b3f8179f717d9bd", + "sample2_normal.bam:md5,c18bbb1bc05b0ec830fa1ac3c6ef542f", + "sample2_normal.bam.bai:md5,de77684ef264476b3e0531b61ca23740", + "sample2_tumor.bam:md5,8b8cb5ac7668b8ac2f4097f584481d35", + "sample2_tumor.bam.bai:md5,3b46456b0e7b00688519dfb25c8e3c88", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,4d8f4393e3aebe4e945c0b8236cf3b3e", - "sample2_whatshap_stats.log:md5,10bba7bae6dd99b989ece5e5dac7a8f9", - "sample2_whatshap_stats.tsv:md5,bb46226e486af9026ab76e014624e903", + "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", + "sample2_whatshap_stats.log:md5,75a69a8e651979e25467d6e3c84cdf90", + "sample2_whatshap_stats.tsv:md5,92d7234e355833b1ec6f54951c38c09d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -797,10 +800,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:54:42.01587726", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T16:39:52.362456747" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index d0a4150e..c7b025c3 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -47,9 +47,6 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, - "DEEPSOMATIC_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -59,9 +56,6 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, - "DEEPVARIANT_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -103,6 +97,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -581,27 +578,27 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,28ad28688852dfe5c2527c88fe52d2c7", - "sample1_normal.bam.bai:md5,3f1a1e308eb5f7b6a3805e76ff7034a3", - "sample1_tumor.bam:md5,e25dec2664b709de329909c5f1be74f2", - "sample1_tumor.bam.bai:md5,04d8b757427e0ff00b19aa338c681a54", + "sample1_normal.bam:md5,35824329c737d89e70234e0b8e2de3fa", + "sample1_normal.bam.bai:md5,2eccbe13acc6273ebf6332ec5a5da815", + "sample1_tumor.bam:md5,969daba408c88235a28c6ed38d513965", + "sample1_tumor.bam.bai:md5,4face49fc270117ab49d72d1413a7efb", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,19483ff8a25ef082cde1d2c7d5faac6f", - "sample1_whatshap_stats.log:md5,ede92982bd2c122b30c5625ff1627c46", - "sample1_whatshap_stats.tsv:md5,4c347cec00fefec30623219a865e3eb3", + "sample1_whatshap_stats.gtf:md5,b85799ad0a512d6c30ff05f8b5dca54a", + "sample1_whatshap_stats.log:md5,5889131d24524deb76e5d585e3a2dd3f", + "sample1_whatshap_stats.tsv:md5,648f4ba86c78fc80d388c95193e3c4f4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", - "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", + "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", @@ -610,9 +607,9 @@ "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", - "sample2_whatshap_stats.log:md5,f9628717e0c4944888687c13098c62eb", - "sample2_whatshap_stats.tsv:md5,4fd295d10a8c461fd2c91851027503a4", + "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", + "sample2_whatshap_stats.log:md5,e7336c45aa5738a67f032898accb305e", + "sample2_whatshap_stats.tsv:md5,e478af5b3adb92a590a485f444a0dabe", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -639,6 +636,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T11:10:43.315178155" + "timestamp": "2026-09-18T17:28:32.306705325" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 4c20adca..8ccb3085 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -23,9 +23,6 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, - "DEEPSOMATIC_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -35,9 +32,6 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, - "DEEPVARIANT_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -79,6 +73,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -584,6 +581,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T11:19:11.135286604" + "timestamp": "2026-09-18T17:36:01.066378821" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 47b61a3f..c0b8e75a 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -64,6 +64,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -521,38 +524,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,772e41f7cd86c03a22afbe5ec0592a6b", - "sample1_normal.bam.bai:md5,1b501f6a11efe5d2e6f47b7f1523220b", - "sample1_tumor.bam:md5,c8315c80dc92dfb5d874aef3f5dd46fb", - "sample1_tumor.bam.bai:md5,bc35f807be4b93fc795a14d701469367", + "sample1_normal.bam:md5,9a73c3f90bc4d9a140bcf8f652b0f269", + "sample1_normal.bam.bai:md5,37a866c569f24ed2b38f093f9475b9d6", + "sample1_tumor.bam:md5,dc15bf0e9ff1d401491347c15c408b6d", + "sample1_tumor.bam.bai:md5,fa57db4206692079d9b2084a866bf411", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,eff050a68e36e778b06e0ec19435c569", - "sample1_whatshap_stats.log:md5,76b73731f74fe32ef2d11f6bb0a0f71a", - "sample1_whatshap_stats.tsv:md5,f566ae25b3c5a8f7e94b3d6c1b0417f8", + "sample1_whatshap_stats.gtf:md5,36bda647c08358df0eac0be321c24b20", + "sample1_whatshap_stats.log:md5,938f792fd22bb658a2c7cb1b7653035f", + "sample1_whatshap_stats.tsv:md5,cf8917be389dfef1344eeb6b7e99c7c5", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,3157bd11ba095a884c7951aafcfcfb1c", - "sample2_normal.bam.bai:md5,edebda44c4383173caea728acde4ac43", - "sample2_tumor.bam:md5,47b2c5f86e0493ba94ff72cea77eeae3", - "sample2_tumor.bam.bai:md5,abf2c290c815f54c2b3f8179f717d9bd", + "sample2_normal.bam:md5,c18bbb1bc05b0ec830fa1ac3c6ef542f", + "sample2_normal.bam.bai:md5,de77684ef264476b3e0531b61ca23740", + "sample2_tumor.bam:md5,8b8cb5ac7668b8ac2f4097f584481d35", + "sample2_tumor.bam.bai:md5,3b46456b0e7b00688519dfb25c8e3c88", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,4d8f4393e3aebe4e945c0b8236cf3b3e", - "sample2_whatshap_stats.log:md5,10bba7bae6dd99b989ece5e5dac7a8f9", - "sample2_whatshap_stats.tsv:md5,bb46226e486af9026ab76e014624e903", + "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", + "sample2_whatshap_stats.log:md5,75a69a8e651979e25467d6e3c84cdf90", + "sample2_whatshap_stats.tsv:md5,92d7234e355833b1ec6f54951c38c09d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -575,10 +578,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:50:22.195327407", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T17:44:15.090049384" } } \ No newline at end of file diff --git a/tests/union.nf.test b/tests/union.nf.test index 2fe8925b..aa14800b 100644 --- a/tests/union.nf.test +++ b/tests/union.nf.test @@ -115,4 +115,46 @@ nextflow_pipeline { ) } } + + test("-profile test, union combine mode, smallvar_filter_pass=false") { + + when { + params { + outdir = "$outputDir" + germline_var_combine = 'all' + somatic_var_combine = 'all' + germline_var_keep = 'clair, deepvariant' + somatic_var_keep = 'clair, deepsomatic' + smallvar_filter_pass = false + } + } + + then { + assertAll( + { assert workflow.success }, + + // ── No PASS filter process may run ─────────────────────────── + // VCF_PASS_FILTER passes its input straight through when the param is + // false, so PASS_FILTER never executes and never reports a version. + { + def versions = file("$outputDir/pipeline_info/lrsomatic_software_mqc_versions.yml") + assert versions.exists() + assert !versions.text.contains('PASS_FILTER') + }, + + // ── Phased VCFs still exist and have data ──────────────────── + { + ['sample1', 'sample2', 'sample3'].each { s -> + def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") + def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") + assert germline.exists() + assert somatic.exists() + assert germline.size() > 0 + assert somatic.size() > 0 + } + } + ) + } + } + } diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 2e110695..0663c675 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -44,9 +44,6 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, - "DEEPSOMATIC_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -56,9 +53,6 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, - "DEEPVARIANT_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -100,6 +94,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -581,27 +578,27 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,cd28bdb7edefa810b86a12029783e1b6", - "sample1_normal.bam.bai:md5,6c19fcce6bd9050fb03a00e66ba1b866", - "sample1_tumor.bam:md5,dcb89119e272efbe0168d1737f714a8a", - "sample1_tumor.bam.bai:md5,9577c39e6f59056421a8192b0ba43acc", + "sample1_normal.bam:md5,271da2f58d613f1f101a12ef06eca766", + "sample1_normal.bam.bai:md5,c51c824e0ac5451f9c10eee78f40cb98", + "sample1_tumor.bam:md5,f5f68131d379181bbc335c9530e9cc7e", + "sample1_tumor.bam.bai:md5,75e62a6480344075038ce6ad2499bfbb", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,43773d064cd353d1dbbae45adb3be334", - "sample1_whatshap_stats.log:md5,13f4530236189960576b5bc45e63bc1a", - "sample1_whatshap_stats.tsv:md5,bf14b5a1fd9d95797f36a48e7e16953d", + "sample1_whatshap_stats.gtf:md5,2b8ff0a9e055c9b4432bc3da9cf98a4d", + "sample1_whatshap_stats.log:md5,70a5270fec94f8b569a4a7bda49751db", + "sample1_whatshap_stats.tsv:md5,1eab3a16f1a51016c3b74f104de152a6", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", - "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", + "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", @@ -610,9 +607,9 @@ "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", - "sample2_whatshap_stats.log:md5,af84d397bf163be8de77fc3c7401493b", - "sample2_whatshap_stats.tsv:md5,fb38bde18fbfc61e52947b68d4260829", + "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", + "sample2_whatshap_stats.log:md5,be94280217c27d05773afa1465152c8a", + "sample2_whatshap_stats.tsv:md5,0c97b6033a149898e88aaf9c632b2de4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -639,6 +636,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T11:35:45.671299133" + "timestamp": "2026-09-18T17:54:05.409593728" } } \ No newline at end of file From 9a1e8e663de353a6b783df9b18c404daf04d0de6 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Mon, 21 Sep 2026 11:56:57 +0200 Subject: [PATCH 03/36] testing --- CHANGELOG.md | 2 + conf/modules.config | 19 ++++- docs/usage.md | 54 ++++++++++---- modules/local/bcftools/view/main.nf | 4 +- modules/local/vcfsplit/main.nf | 28 ++++++-- modules/local/vcftag/environment.yml | 7 ++ modules/local/vcftag/main.nf | 53 ++++++++++++++ modules/local/vcftag/tests/main.nf.test | 68 ++++++++++++++++++ subworkflows/local/phasing_haplotyping.nf | 52 +++++++++----- subworkflows/local/small_variant_consensus.nf | 58 +++++++++------ .../local/tumor_only/tumoronly_smallvar.nf | 4 +- subworkflows/local/vcf_pass_filter.nf | 7 +- tests/fixtures/vcftag_input.vcf | 11 +++ tests/fixtures/vcftag_input.vcf.gz | Bin 0 -> 505 bytes tests/fixtures/vcftag_input.vcf.gz.tbi | Bin 0 -> 112 bytes 15 files changed, 303 insertions(+), 64 deletions(-) create mode 100644 modules/local/vcftag/environment.yml create mode 100644 modules/local/vcftag/main.nf create mode 100644 modules/local/vcftag/tests/main.nf.test create mode 100644 tests/fixtures/vcftag_input.vcf create mode 100644 tests/fixtures/vcftag_input.vcf.gz create mode 100644 tests/fixtures/vcftag_input.vcf.gz.tbi diff --git a/CHANGELOG.md b/CHANGELOG.md index e316511a..7a3a8f3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so every germline record sitting at a somatic coordinate was retained; on B2037620 it removed none of the 13,708,100 germline records, and `variants/phased/somatic_smallvariants.vcf.gz` ended up holding 4,129,270 germline variants that the report could not distinguish from somatic ones. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) now stamps `INFO/SOMATIC` and `INFO/GERMLINE` on the two arms immediately before they are merged for phasing — the only point at which origin is unambiguous for every caller, since `GERMLINE_CONSENSUS` can emit records that never passed through `VCFSPLIT` — and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. Longphase preserves custom INFO keys, verified on v2.0.1. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits; it still normalises the germline split's `FILTER` to `PASS` so that downstream tools which filter on `PASS` continue to see every record, but that value is no longer destroyed. Germline calls remain published in full under `variants/phased/germline_smallvariants.vcf.gz`, `vep/germline/` and `variants//` (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620), and with `'deepsomatic'` it would have dropped every DeepSomatic-private call (26,560 `PASS` records on the same sample). Both private sets are now kept alongside the shared record, matching `nextflow_schema.json` ("keeps all variants from both callers"); `prioritize_caller_*` selects only whose record represents a shared variant. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. Clair3 and ClairS are far less extreme but still kept their `LowQual` and `NonSomatic` records, so with the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`, and signature fitting is unchanged because `SIGNATURES_BCFTOOLS_VIEW` already applied its own `PASS` filter. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP and the report see far fewer variants. Set `--smallvar_filter_pass false` for the previous behaviour. The per-caller VCFs published under `variants/deepvariant`, `variants/deepsomatic`, `variants/clair3` and `variants/clairs` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). - [#186](https://github.com/IntGenomicsLab/lrsomatic/pull/186) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.0, which adds tickbox dropdown filters on the categorical columns of both variant tables and turns the report's gene panel selector into checkboxes (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 078a1db1..b41bfdb0 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -401,6 +401,21 @@ process { enabled: false ] } + // VCFTAG stamps an INFO provenance flag on each arm before germline and somatic are merged + // for somatic phasing, so PHASING_HAPLOTYPING:BCFTOOLS_VIEW can select the somatic arm by + // origin instead of by position. Intermediates only; the published VCFs come from Longphase. + withName: '.*:TAG_SOMATIC' { + ext.prefix = { "${meta.id}_somatic_tagged" } + publishDir = [ + enabled: false + ] + } + withName: '.*:TAG_GERMLINE' { + ext.prefix = { "${meta.id}_germline_tagged" } + publishDir = [ + enabled: false + ] + } withName: '.*:PHASING_HAPLOTYPING:BCFTOOLS_VIEW' { ext.prefix = { "somatic_smallvariants" } publishDir = [ @@ -655,7 +670,9 @@ process { // keep their LowQual and NonSomatic records; these filtered copies are what the // consensus, phasing, VEP and report steps consume. The per-caller VCFs under // variants// are published from the caller processes and are left untouched. - // ClairS-TO has no entry here: VCFSPLIT already restricts it to PASS. + // ClairS-TO has no entry here: VCFSPLIT already restricts its SOMATIC split to PASS. + // Note this is not true of the germline split, whose FILTER is rewritten to PASS rather + // than filtered; germline/somatic origin is carried in INFO by VCFTAG instead. // --write-index is required because the module's index output is optional, and the // downstream join() would otherwise drop every sample. withName: '.*:DEEPVARIANT_PASS_FILTER:PASS_FILTER' { diff --git a/docs/usage.md b/docs/usage.md index 488fd69f..3ddb2a4d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -353,15 +353,15 @@ The tools run from a purpose-built image (`ghcr.io/ljwharbers/sigprofiler`) beca These options control how variants from multiple callers are filtered and merged. -| Parameter | Description | -| ------------------------------ | --------------------------------------------------------------------------------------------------- | -| `--germline_var_keep` | Expression or threshold for retaining germline variants after calling. Default = `null` | -| `--somatic_var_keep` | Expression or threshold for retaining somatic variants after calling. Default = `null` | -| `--germline_var_combine` | Strategy for combining germline variant caller outputs (e.g. union, intersection). Default = `null` | -| `--somatic_var_combine` | Strategy for combining somatic variant caller outputs (e.g. union, intersection). Default = `null` | -| `--prioritize_caller_germline` | Comma-separated caller priority order used when combining germline calls. Default = `null` | -| `--prioritize_caller_somatic` | Comma-separated caller priority order used when combining somatic calls. Default = `null` | -| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | +| Parameter | Description | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------- | +| `--germline_var_keep` | Expression or threshold for retaining germline variants after calling. Default = `null` | +| `--somatic_var_keep` | Expression or threshold for retaining somatic variants after calling. Default = `null` | +| `--germline_var_combine` | How to combine germline caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | +| `--somatic_var_combine` | How to combine somatic caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | +| `--prioritize_caller_germline` | Whose record to use for variants called by both germline callers: `deepvariant` or `clair`. Default = `clair` | +| `--prioritize_caller_somatic` | Whose record to use for variants called by both somatic callers: `deepsomatic` or `clair`. Default = `clair` | +| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds @@ -374,9 +374,39 @@ by three orders of magnitude and produces a meaningless mutation burden. `--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's VCF that is handed to the caller consensus, phasing, VEP and the report. Set it to `false` to restore the previous unfiltered behaviour. In tumor-only mode ClairS-TO -is unaffected by the setting: `VCFSPLIT` already restricts it to `PASS`. The -per-caller VCFs published under `//variants/` are never -filtered, so no calls are lost from the results directory. +is unaffected by the setting: `VCFSPLIT` already restricts its somatic split to +`PASS`. The per-caller VCFs published under `//variants/` +are never filtered, so no calls are lost from the results directory. + +`consensus` keeps only variants called by both callers; `all` keeps the union, i.e. +every variant called by either. In both modes `--prioritize_caller_*` chooses only +whose record represents a variant that both callers found -- it never decides which +variants are kept. + +#### Germline and somatic provenance + +Germline and somatic small variants are merged into one VCF for somatic phasing, +because Longphase needs all variant sites in a single file to produce consistent +phase blocks. The somatic arm is then recovered from the phased result. + +That recovery selects on an `INFO/SOMATIC` flag stamped on each arm before the merge, +not on position. A positional restriction cannot separate the two populations: a +germline record at the same coordinate as a somatic call is indistinguishable from +it, and `FILTER` is no help either, since `VCFSPLIT` normalises the ClairS-TO +germline split to `PASS` so that downstream tools which filter on `PASS` still see +every record. + +Three INFO fields carry this provenance: + +| Field | Meaning | +| ----- | ------- | +| `SOMATIC` | Record came from the somatic call set | +| `GERMLINE` | Record came from the germline call set | +| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS` | + +Germline calls dropped from `variants/phased/somatic_smallvariants.vcf.gz` are not +lost: they remain in `variants/phased/germline_smallvariants.vcf.gz`, in +`vep/germline/`, and in the unfiltered per-caller VCFs under `variants//`. #### PON Options diff --git a/modules/local/bcftools/view/main.nf b/modules/local/bcftools/view/main.nf index 652da9ac..ae81084e 100644 --- a/modules/local/bcftools/view/main.nf +++ b/modules/local/bcftools/view/main.nf @@ -8,7 +8,7 @@ process BCFTOOLS_VIEW { : 'community.wave.seqera.io/library/bcftools_htslib:0a3fa2654b52006f'}" input: - tuple val(meta), path(vcf), path(tbi), path(targets), path(targets_tbi) + tuple val(meta), path(vcf), path(tbi) output: tuple val(meta), path("*.vcf.gz"), emit: vcf @@ -23,7 +23,7 @@ process BCFTOOLS_VIEW { def prefix = task.ext.prefix ?: "${meta.id}" """ bcftools view \\ - -T ${targets} \\ + -i 'INFO/SOMATIC=1' \\ -Oz \\ -W=tbi \\ ${args} \\ diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index f6156d34..b075dcca 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -28,7 +28,16 @@ process VCFSPLIT { bcftools view -i 'FILTER="PASS"' $snv_vcf | bgzip -c > snv_pass.vcf.gz tabix -p vcf indels_pass.vcf.gz tabix -p vcf snv_pass.vcf.gz - bcftools concat -a -Oz -o somatic.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz + bcftools concat -a -Oz -o somatic_tmp.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz + tabix -p vcf somatic_tmp.vcf.gz + + # Record the caller's original FILTER in INFO. These records are already PASS, but stamping + # both splits keeps them symmetric and self-describing alongside the germline arm below. + bcftools view somatic_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + /^##/ { print; next } + /^#CHROM/ { print "##INFO="; print; next } + { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; print } + ' | bgzip -c > somatic.vcf.gz tabix -p vcf somatic.vcf.gz bcftools view -i 'FILTER~"NonSomatic" || INFO/Verdict_Germline=1' $indel_vcf | bgzip -c > indels_filtered.vcf.gz @@ -38,13 +47,22 @@ process VCFSPLIT { bcftools concat -a -Oz -o germline_tmp.vcf.gz indels_filtered.vcf.gz snv_filtered.vcf.gz tabix -p vcf germline_tmp.vcf.gz - bcftools view germline_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} /^#/ {print} !/^#/ { \$7="PASS"; print }' | \ - bgzip -c > germline.vcf.gz + # FILTER is normalised to PASS so that downstream tools which filter on PASS -- implicitly or + # otherwise -- see every germline record. The caller's original FILTER (typically NonSomatic) + # would otherwise be destroyed here, which is what previously made germline records + # indistinguishable from somatic ones once the two sets were merged for phasing; it is kept in + # INFO/ORIG_FILTER instead. Germline/somatic provenance itself is stamped later, by + # PHASING_HAPLOTYPING:TAG_GERMLINE / TAG_SOMATIC, which covers callers that bypass VCFSPLIT. + bcftools view germline_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + /^##/ { print; next } + /^#CHROM/ { print "##INFO="; print; next } + { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; \$7 = "PASS"; print } + ' | bgzip -c > germline.vcf.gz tabix -p vcf germline.vcf.gz # Cleanup intermediate files - rm indels_pass.vcf.gz snv_pass.vcf.gz - rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi + rm indels_pass.vcf.gz snv_pass.vcf.gz somatic_tmp.vcf.gz + rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi somatic_tmp.vcf.gz.tbi """ stub: diff --git a/modules/local/vcftag/environment.yml b/modules/local/vcftag/environment.yml new file mode 100644 index 00000000..b276efd9 --- /dev/null +++ b/modules/local/vcftag/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - bioconda::bcftools=1.20 diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf new file mode 100644 index 00000000..851dff6b --- /dev/null +++ b/modules/local/vcftag/main.nf @@ -0,0 +1,53 @@ +process VCFTAG { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bcftools:1.20--h8b25389_0': + 'biocontainers/bcftools:1.20--h8b25389_0' }" + + input: + tuple val(meta), path(vcf), path(tbi) + val flag + + output: + tuple val(meta), path("${prefix}.vcf.gz") , emit: vcf + tuple val(meta), path("${prefix}.vcf.gz.tbi") , emit: tbi + + tuple val("${task.process}"), val('bcftools'), eval("bcftools --version |& sed '1!d ; s/bcftools //'"), topic: versions, emit: versions_bcftools + + when: + task.ext.when == null || task.ext.when + + script: + prefix = task.ext.prefix ?: "${meta.id}_${flag.toLowerCase()}" + """ + # Stamp a constant INFO flag recording which arm this record came from, and normalise FILTER + # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and + # filter on FILTER, so a FILTER-based label does not survive to where it is needed. + # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. + bcftools view ${vcf} | awk -v flag="${flag}" 'BEGIN{FS=OFS="\\t"} + /^##/ { print; next } + /^#CHROM/ { + print "##INFO=" + print + next + } + { + \$7 = "PASS" + \$8 = (\$8 == "." || \$8 == "") ? flag : \$8 ";" flag + print + } + ' | bgzip -c > ${prefix}.vcf.gz + + tabix -p vcf ${prefix}.vcf.gz + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}_${flag.toLowerCase()}" + """ + echo "" | gzip > ${prefix}.vcf.gz + touch ${prefix}.vcf.gz.tbi + """ +} diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test new file mode 100644 index 00000000..18b79c41 --- /dev/null +++ b/modules/local/vcftag/tests/main.nf.test @@ -0,0 +1,68 @@ +nextflow_process { + + name "Test Process VCFTAG" + script "../main.nf" + process "VCFTAG" + + tag "modules" + tag "modules_local" + tag "vcftag" + + // Runs for real (no -stub): the tagging is an awk program embedded in the Nextflow script + // block, so the escaping only holds if it is actually executed. + test("stamps the flag, declares its header and normalises FILTER") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file("\${projectDir}/tests/fixtures/vcftag_input.vcf.gz", checkIfExists: true), + file("\${projectDir}/tests/fixtures/vcftag_input.vcf.gz.tbi", checkIfExists: true) + ] + input[1] = 'SOMATIC' + """ + } + } + + then { + assert process.success + + def lines = path(process.out.vcf[0][1]).vcf.variantsAsStrings + def header = path(process.out.vcf[0][1]).vcf.header.getMetaDataInSortedOrder().join('\n') + + assertAll( + // every input record survives -- tagging must not filter + { assert lines.size() == 4 }, + // the flag is declared, so bcftools can query it downstream + { assert header.contains('ID=SOMATIC') }, + // every record carries the flag, including the one whose INFO was '.' + { assert lines.every { it.contains('SOMATIC') } }, + // FILTER is normalised to PASS on all records, including NonSomatic and RefCall + { assert lines.every { it.split('\t')[6] == 'PASS' } }, + // a pre-existing INFO field is preserved rather than overwritten + { assert lines.any { it.contains('EXISTING') && it.contains('CALLER=clairs-to') } }, + // the record whose INFO was '.' gets the bare flag, not '.;SOMATIC' + { assert lines.any { it.split('\t')[7] == 'SOMATIC' } } + ) + } + } + + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [ id:'test' ], [], [] ] + input[1] = 'GERMLINE' + """ + } + } + + then { + assert process.success + } + } +} diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 84bde7d0..f68deb57 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -8,6 +8,8 @@ include { SAMTOOLS_INDEX } from '../../module include { BCFTOOLS_CONCAT } from '../../modules/nf-core/bcftools/concat/main' include { BCFTOOLS_SORT } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_VIEW } from '../../modules/local/bcftools/view/main.nf' +include { VCFTAG as TAG_SOMATIC } from '../../modules/local/vcftag/main.nf' +include { VCFTAG as TAG_GERMLINE } from '../../modules/local/vcftag/main.nf' workflow PHASING_HAPLOTYPING { @@ -139,10 +141,30 @@ workflow PHASING_HAPLOTYPING { } + // + // MODULE: VCFTAG (label: process_single), aliased TAG_SOMATIC / TAG_GERMLINE + // Stamp each arm with an INFO provenance flag before they are merged. This is the only + // point where germline-vs-somatic origin is unambiguous for every caller: GERMLINE_CONSENSUS + // can emit records that never passed through VCFSPLIT, so tagging further upstream would + // leave holes. After the merge the two populations are otherwise indistinguishable -- both + // carry FILTER=PASS, and VCFSPLIT rewrites the ClairS-TO germline FILTER to PASS as well. + // LongPhase preserves custom INFO keys, so the flags survive phasing (verified on v2.0.1). + // + TAG_SOMATIC ( somatic_vcf, 'SOMATIC' ) + TAG_GERMLINE( germline_vcf, 'GERMLINE' ) + + TAG_SOMATIC.out.vcf + .join(TAG_SOMATIC.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .set{ tagged_somatic_vcf } + TAG_GERMLINE.out.vcf + .join(TAG_GERMLINE.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .set{ tagged_germline_vcf } + // tagged_*_vcf: [meta, vcf, tbi] + // Merge germline and somatic VCFs into a single file for somatic phasing // Longphase requires all variant sites in one VCF to produce a consistent phase block - germline_vcf - .join(somatic_vcf) + tagged_germline_vcf + .join(tagged_somatic_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> def vcfs = [som_vcf, germ_vcf] // somatic first (higher priority in phasing) def tbis = [som_tbi, germ_tbi] @@ -178,7 +200,7 @@ workflow PHASING_HAPLOTYPING { if (!params.skip_modcall) { // With modcall: include base-modification VCF as additional phasing evidence normal_bams_w_tumoronly_ch - .join(germline_vcf) + .join(tagged_germline_vcf) .join(LONGPHASE_MODCALL_GERMLINE.out.mod_vcf) .map { meta, bam, bai, vcf, _tbi, mods-> def svs = [] // SVs for phasing are not used here @@ -200,7 +222,7 @@ workflow PHASING_HAPLOTYPING { else { // Without modcall: empty lists for SVs and mods normal_bams_w_tumoronly_ch - .join(germline_vcf) + .join(tagged_germline_vcf) .map { meta, bam, bai, vcf, _tbi -> def svs = [] def mods = [] @@ -258,23 +280,17 @@ workflow PHASING_HAPLOTYPING { // // MODULE: BCFTOOLS_VIEW (label: process_medium) - // Filter the phased somatic+germline VCF to somatic-only positions. - // Uses the original somatic VCF as a targets (-T) file so only positions - // called as somatic are retained. Phase tags (PS/HP) on somatic variants - // are preserved; germline records are dropped. - // Input: [meta, phased_combined_vcf, phased_combined_tbi, somatic_vcf, somatic_tbi] + // Reduce the phased somatic+germline VCF to the somatic arm, selecting on the INFO/SOMATIC + // flag stamped before the merge. This selects by provenance rather than by position: the + // previous `-T ` targets file matched CHROM/POS only, so every germline record + // co-located with a somatic call was retained and became indistinguishable from a somatic + // one downstream. Phase tags (PS/HP) on somatic variants are preserved; germline records + // are dropped here but remain published in full under variants/phased/ and vep/germline/. + // Input: [meta, phased_combined_vcf, phased_combined_tbi] // Output: .vcf -- [meta, vcf.gz] -- phased somatic-only VCF // .tbi -- [meta, tbi] // - phased_somatic_germline_vcf - .join(somatic_vcf) - .map { meta, phased_vcf, phased_tbi, som_vcf, som_tbi -> - return [ meta, phased_vcf, phased_tbi, som_vcf, som_tbi ] - } - .set { bcftools_view_input_ch } - // bcftools_view_input_ch: [meta, phased_combined_vcf, tbi, somatic_vcf, somatic_tbi] - - BCFTOOLS_VIEW ( bcftools_view_input_ch ) + BCFTOOLS_VIEW ( phased_somatic_germline_vcf ) BCFTOOLS_VIEW.out.vcf .join(BCFTOOLS_VIEW.out.tbi) diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index d3499cf8..d6e7e5b4 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -17,7 +17,7 @@ workflow SMALL_VARIANT_CONSENSUS { fasta // [[:], fasta] _fai // [[:], fai] prioritize_caller // str: which caller's calls take priority ('deepvariant'/'deepsomatic' or 'clair') - combine_method // str: 'consensus' (intersection only) or 'all' (intersection + private calls from priority caller) + combine_method // str: 'consensus' (shared calls only) or 'all' (union of both callers' calls) main: @@ -205,6 +205,9 @@ workflow SMALL_VARIANT_CONSENSUS { BCFTOOLS_ISEC.out.clair_consensus_vcf .set{isec_consensus_vcf} } + else { + error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") + } // BCFTOOLS_ISEC outputs hardcoded names (0002.vcf.gz) inside a prefix directory. // Nextflow stages files using basename only, so both germline and somatic consensus // VCFs would collide as "0002.vcf.gz" in downstream PHASING_HAPLOTYPING:BCFTOOLS_CONCAT. @@ -216,45 +219,56 @@ workflow SMALL_VARIANT_CONSENSUS { } else if (combine_method == 'all') { - // Take the intersection PLUS the private calls from the prioritized caller - // (private calls from the non-priority caller are discarded) + // Union: every variant called by either caller. Variants called by both contribute a + // single record, taken from the prioritized caller; the private calls of BOTH callers + // are kept. This matches `*_var_combine = 'all'` in nextflow_schema.json ("keeps all + // variants from both callers"); `prioritize_caller` only selects whose record is used + // for the shared variants, never which calls are kept. + // The three isec sets are disjoint by construction, so BCFTOOLS_CONCAT needs no -d. if (prioritize_caller in ['deepvariant', 'deepsomatic']) { - // consensus (DeepVariant record) + DeepVariant-private variants + // shared (DeepVariant record) + DeepVariant-private + Clair-private BCFTOOLS_ISEC.out.deepvar_consensus_vcf .join(BCFTOOLS_ISEC.out.deepvar_consensus_tbi) + .join(BCFTOOLS_ISEC.out.deepvar_private_vcf) + .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) .join(BCFTOOLS_ISEC.out.clair_private_vcf) .join(BCFTOOLS_ISEC.out.clair_private_tbi) - .map{ meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> - return[meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi]] + .map{ meta, shared_vcf, shared_tbi, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> + return[meta, [shared_vcf, deepvar_vcf, clair_vcf], [shared_tbi, deepvar_tbi, clair_tbi]] } .set{concat_input} - // concat_input: [meta, [consensus_vcf, private_vcf], [consensus_tbi, private_tbi]] - BCFTOOLS_CONCAT(concat_input) - BCFTOOLS_CONCAT.out.vcf - .set{concat_out} } else if (prioritize_caller == 'clair') { - // consensus (Clair record) + Clair-private variants - BCFTOOLS_ISEC.out.deepvar_private_vcf - .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) - .join(BCFTOOLS_ISEC.out.clair_consensus_vcf) + // shared (Clair record) + DeepVariant-private + Clair-private + BCFTOOLS_ISEC.out.clair_consensus_vcf .join(BCFTOOLS_ISEC.out.clair_consensus_tbi) - .map{ meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> - return[meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi]] + .join(BCFTOOLS_ISEC.out.deepvar_private_vcf) + .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) + .join(BCFTOOLS_ISEC.out.clair_private_vcf) + .join(BCFTOOLS_ISEC.out.clair_private_tbi) + .map{ meta, shared_vcf, shared_tbi, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> + return[meta, [shared_vcf, deepvar_vcf, clair_vcf], [shared_tbi, deepvar_tbi, clair_tbi]] } .set{concat_input} - // concat_input: [meta, [private_vcf, consensus_vcf], [private_tbi, consensus_tbi]] - BCFTOOLS_CONCAT(concat_input) - BCFTOOLS_CONCAT.out.vcf - .set{concat_out} } - // concat_out: [meta, vcf] -- unsorted concatenated VCF (consensus + priority-caller-private) + else { + error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") + } + // concat_input: [meta, [shared_vcf, deepvar_private_vcf, clair_private_vcf], [tbis...]] + BCFTOOLS_CONCAT(concat_input) + BCFTOOLS_CONCAT.out.vcf + .set{concat_out} + // concat_out: [meta, vcf] -- unsorted union of both callers' calls BCFTOOLS_SORT(concat_out) BCFTOOLS_SORT.out.vcf .set{vcf} BCFTOOLS_SORT.out.tbi .set{tbi} - // vcf/tbi: [meta, vcf/tbi] -- sorted combined VCF + // vcf/tbi: [meta, vcf/tbi] -- sorted union VCF + } + + else { + error("combine_method must be 'consensus' or 'all', got '${combine_method}'") } emit: diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 299bc0c7..050375ed 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -128,7 +128,9 @@ workflow TUMORONLY_SMALLVAR { // DeepVariant emits a record for every site it evaluates, not just its calls, so // most records are RefCall. ClairS-TO needs no equivalent step here because - // VCFSPLIT already restricts it to PASS. The VCF published under + // VCFSPLIT already restricts its SOMATIC split to PASS -- note that its GERMLINE split + // is not PASS-filtered but PASS-rewritten, so a PASS filter would not reduce it and + // germline/somatic origin is carried in INFO by VCFTAG instead. The VCF published under // variants/deepvariant/ is unaffected. DEEPVARIANT_PASS_FILTER ( DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) diff --git a/subworkflows/local/vcf_pass_filter.nf b/subworkflows/local/vcf_pass_filter.nf index 51f0833a..f52e0cb9 100644 --- a/subworkflows/local/vcf_pass_filter.nf +++ b/subworkflows/local/vcf_pass_filter.nf @@ -22,12 +22,13 @@ workflow VCF_PASS_FILTER { main: if (params.smallvar_filter_pass) { - // --write-index is set in conf/modules.config because the module's index output is - // optional, and the join() below would otherwise drop every sample. + // The module declares `emit: index, optional: true`, so the index only exists because + // conf/modules.config puts --write-index=tbi in ext.args for every alias. failOnMismatch + // turns a missing index into an immediate error instead of silently dropping the sample. PASS_FILTER ( vcfs, [], [], [] ) PASS_FILTER.out.vcf - .join(PASS_FILTER.out.index) + .join(PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) .set{ filtered } } else { diff --git a/tests/fixtures/vcftag_input.vcf b/tests/fixtures/vcftag_input.vcf new file mode 100644 index 00000000..34fda4cc --- /dev/null +++ b/tests/fixtures/vcftag_input.vcf @@ -0,0 +1,11 @@ +##fileformat=VCFv4.2 +##INFO= +##INFO= +##FILTER= +##FILTER= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT testsample +chr1 100 . A C 30 PASS CALLER=clairs-to GT:DP 0/1:20 +chr1 200 . G T 20 NonSomatic CALLER=clairs-to;EXISTING GT:DP 0/1:18 +chr1 300 . T A 10 RefCall . GT:DP 0/0:12 +chr1 400 . C G 40 . CALLER=deepsomatic GT:DP 1/1:25 diff --git a/tests/fixtures/vcftag_input.vcf.gz b/tests/fixtures/vcftag_input.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..988819c22c88bc153d91397e8db80263c9cbe483 GIT binary patch literal 505 zcmb2|=3rp}f&Xj_PR>jWCXBHc0<)V91lT`3{}SBtE9{_W^?fZ6S z+fkQE(UJTAUdY?k-QWv4HmA{{%kZ*k5#{^G^y?=~%1eqZH8YXkG$`I+ZUHL{{s zwcAhK9qOH-yzYwWPR861clbT6iWyu#YJ6QT-L}|IDd6RbRxzCyY|j?ol;{oWNh{vM z9p5Nck#g|U$Cv*SC0SN?itn9rblM&<{|5TzIE<&hk1xcXQ_i2S;UPlT%R{p`otT--6BcUf0b|oU_Md!IY&7=E{q;aZGA# zR=D!{^11#z>A&-lz`W};9Sw1slNjynB(jR?) h|F!eEB^Ifge7nt;pScx%4H&2B?v!R=2FEps008J@+w1@U literal 0 HcmV?d00001 diff --git a/tests/fixtures/vcftag_input.vcf.gz.tbi b/tests/fixtures/vcftag_input.vcf.gz.tbi new file mode 100644 index 0000000000000000000000000000000000000000..3e8c3efd859874550e714bc5ac9aa7e101d7798f GIT binary patch literal 112 zcmb2|=3rp}f&Xj_PR>jW!3^AmpHfm%5)u-U5)v9N@&Li9fLT);8x<`VpAgV_sL2v| t Date: Mon, 21 Sep 2026 13:14:12 +0200 Subject: [PATCH 04/36] fix bugs --- CHANGELOG.md | 2 +- docs/usage.md | 10 +++++----- modules/local/vcfsplit/main.nf | 26 ++++++++++++++++++++------ modules/local/vcftag/main.nf | 4 ++-- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3a8f3e..850dab8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so every germline record sitting at a somatic coordinate was retained; on B2037620 it removed none of the 13,708,100 germline records, and `variants/phased/somatic_smallvariants.vcf.gz` ended up holding 4,129,270 germline variants that the report could not distinguish from somatic ones. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) now stamps `INFO/SOMATIC` and `INFO/GERMLINE` on the two arms immediately before they are merged for phasing — the only point at which origin is unambiguous for every caller, since `GERMLINE_CONSENSUS` can emit records that never passed through `VCFSPLIT` — and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. Longphase preserves custom INFO keys, verified on v2.0.1. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits; it still normalises the germline split's `FILTER` to `PASS` so that downstream tools which filter on `PASS` continue to see every record, but that value is no longer destroyed. Germline calls remain published in full under `variants/phased/germline_smallvariants.vcf.gz`, `vep/germline/` and `variants//` (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so every germline record sitting at a somatic coordinate was retained; on B2037620 it removed none of the 13,708,100 germline records, and `variants/phased/somatic_smallvariants.vcf.gz` ended up holding 4,129,270 germline variants that the report could not distinguish from somatic ones. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) now stamps `INFO/SOMATIC` and `INFO/GERMLINE` on the two arms immediately before they are merged for phasing — the only point at which origin is unambiguous for every caller, since `GERMLINE_CONSENSUS` can emit records that never passed through `VCFSPLIT` — and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. Longphase preserves custom INFO keys, verified on v2.0.1. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits (multi-valued `FILTER` is joined with `,`, since `;` separates INFO fields); it still normalises the germline split's `FILTER` to `PASS` so that downstream tools which filter on `PASS` continue to see every record, but that value is no longer destroyed. Germline calls remain published in full under `variants/phased/germline_smallvariants.vcf.gz`, `vep/germline/` and `variants//` (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620), and with `'deepsomatic'` it would have dropped every DeepSomatic-private call (26,560 `PASS` records on the same sample). Both private sets are now kept alongside the shared record, matching `nextflow_schema.json` ("keeps all variants from both callers"); `prioritize_caller_*` selects only whose record represents a shared variant. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. Clair3 and ClairS are far less extreme but still kept their `LowQual` and `NonSomatic` records, so with the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`, and signature fitting is unchanged because `SIGNATURES_BCFTOOLS_VIEW` already applied its own `PASS` filter. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP and the report see far fewer variants. Set `--smallvar_filter_pass false` for the previous behaviour. The per-caller VCFs published under `variants/deepvariant`, `variants/deepsomatic`, `variants/clair3` and `variants/clairs` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). diff --git a/docs/usage.md b/docs/usage.md index 3ddb2a4d..87e7cf53 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -398,11 +398,11 @@ every record. Three INFO fields carry this provenance: -| Field | Meaning | -| ----- | ------- | -| `SOMATIC` | Record came from the somatic call set | -| `GERMLINE` | Record came from the germline call set | -| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS` | +| Field | Meaning | +| ------------- | ---------------------------------------------------------------------------------------------------------------- | +| `SOMATIC` | Record came from the somatic call set | +| `GERMLINE` | Record came from the germline call set | +| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS`. Multiple filters are joined with `,` | Germline calls dropped from `variants/phased/somatic_smallvariants.vcf.gz` are not lost: they remain in `variants/phased/germline_smallvariants.vcf.gz`, in diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index b075dcca..d773494a 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -33,10 +33,15 @@ process VCFSPLIT { # Record the caller's original FILTER in INFO. These records are already PASS, but stamping # both splits keeps them symmetric and self-describing alongside the germline arm below. - bcftools view somatic_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + # The header line's double quotes arrive via -v q. Escaped quotes inside a Nextflow script + # block are fragile, and losing the escape silently produces an unparseable VCF header. + # FILTER is ";"-delimited but ";" separates INFO fields, so it is stored as ",". + bcftools view somatic_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } - { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; print } + /^#CHROM/ { print "##INFO="; print; next } + { of = \$7; gsub(/;/, ",", of) + \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of + print } ' | bgzip -c > somatic.vcf.gz tabix -p vcf somatic.vcf.gz @@ -53,13 +58,22 @@ process VCFSPLIT { # indistinguishable from somatic ones once the two sets were merged for phasing; it is kept in # INFO/ORIG_FILTER instead. Germline/somatic provenance itself is stamped later, by # PHASING_HAPLOTYPING:TAG_GERMLINE / TAG_SOMATIC, which covers callers that bypass VCFSPLIT. - bcftools view germline_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + bcftools view germline_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } - { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; \$7 = "PASS"; print } + /^#CHROM/ { print "##INFO="; print; next } + { of = \$7; gsub(/;/, ",", of) + \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of + \$7 = "PASS" + print } ' | bgzip -c > germline.vcf.gz tabix -p vcf germline.vcf.gz + # Read both headers back. tabix will happily index a VCF whose header htslib cannot parse, so + # without this a malformed header surfaces as a confusing failure in a later process instead + # of here. set -e is in effect, so a bad header fails this task. + bcftools view -h somatic.vcf.gz > /dev/null + bcftools view -h germline.vcf.gz > /dev/null + # Cleanup intermediate files rm indels_pass.vcf.gz snv_pass.vcf.gz somatic_tmp.vcf.gz rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi somatic_tmp.vcf.gz.tbi diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf index 851dff6b..3fd9dc81 100644 --- a/modules/local/vcftag/main.nf +++ b/modules/local/vcftag/main.nf @@ -27,10 +27,10 @@ process VCFTAG { # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and # filter on FILTER, so a FILTER-based label does not survive to where it is needed. # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. - bcftools view ${vcf} | awk -v flag="${flag}" 'BEGIN{FS=OFS="\\t"} + bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } /^#CHROM/ { - print "##INFO=" + print "##INFO=" print next } From 5b38dcff17ab195ce593cab80a3e4b958c08ee1e Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 10:30:03 +0200 Subject: [PATCH 05/36] bump wakhan (bug) --- CHANGELOG.md | 1 + modules/local/wakhan/environment.yml | 2 +- modules/local/wakhan/main.nf | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 850dab8c..48e9a388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3, `src/coverage/processing.py:121` averaged a BAF bin with `haplotype_1_coverage[haplotype_1_position.index(min(l2)):haplotype_1_position.index(max(l2))]`; because `.index()` searches the full position list by value, the slice is empty whenever that list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError: mean requires at least one data point` (it also always dropped the bin's last SNP). Upstream replaced the block in 0.4.4 with a guarded loop that emits `CENTROMERE_SENTINEL` for bins holding fewer than `MIN_SNPS_PER_BAF_BIN` het SNPs. The failure was deterministic, so the retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). - [#193](https://github.com/IntGenomicsLab/lrsomatic/pull/193) - Fixed the documented way of enabling EVE. `--vep_eve https://evemodel.org/api/proteins/bulk/download/` was rejected at launch, because the check for "still needs reshaping" keyed on a `.zip` suffix and that endpoint carries no file extension, so the URL was taken for a finished file and an index demanded for it. The check now asks whether the value already is a prepared (bgzipped) file instead (@AmberVerhasselt). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs a patched modkit 0.6.4 built from [ljwharbers/modkit@pacbio-conflict-fix](https://github.com/ljwharbers/modkit/tree/pacbio-conflict-fix): `ghcr.io/ljwharbers/modkit:0.6.4-pacbiofix-6e0afa2` under Docker, and the native SIF `oras://ghcr.io/ljwharbers/modkit-sif:0.6.4-pacbiofix-6e0afa2` under Singularity/Apptainer (unless `singularity_pull_docker_container` is set, which falls back to the Docker image). It keeps reads whose 5mC and 5hmC probabilities sum above 1, fixes pileup on PacBio-style MM tags, and honours `--phased` and `--modified-bases` in the general pileup workers that modkit uses for PacBio BAMs with 6mA calls (stock modkit wrote empty `_hp1`/`_hp2` files and an `h` row next to every `m` row for those). Stock modkit 0.4.3-0.6.4 silently dropped 32-65 % of reads from recent PacBio HiFi BAMs (Jasmine >= 26.1.3) and returned empty `--cpg` pileups ([nanoporetech/modkit#612](https://github.com/nanoporetech/modkit/issues/612); fixes proposed upstream in [nanoporetech/modkit#720](https://github.com/nanoporetech/modkit/pull/720)). The image is `linux/amd64` only and Conda is not supported for this step: `MODKIT_PILEUP` stops with an error under `-profile conda`/`mamba`, use `--skip_modkit` there. The module should return to the nf-core biocontainer once a modkit release includes the fix (@ljwharbers). - [#181](https://github.com/IntGenomicsLab/lrsomatic/pull/181) - NanoPlot pre- and post-alignment statistics now reach MultiQC. `ch_nanoplot_pre_txt` and `ch_nanoplot_post_txt` were declared empty and mixed into the MultiQC inputs, but never assigned from `NANOPLOT_PRE.out.txt` / `NANOPLOT_POST.out.txt`, so the NanoStat section was silently missing from every report (@ljwharbers). diff --git a/modules/local/wakhan/environment.yml b/modules/local/wakhan/environment.yml index 6b0cdb43..33c3c873 100644 --- a/modules/local/wakhan/environment.yml +++ b/modules/local/wakhan/environment.yml @@ -4,4 +4,4 @@ channels: - conda-forge - bioconda dependencies: - - "bioconda::wakhan=0.4.3" + - "bioconda::wakhan=0.4.4" diff --git a/modules/local/wakhan/main.nf b/modules/local/wakhan/main.nf index ad7aba5e..b461baaf 100644 --- a/modules/local/wakhan/main.nf +++ b/modules/local/wakhan/main.nf @@ -4,8 +4,8 @@ process WAKHAN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/wakhan:0.4.3--pyhdfd78af_0': - 'biocontainers/wakhan:0.4.3--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/wakhan:0.4.4--pyhdfd78af_0': + 'biocontainers/wakhan:0.4.4--pyhdfd78af_0' }" input: tuple val(meta), path(tumor_input), path(tumor_index), path(normal_input), path(normal_index), path(vcf), path(breakpoints) @@ -40,7 +40,7 @@ process WAKHAN { // and LRSOMATICREPORT resolves them by solution_/ path tuple val(meta), path("solution_*", type: 'dir') , emit: solution_dirs, optional: true // WARN: Manually update version information as tool does not provide on CLI - tuple val("${task.process}"), val('wakhan'), val("0.4.3"), topic: versions, emit: versions_wakhan + tuple val("${task.process}"), val('wakhan'), val("0.4.4"), topic: versions, emit: versions_wakhan when: task.ext.when == null || task.ext.when From da99d0bc8b9aa4d135afd155f4387f50b2d7a278 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:38:56 +0200 Subject: [PATCH 06/36] Split multi-allelics before intersection, harden the consensus channel joins BCFTOOLS_NORM ran with -Oz only, so multi-allelic records were never split. BCFTOOLS_ISEC matches on exact CHROM/POS/REF/ALT, so a site one caller reports as A>G,GT and the other as A>G never intersected: in consensus mode the variant was dropped, in union mode it appeared twice. ClairS-TO emits no multi-allelic records while DeepVariant emits ~2.5% (128,115 of 5,058,527 on B1975944), so every DeepVariant multi-allelic site was systematically excluded from the consensus. Adding -m -any splits them first; verified on real DeepSomatic output (45,638 -> 45,677 records over chr1:1-5Mb, 39 split, none left multi-allelic). This also makes the AF declaration mismatch harmless: ClairS-TO declares FORMAT/AF as Number=1 while DeepVariant and DeepSomatic declare Number=A, and bcftools concat only warns before keeping the first file's definition. Once every record is biallelic the two declarations are equivalent. The comment on STANDARDIZE_AF is corrected accordingly: it claimed DeepVariant emits VAF, but all four callers emit FORMAT/AF and none emits VAF, so the rename is a no-op under the default prioritize_caller='clair' and is kept as a guarantee for WAKHAN rather than as a live conversion. Two silent-failure paths in SMALL_VARIANT_CONSENSUS are closed. The caller branch had no `other` arm, so an unrecognised meta.caller removed the sample from every downstream result with a successful exit; it now errors. The DeepVariant/Clair join was a plain join, so a sample present for one caller but not the other was dropped just as quietly; it now uses failOnMismatch and failOnDuplicate, matching PHASING_HAPLOTYPING. Record counts in the merged output change, so the snapshots need regenerating. Co-Authored-By: Claude Opus 5 (1M context) --- conf/modules.config | 7 +++++- subworkflows/local/small_variant_consensus.nf | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index ad58f252..99872d66 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -121,8 +121,13 @@ process { withName: '.*:BCFTOOLS_NORM' { ext.prefix = { "${meta.id}.${meta.caller}_norm" } + // -m -any splits multi-allelic records into biallelic ones. BCFTOOLS_ISEC matches on exact + // CHROM/POS/REF/ALT, so without this a site one caller reports as A>G,GT and the other as + // A>G never intersects: in consensus mode the variant is dropped, in union mode it appears + // twice. ClairS-TO emits no multi-allelic records at all while DeepVariant emits ~2.5%, so + // every DeepVariant multi-allelic site was systematically excluded from the consensus. ext.args = { - "-Oz" + "-m -any -Oz" } publishDir = [ enabled: false diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index f45e959b..43e5114a 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -45,6 +45,16 @@ workflow SMALL_VARIANT_CONSENSUS { // MODULE: STANDARDIZE_AF (BCFTOOLS_ANNOTATE alias, label: process_low) -- rename the AF FORMAT field to the priority caller's: // FORMAT/AF -> FORMAT/VAF when prioritize_caller is 'deepvariant'/'deepsomatic' // FORMAT/VAF -> FORMAT/AF when prioritize_caller is 'clair' + // This guarantees the merged VCF exposes allele frequency under a single FORMAT key, which is + // what WAKHAN consumes. Every caller currently emits FORMAT/AF and none emits VAF (verified + // against Clair3, ClairS-TO, DeepVariant and DeepSomatic output), so under the default + // prioritize_caller='clair' it is a no-op; it is kept as the guarantee, not the mechanism. + // + // The callers do disagree on the AF *declaration*: ClairS-TO says Number=1, DeepVariant and + // DeepSomatic say Number=A. bcftools concat only warns and keeps the first file's definition. + // Renaming cannot fix that and `annotate -h` cannot override an existing FORMAT definition, + // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record + // reaching here carries one ALT and one AF value, making the two declarations equivalent. // if (combine_method == 'all') { normalized_vcfs @@ -105,12 +115,20 @@ workflow SMALL_VARIANT_CONSENSUS { // annotated_vcfs: [meta(+caller), vcf, tbi] -- VCF with CALLER INFO tag // Branch annotated VCFs by caller family for the intersection step + // An unrecognised meta.caller would silently vanish without the `other` arm, taking the whole + // sample out of the results with a successful exit. annotated_vcfs .branch { meta, _vcfs, _tbi -> deepvariant: meta.caller in [ 'deepvariant', 'deepsomatic' ] clair: meta.caller in ['clair3','clairs-to','clairs'] + other: true } .set{annotated_vcfs_branched} + + annotated_vcfs_branched.other + .map { meta, _vcfs, _tbi -> + error("SMALL_VARIANT_CONSENSUS: unrecognised meta.caller '${meta.caller}' for sample '${meta.id}'; expected one of [deepvariant, deepsomatic, clair3, clairs-to, clairs]") + } // annotated_vcfs_branched.deepvariant: [meta(caller=deepvariant/deepsomatic), vcf, tbi] // annotated_vcfs_branched.clair: [meta(caller=clair3/clairs-to/clairs), vcf, tbi] @@ -153,8 +171,10 @@ workflow SMALL_VARIANT_CONSENSUS { // deepvariant_ch: [meta (no caller), vcf, tbi] // Join DeepVariant and Clair VCFs per sample into a single tuple for BCFTOOLS_ISEC + // failOnMismatch: a sample present for one caller but not the other would otherwise be dropped + // from every downstream result while the run still reported success. deepvariant_ch - .join(clair_ch) + .join(clair_ch, failOnMismatch: true, failOnDuplicate: true) .map { meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> def vcfs = [deepvar_vcf, clair_vcf] def tbis = [deepvar_tbi, clair_tbi] From b646c20878318e6627c9ae763085f2869b46dc36 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:42:38 +0200 Subject: [PATCH 07/36] Adjudicate the tumour-only germline arm with DeepSomatic's verdict With --germline_var_keep deepvariant the tumour-only germline arm runs DeepVariant on the tumour BAM. DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is only PASS/RefCall/LowQual/NoCall -- so those calls mix germline and clonal somatic variants and cannot be separated on their own. Published unchanged, germline_smallvariants.vcf.gz and vep/germline/ carried most of the somatic call set. DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic variants"), PON, RefCall or PASS. That verdict is now transferred onto the DeepVariant records as INFO/DS_VERDICT and used to select the germline arm, mirroring how ClairS-TO adjudicates its own calls through NonSomatic and VCFSPLIT. This is a caller's own adjudication rather than positional subtraction of the somatic call set: on B1975944 only 1.14% (57,684) of DeepVariant's 5,058,527 PASS calls carry a positive somatic verdict, against ~87% positional overlap with the union somatic set. Only positively adjudicated germline sites are kept (GERMLINE or PON). RefCall and sites DeepSomatic never evaluated are dropped rather than assumed germline, so the arm is not overpopulated with unadjudicated calls; it keeps 83.1% of the input. The verdict stays in INFO/DS_VERDICT so the decision is auditable in the published VCF. The transfer is three independent bcftools invocations, so it is three aliased instances of the existing upstream modules (DS_VERDICT_QUERY, DS_VERDICT_ANNOTATE, DS_GERMLINE_SELECT) rather than a new bespoke process. DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS over 13.7M records), so transferring it as a string cannot inject the ";" that would break INFO parsing. DEEPSOMATIC now runs before DEEPVARIANT in the subworkflow because the verdict is built from its raw VCF, before the PASS filter discards the GERMLINE records. The deep family must therefore be enabled as a pair: --germline_var_keep deepvariant without deepsomatic in --somatic_var_keep is rejected at launch rather than silently producing an unadjudicated germline arm. Paired mode is untouched -- both Clair3 and DeepVariant already run on the normal BAM there, so the germline arm needs no adjudication. Also corrects the --smallvar_filter_pass docs: VCFTAG normalises FILTER to PASS, so `false` does not restore the pre-filter behaviour as previously claimed. Germline record counts change, so the snapshots need regenerating. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + conf/modules.config | 39 +++++ docs/usage.md | 35 ++++- .../local/tumor_only/tumoronly_smallvar.nf | 148 ++++++++++++------ .../utils_nfcore_lrsomatic_pipeline/main.nf | 24 +++ 5 files changed, 198 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04dd66d4..2710942b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. Note that `--smallvar_filter_pass false` does **not** fully restore the previous behaviour: `VCFTAG` normalises `FILTER` to `PASS`, so the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` no longer removes anything. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). diff --git a/conf/modules.config b/conf/modules.config index 99872d66..1cf9262a 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,6 +166,45 @@ process { enabled: false ] } + // GERMLINE VERDICT TRANSFER (tumor-only deep family). These three selectors are matched on the + // alias, so they do not disturb the generic BCFTOOLS_QUERY/ANNOTATE/VIEW entries below. + withName: '.*:DS_VERDICT_QUERY' { + ext.prefix = { "${meta.id}.ds_verdict" } + // FILTER is single-valued in DeepSomatic output (RefCall/GERMLINE/PON/PASS), so it cannot + // inject the ";" that would break INFO parsing once transferred. + ext.args = { + "-f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" + } + publishDir = [ + enabled: false + ] + } + + withName: '.*:DS_VERDICT_ANNOTATE' { + ext.prefix = { "${meta.id}.deepvariant_verdict" } + ext.args = { + '''-h <(echo '##INFO=') \ + -c CHROM,POS,REF,ALT,INFO/DS_VERDICT \ + -Oz \ + -W=tbi''' + } + publishDir = [ + enabled: false + ] + } + + withName: '.*:DS_GERMLINE_SELECT' { + ext.prefix = { "${meta.id}.deepvariant_germline" } + // Keep only positively-adjudicated germline sites. RefCall and sites DeepSomatic never + // evaluated (DS_VERDICT absent) are dropped rather than assumed germline. + ext.args = { + "-i 'INFO/DS_VERDICT=\"GERMLINE\" || INFO/DS_VERDICT=\"PON\"' --output-type z --write-index=tbi" + } + publishDir = [ + enabled: false + ] + } + withName: '.*:BCFTOOLS_QUERY' { ext.args = { "-f '%CHROM\t%POS\t%REF\t%ALT\t${meta.caller}\n'" diff --git a/docs/usage.md b/docs/usage.md index 7e62197c..bce38531 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -135,6 +135,23 @@ For structural variants, the CHM13 panel of normals is a merged panel combining For tumour-only small variants, ClairS-TO separates germline from somatic calls with a panel of normals and with its Verdict module, which tags each call as germline, somatic or subclonal somatic from tumour purity and allele-specific copy number. `--genome CHM13` supplies five CHM13 PON VCFs (gnomAD, dbSNP, 1000 Genomes, CoLoRSdb and ASAP), which **replace** the GRCh38 databases inside the container. Unless `--skip_ascat` is set, purity and copy number come from the pipeline's own ASCAT run (`CLAIRSTO_VERDICT_TAG`); only with `--skip_ascat` does ClairS-TO estimate them itself, from assembly-specific loci, allele and GC content files. A GRCh38 resource set on a CHM13 run leaves germline variants untagged. +When `--germline_var_keep` includes `deepvariant`, the tumour-only germline arm +runs DeepVariant on the **tumour** BAM. DeepVariant is a germline caller with no +somatic discrimination, so on its own those calls mix germline and clonal somatic +variants. The pipeline therefore transfers DeepSomatic's verdict onto them: +DeepSomatic evaluates the same sites and labels each `GERMLINE`, `PON`, `RefCall` +or `PASS`, and that label is recorded in `INFO/DS_VERDICT`. Only positively +adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; +`RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed +germline. On a 30x tumour-only sample this keeps about 83% of DeepVariant's +`PASS` calls and removes roughly 1% that DeepSomatic positively calls somatic. + +Because of this, `deepvariant` and `deepsomatic` must be enabled together: +`--germline_var_keep deepvariant` without `deepsomatic` in `--somatic_var_keep` is +rejected at launch. Note that even after adjudication the tumour-only germline arm +is a tumour-derived proxy, not a call set from normal tissue, and should not be +used for secondary findings without that caveat. + With `--genome CHM13 --skip_ascat` the pipeline builds a CHM13 resource set from the ASCAT files it already downloads, so no extra setup is needed. LogR correction is GC-only, as ClairS-TO recommends for CHM13: no replication timing file is published for the assembly. Without `--skip_ascat` nothing is built, because the tagging comes from ASCAT's own tables. To use a resource set of your own — another assembly, or a CHM13 set carrying an `RT_.txt` for replication timing correction — pass `--clairsto_cna_resources` together with `--skip_ascat`. Without `--skip_ascat` it is ignored, with a warning. Expected layout: @@ -387,11 +404,19 @@ otherwise be "every site every caller looked at", which inflates the phased VCFs by three orders of magnitude and produces a meaningless mutation burden. `--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's -VCF that is handed to the caller consensus, phasing, VEP and the report. Set it to -`false` to restore the previous unfiltered behaviour. In tumor-only mode ClairS-TO -is unaffected by the setting: `VCFSPLIT` already restricts its somatic split to -`PASS`. The per-caller VCFs published under `//variants/` -are never filtered, so no calls are lost from the results directory. +VCF that is handed to the caller consensus, phasing, VEP and the report. In +tumor-only mode ClairS-TO is unaffected by the setting: `VCFSPLIT` already +restricts its somatic split to `PASS`, and its germline split is `PASS`-rewritten +rather than `PASS`-filtered. The per-caller VCFs published under +`//variants/` are never filtered, so no calls are lost +from the results directory. + +Setting `--smallvar_filter_pass false` does **not** fully restore the pre-filter +behaviour. `VCFTAG` normalises `FILTER` to `PASS` on both arms before phasing, so +the later `--apply-filters PASS` on the signature input no longer removes +anything: with the filter off, non-`PASS` records reach SigProfiler that +previously could not. Use it to inspect the unfiltered call set, not to reproduce +older results. `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 2fbd1067..cb89b849 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -11,6 +11,12 @@ include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subwork include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +// Germline verdict transfer: DeepSomatic adjudicates DeepVariant's tumor-derived germline calls. +// Three independent bcftools invocations, so three aliased instances of the upstream modules. +include { BCFTOOLS_QUERY as DS_VERDICT_QUERY } from '../../../modules/nf-core/bcftools/query/main' +include { BCFTOOLS_ANNOTATE as DS_VERDICT_ANNOTATE } from '../../../modules/nf-core/bcftools/annotate/main' +include { BCFTOOLS_VIEW as DS_GERMLINE_SELECT } from '../../../modules/nf-core/bcftools/view/main' + workflow TUMORONLY_SMALLVAR { @@ -132,6 +138,50 @@ workflow TUMORONLY_SMALLVAR { // clairsto_somatic_ch: [meta(+caller:'clairs-to'), vcf, tbi] -- somatic variants } + // DEEPSOMATIC in tumor-only mode: normal BAM/BAI are empty lists + if(somatic_var_keep.contains('deepsomatic')) { + tumor_bams + .map { meta, tumor_bam, tumor_bai -> + def normal_bam = [] + def normal_bai = [] + return [meta,normal_bam,normal_bai,tumor_bam,tumor_bai] + } + .set{deepsomatic_input_ch} + // deepsomatic_input_ch: [meta, [], [], tumor_bam, tumor_bai] + // empty normal_bam/bai signals tumor-only mode to DEEPSOMATIC subworkflow + + // + // SUBWORKFLOW: DEEPSOMATIC (local) + // Input: [meta, [], [], tumor_bam, tumor_bai] -- tumor-only (no normal) + // [[:],[]] / fasta / fai / [[:],[]] + // Output: .vcf -- [meta, vcf] + // .vcf_index -- [meta, tbi] + // + DEEPSOMATIC ( + deepsomatic_input_ch, + [[:],[]], // intervals (empty = genome-wide) + fasta, + fai, + [[:],[]], // GZI (empty if FASTA is uncompressed) + ds_pon_channel + ) + // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), + // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT + // already restricts it to PASS. The VCF published under variants/deepsomatic/ is + // unaffected. + DEEPSOMATIC_PASS_FILTER ( + DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + ) + + DEEPSOMATIC_PASS_FILTER.out.vcf + .map{ meta, vcf, tbi -> + def new_meta = meta + [caller:'deepsomatic'] + return [new_meta, vcf, tbi] + } + .set{deepsomatic_ch} + // deepsomatic_ch: [meta(+caller:'deepsomatic'), vcf, tbi] + } + // DEEPVARIANT: germline-only variant calling (no somatic mode for tumor-only) if(germline_var_keep.contains('deepvariant')) { @@ -168,13 +218,66 @@ workflow TUMORONLY_SMALLVAR { DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) ) + // GERMLINE VERDICT TRANSFER (tumor-only, deep family) + // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is + // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its + // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the + // germline VCF therefore carries most of the somatic call set. + // + // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic + // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO + // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 + // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% + // (57,684) PASS -- the last being real somatic calls that must not be published as germline. + // + // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and + // unevaluated sites are dropped rather than assumed germline. The verdict stays in + // INFO/DS_VERDICT so the decision is auditable in the published VCF. + // + // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified + // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would + // break INFO parsing. + // + // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) + // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter + // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER + // + DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) + + // + // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) + // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. + // DEEPVARIANT_PASS_FILTER.out.vcf + .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) + .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) + .map { meta, vcf, tbi, annotations, annotations_index -> + def columns = [] // no extra column specs + def header_lines = [] // no extra header lines + def rename_chrs = [] // no chromosome renaming + return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] + } + .set{ ds_verdict_annotate_input } + + DS_VERDICT_ANNOTATE ( ds_verdict_annotate_input ) + + // + // MODULE: DS_GERMLINE_SELECT (BCFTOOLS_VIEW alias, label: process_medium) + // Keeps only the positively-adjudicated germline records (see ext.args in conf/modules.config). + // + DS_GERMLINE_SELECT ( + DS_VERDICT_ANNOTATE.out.vcf.join(DS_VERDICT_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true), + [], [], [] + ) + + DS_GERMLINE_SELECT.out.vcf + .join(DS_GERMLINE_SELECT.out.index, failOnMismatch: true, failOnDuplicate: true) .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] } .set{deepvariant_ch} - // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] + // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] -- germline-adjudicated only } // COMBINE GERMLINE VARIANTS @@ -207,49 +310,6 @@ workflow TUMORONLY_SMALLVAR { .set{germline_vcf} } - // DEEPSOMATIC in tumor-only mode: normal BAM/BAI are empty lists - if(somatic_var_keep.contains('deepsomatic')) { - tumor_bams - .map { meta, tumor_bam, tumor_bai -> - def normal_bam = [] - def normal_bai = [] - return [meta,normal_bam,normal_bai,tumor_bam,tumor_bai] - } - .set{deepsomatic_input_ch} - // deepsomatic_input_ch: [meta, [], [], tumor_bam, tumor_bai] - // empty normal_bam/bai signals tumor-only mode to DEEPSOMATIC subworkflow - - // - // SUBWORKFLOW: DEEPSOMATIC (local) - // Input: [meta, [], [], tumor_bam, tumor_bai] -- tumor-only (no normal) - // [[:],[]] / fasta / fai / [[:],[]] - // Output: .vcf -- [meta, vcf] - // .vcf_index -- [meta, tbi] - // - DEEPSOMATIC ( - deepsomatic_input_ch, - [[:],[]], // intervals (empty = genome-wide) - fasta, - fai, - [[:],[]], // GZI (empty if FASTA is uncompressed) - ds_pon_channel - ) - // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), - // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT - // already restricts it to PASS. The VCF published under variants/deepsomatic/ is - // unaffected. - DEEPSOMATIC_PASS_FILTER ( - DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) - ) - - DEEPSOMATIC_PASS_FILTER.out.vcf - .map{ meta, vcf, tbi -> - def new_meta = meta + [caller:'deepsomatic'] - return [new_meta, vcf, tbi] - } - .set{deepsomatic_ch} - // deepsomatic_ch: [meta(+caller:'deepsomatic'), vcf, tbi] - } // COMBINE SOMATIC VARIATION if (somatic_var_keep.size() > 1) { diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index a64c405c..072a9d5f 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -239,6 +239,30 @@ workflow PIPELINE_COMPLETION { def validateInputParameters() { genomeExistsError() validateReportGenePanels() + validateDeepCallerPairing() +} + +// +// The tumor-only germline arm runs DeepVariant on the TUMOR BAM, which cannot separate germline +// from clonal somatic calls on its own. TUMORONLY_SMALLVAR resolves that by transferring +// DeepSomatic's FILTER verdict (GERMLINE/PON/RefCall/PASS) onto those calls, so DeepVariant +// without DeepSomatic would leave the germline arm unadjudicated -- and would reference +// DEEPSOMATIC.out before it exists. Require the deep family to be fully on or fully off. +// +def callerList(value) { + if (value instanceof List) { + return value + } + return value.toString().tokenize(',').collect { token -> token.trim() } +} + +def validateDeepCallerPairing() { + def germline = callerList(params.germline_var_keep) + def somatic = callerList(params.somatic_var_keep) + + if (germline.contains('deepvariant') && !somatic.contains('deepsomatic')) { + error("--germline_var_keep includes 'deepvariant' but --somatic_var_keep does not include 'deepsomatic'. In tumor-only mode the DeepVariant germline calls are adjudicated with DeepSomatic's verdict, so the two must be enabled together. Add 'deepsomatic' to --somatic_var_keep, or drop 'deepvariant' from --germline_var_keep.") + } } // From 33bb9b2223619acbda5b3aa2f81d8ef4b5dd9f70 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:46:51 +0200 Subject: [PATCH 08/36] Preserve the caller's FILTER through VCFTAG so the PASS filter still bites VCFTAG rewrites FILTER to PASS on both arms before phasing, which is needed so that downstream tools filtering on PASS still see the germline records ClairS-TO marks NonSomatic. But it did so destructively, and every record reaching the published phased VCFs, VEP and the report therefore read FILTER=PASS with the caller's verdict gone. That also silently disabled the signature-stage filter. SIGNATURES_BCFTOOLS_VIEW ran --apply-filters PASS on a call set whose FILTER had already been normalised to PASS, so it could never remove anything: with --smallvar_filter_pass false, RefCall/LowQual/GERMLINE/PON records reached SigProfilerMatrixGenerator that previously could not, and the matrices were computed over mostly-reference sites. The parameter was documented as restoring the previous behaviour, which it did not. VCFTAG now records the caller's FILTER in INFO/ORIG_FILTER before overwriting it, as VCFSPLIT already did for the ClairS-TO arm, and the signature filter tests that field instead. VCFSPLIT's stamp is respected rather than duplicated: both the header line and the per-record field are added only when absent, since a duplicate INFO key makes the record unparseable. Multi-valued FILTER is joined with "," because ";" separates INFO fields, and a FILTER of "." is recorded as "." rather than skipped. bcftools accepts only one of -i/-e, so the ALT="*" exclusion is folded into the same include expression. Verified against the module fixture with bcftools 1.20, including a record that already carried ORIG_FILTER and one with a multi-valued FILTER: exactly one ORIG_FILTER header line, no double stamping, and htslib parses the result. The module test is extended to cover these cases. Note that the module test cannot currently run locally: nf-test 0.9.3 generates `VCFTAG(*input)` and the available Nextflow 26.04.1 rejects the spread operator under its strict syntax, while the pipeline requires >=25.10.4 so 25.04.6 will not run either. This is pre-existing and reproduces unchanged at 5b38dcf. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- conf/modules.config | 10 ++++++++-- docs/usage.md | 13 +++++++------ modules/local/vcftag/main.nf | 21 ++++++++++++++++++++- modules/local/vcftag/tests/main.nf.test | 14 ++++++++++++-- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2710942b..4789d488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. Note that `--smallvar_filter_pass false` does **not** fully restore the previous behaviour: `VCFTAG` normalises `FILTER` to `PASS`, so the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` no longer removes anything. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now runs [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) v1.6.0 from `ghcr.io/ljwharbers/lrsomatic-report:1.6.0` (`oras://ghcr.io/ljwharbers/lrsomatic-report-sif:1.6.0` under Singularity/Apptainer) instead of the source vendored at `assets/lrsomatic_report`, which is deleted; updating the tool is now a tag bump. The image is `linux/amd64` only and **Conda is not supported for this step**: the module errors under `-profile conda`/`mamba`, `conda` is dropped from the nf-test CI matrix, and `--skip_report` gives a conda run (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 1cf9262a..29b9d235 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -717,8 +717,14 @@ process { } withName : '.*:SIGNATURES_BCFTOOLS_VIEW' { - // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only - ext.args = { "--apply-filters PASS --types snps,mnps,indels --exclude 'ALT=\"*\"' --output-type v" } + // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only. + // The PASS test reads INFO/ORIG_FILTER, not FILTER: VCFTAG normalises FILTER to PASS on + // every record before phasing, so --apply-filters PASS here could never remove anything and + // non-PASS records reached SigProfiler whenever --smallvar_filter_pass was false. + // ORIG_FILTER carries the caller's own verdict, so the filter bites again. "." is kept + // because an unfiltered record is not a rejected one. bcftools allows only one of -i/-e, + // so the ALT="*" exclusion is folded into the same expression. + ext.args = { "-i '(INFO/ORIG_FILTER=\"PASS\" || INFO/ORIG_FILTER=\".\") && ALT!=\"*\"' --types snps,mnps,indels --output-type v" } ext.prefix = { "${meta.id}_somatic_pass" } publishDir = [ enabled: false diff --git a/docs/usage.md b/docs/usage.md index bce38531..cab62881 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -411,12 +411,13 @@ rather than `PASS`-filtered. The per-caller VCFs published under `//variants/` are never filtered, so no calls are lost from the results directory. -Setting `--smallvar_filter_pass false` does **not** fully restore the pre-filter -behaviour. `VCFTAG` normalises `FILTER` to `PASS` on both arms before phasing, so -the later `--apply-filters PASS` on the signature input no longer removes -anything: with the filter off, non-`PASS` records reach SigProfiler that -previously could not. Use it to inspect the unfiltered call set, not to reproduce -older results. +Set it to `false` to restore the previous unfiltered behaviour. `VCFTAG` +normalises `FILTER` to `PASS` on both arms before phasing, so the caller's own +verdict is preserved in `INFO/ORIG_FILTER` and the signature input is filtered on +that field rather than on `FILTER`; without this the `PASS` filter in front of +SigProfiler could never remove anything once `FILTER` had been rewritten. Every +published VCF downstream of phasing therefore reads `FILTER=PASS`, with the +original value available in `INFO/ORIG_FILTER`. `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf index 3fd9dc81..a7d81868 100644 --- a/modules/local/vcftag/main.nf +++ b/modules/local/vcftag/main.nf @@ -27,20 +27,39 @@ process VCFTAG { # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and # filter on FILTER, so a FILTER-based label does not survive to where it is needed. # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. - bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"} + # + # The caller's own FILTER is preserved in INFO/ORIG_FILTER before it is overwritten, so the + # verdict is not destroyed by the normalisation. FILTER is ";"-delimited but ";" separates INFO + # fields, so it is stored as ",". VCFSPLIT already stamps ORIG_FILTER on the ClairS-TO arm; both + # the header line and the per-record field are therefore added only when not already present, + # since a duplicate INFO key would make the record unparseable. + bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"; seen_orig=0} + /^##INFO=" + if (!seen_orig) { + print "##INFO=" + } print next } { + if (\$8 !~ /(^|;)ORIG_FILTER=/) { + of = \$7 + gsub(/;/, ",", of) + \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of + } \$7 = "PASS" \$8 = (\$8 == "." || \$8 == "") ? flag : \$8 ";" flag print } ' | bgzip -c > ${prefix}.vcf.gz + # tabix will index a VCF whose header htslib cannot parse, so read it back here rather than + # letting a malformed header surface as a confusing failure in a later process. + bcftools view -h ${prefix}.vcf.gz > /dev/null + tabix -p vcf ${prefix}.vcf.gz """ diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test index 18b79c41..558f2dd7 100644 --- a/modules/local/vcftag/tests/main.nf.test +++ b/modules/local/vcftag/tests/main.nf.test @@ -42,8 +42,18 @@ nextflow_process { { assert lines.every { it.split('\t')[6] == 'PASS' } }, // a pre-existing INFO field is preserved rather than overwritten { assert lines.any { it.contains('EXISTING') && it.contains('CALLER=clairs-to') } }, - // the record whose INFO was '.' gets the bare flag, not '.;SOMATIC' - { assert lines.any { it.split('\t')[7] == 'SOMATIC' } } + // the record whose INFO was '.' gets no leading '.', just the stamped fields + { assert lines.any { it.split('\t')[7] == 'ORIG_FILTER=RefCall;SOMATIC' } }, + // the caller's FILTER is preserved before being normalised away + { assert header.contains('ID=ORIG_FILTER') }, + { assert lines.every { it.contains('ORIG_FILTER=') } }, + { assert lines.any { it.contains('ORIG_FILTER=PASS') } }, + { assert lines.any { it.contains('ORIG_FILTER=NonSomatic') } }, + { assert lines.any { it.contains('ORIG_FILTER=RefCall') } }, + // a record whose FILTER was '.' records '.' rather than being skipped + { assert lines.any { it.contains('ORIG_FILTER=.') } }, + // ORIG_FILTER is stamped exactly once per record + { assert lines.every { it.split('\t')[7].count('ORIG_FILTER=') == 1 } } ) } } From 33a631badbb087d116190d5ff9c7f560e15749a0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:58:50 +0200 Subject: [PATCH 09/36] Make the VCFTAG test runnable The test read its output through path(...).vcf, an accessor provided by an nft-vcf plugin that nf-test.config does not load, so it failed with MissingPropertyException on every run and had never passed. It now uses the built-in linesGzip accessor instead, which needs no plugin. Co-Authored-By: Claude Opus 5 (1M context) --- modules/local/vcftag/tests/main.nf.test | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test index 558f2dd7..3f4a658f 100644 --- a/modules/local/vcftag/tests/main.nf.test +++ b/modules/local/vcftag/tests/main.nf.test @@ -28,8 +28,11 @@ nextflow_process { then { assert process.success - def lines = path(process.out.vcf[0][1]).vcf.variantsAsStrings - def header = path(process.out.vcf[0][1]).vcf.header.getMetaDataInSortedOrder().join('\n') + // linesGzip is built into nf-test; the .vcf accessor needs an nft-vcf plugin that + // nf-test.config does not load, so this reads the records directly. + def all = path(process.out.vcf[0][1]).linesGzip + def lines = all.findAll { line -> !line.startsWith('#') } + def header = all.findAll { line -> line.startsWith('##') }.join('\n') assertAll( // every input record survives -- tagging must not filter From 825af157f55db18e7e0088660be0a544c77808bb Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 12:06:14 +0200 Subject: [PATCH 10/36] Fold STANDARDIZE_AF into BCFTOOLS_ANNOTATE STANDARDIZE_AF and BCFTOOLS_ANNOTATE were two back-to-back bcftools annotate calls on the same file with disjoint options -- one renaming the allele frequency FORMAT key, the next stamping INFO/CALLER. --rename-annots composes with -a/-c/-h, verified against a real VCF, so a single invocation does both and the alias is removed along with its modules.config block. The rename is selected by meta.rename_to, set only when combine_method is 'all'; in 'consensus' mode every surviving record comes from one caller and needs no rename. Ordering is safe because BCFTOOLS_QUERY reads only CHROM/POS/REF/ALT and does not care whether the rename has happened. The rendered ext.args was checked for all three meta cases (VAF, AF, key absent): the escapes reach printf intact and an absent key yields no --rename-annots. The joins around the annotate step now use failOnMismatch/failOnDuplicate for the same reason as the caller join: a missing annotation table should stop the run, not quietly drop the sample. Co-Authored-By: Claude Opus 5 (1M context) --- conf/modules.config | 22 +++++---- subworkflows/local/small_variant_consensus.nf | 45 ++++++++----------- 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 29b9d235..5ca2a6c5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -143,21 +143,19 @@ process { enabled: false ] } - withName: '.*STANDARDIZE_AF' { - ext.prefix = { "${meta.id}.${meta.caller}_standardized" } - ext.args = { - meta.rename_to == 'VAF' - ? "--rename-annots <(printf 'FORMAT/AF\\tFORMAT/VAF\\n') -Oz -W=tbi" - : "--rename-annots <(printf 'FORMAT/VAF\\tFORMAT/AF\\n') -Oz -W=tbi" - } - publishDir = [ - enabled: false - ] - } withName: '.*:BCFTOOLS_ANNOTATE' { ext.prefix = { "${meta.id}.${meta.caller}" } + // Stamps INFO/CALLER and, in 'all' mode, unifies the allele frequency key in the same + // invocation: --rename-annots composes with -a/-c/-h, so this does not need a second + // bcftools annotate pass. meta.rename_to is set only when SMALL_VARIANT_CONSENSUS is + // merging two callers into one VCF. ext.args = { - '''-h <(echo '##INFO=') \ + def rename = meta.rename_to == 'VAF' + ? "--rename-annots <(printf 'FORMAT/AF\\tFORMAT/VAF\\n') " + : meta.rename_to == 'AF' + ? "--rename-annots <(printf 'FORMAT/VAF\\tFORMAT/AF\\n') " + : "" + rename + '''-h <(echo '##INFO=') \ -c CHROM,POS,REF,ALT,INFO/CALLER \ -Oz \ -W=tbi''' diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index 43e5114a..d50ba7f7 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -2,7 +2,6 @@ include { BCFTOOLS_NORM } from '../../modul include { BCFTOOLS_ISEC } from '../../modules/nf-core/bcftools/isec/main' include { BCFTOOLS_QUERY } from '../../modules/nf-core/bcftools/query/main' include { BCFTOOLS_ANNOTATE } from '../../modules/nf-core/bcftools/annotate/main' -include { BCFTOOLS_ANNOTATE as STANDARDIZE_AF } from '../../modules/nf-core/bcftools/annotate/main' include { BCFTOOLS_CONCAT } from '../../modules/nf-core/bcftools/concat/main' include { BCFTOOLS_SORT } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_SORT as SORT_POST_NORM } from '../../modules/nf-core/bcftools/sort/main' @@ -42,7 +41,7 @@ workflow SMALL_VARIANT_CONSENSUS { // normalized_vcfs: [meta(+caller), vcf.gz, tbi] -- normalised, sorted per-caller VCF // - // MODULE: STANDARDIZE_AF (BCFTOOLS_ANNOTATE alias, label: process_low) -- rename the AF FORMAT field to the priority caller's: + // ALLELE FREQUENCY KEY -- BCFTOOLS_ANNOTATE below renames the AF FORMAT field to the priority caller's: // FORMAT/AF -> FORMAT/VAF when prioritize_caller is 'deepvariant'/'deepsomatic' // FORMAT/VAF -> FORMAT/AF when prioritize_caller is 'clair' // This guarantees the merged VCF exposes allele frequency under a single FORMAT key, which is @@ -56,27 +55,10 @@ workflow SMALL_VARIANT_CONSENSUS { // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record // reaching here carries one ALT and one AF value, making the two declarations equivalent. // - if (combine_method == 'all') { - normalized_vcfs - .map { meta, vcf, tbi -> - def rename_to = prioritize_caller in ['deepvariant', 'deepsomatic'] ? 'VAF' : 'AF' - def new_meta = meta + [rename_to: rename_to] - return [new_meta, vcf, tbi, [], [], [], [], []] - } - .set { standardize_input } - - STANDARDIZE_AF(standardize_input) - - STANDARDIZE_AF.out.vcf - .join(STANDARDIZE_AF.out.tbi) - .map { meta, vcf, tbi -> - def clean_meta = meta.findAll { k, _v -> k != 'rename_to' } - return [clean_meta, vcf, tbi] - } - .set { normalized_vcfs } - // normalized_vcfs: [meta(+caller), vcf, tbi] -- normalised, AF-standardized per-caller VCF - } - // In 'consensus' mode, normalized_vcfs comes from SORT_POST_NORM (post-BCFTOOLS_NORM re-sorting) + // The rename is carried out by BCFTOOLS_ANNOTATE below rather than by a second annotate call: + // meta.rename_to selects the --rename-annots file in conf/modules.config. Only 'all' mode needs + // it, since in 'consensus' mode every surviving record comes from one caller. BCFTOOLS_QUERY + // reads only CHROM/POS/REF/ALT, so it does not care whether the rename has happened yet. // // MODULE: BCFTOOLS_QUERY (label: process_single) @@ -89,13 +71,18 @@ workflow SMALL_VARIANT_CONSENSUS { // Prepare BCFTOOLS_ANNOTATE input: VCF + caller-name annotation file normalized_vcfs - .join(BCFTOOLS_QUERY.out.output) - .join(BCFTOOLS_QUERY.out.index) + .join(BCFTOOLS_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) + .join(BCFTOOLS_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) .map{ meta, vcf, tbi, annotations, annotations_index -> def columns = [] // no extra column specs def header_lines = [] // no extra header lines def rename_chrs = [] // no chromosome renaming - return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] + // 'all' mode merges records from both callers into one VCF, so the allele + // frequency key is unified here; 'consensus' mode needs no rename. + def new_meta = combine_method == 'all' + ? meta + [rename_to: (prioritize_caller in ['deepvariant', 'deepsomatic'] ? 'VAF' : 'AF')] + : meta + return [ new_meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] } .set{annotate_input} // annotate_input: [meta, vcf, tbi, annotations_tsv, annotations_tbi, [], [], []] @@ -110,7 +97,11 @@ workflow SMALL_VARIANT_CONSENSUS { BCFTOOLS_ANNOTATE(annotate_input) BCFTOOLS_ANNOTATE.out.vcf - .join(BCFTOOLS_ANNOTATE.out.tbi) + .join(BCFTOOLS_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .map { meta, vcf, tbi -> + def clean_meta = meta.findAll { k, _v -> k != 'rename_to' } + return [clean_meta, vcf, tbi] + } .set{annotated_vcfs} // annotated_vcfs: [meta(+caller), vcf, tbi] -- VCF with CALLER INFO tag From d4401a46505cd0704a48489cd12893d6736a1198 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 12:08:13 +0200 Subject: [PATCH 11/36] Drop a duplicated CLAIR3 argument and correct a false ordering comment CLAIR3 received --sample_name twice: the module passes it from ext.prefix, which defaults to meta.id, and conf/modules.config passed the identical value again. Harmless with argparse, but it reads as if the name were configurable. Removed the config copy, matching how CLAIRSTO is already handled on dev. The comment in PHASING_HAPLOTYPING claiming the somatic VCF is passed first "(higher priority in phasing)" was false: BCFTOOLS_CONCAT sorts its input file list alphabetically, so with the _germline_tagged/_somatic_tagged prefixes the germline file is always passed first. The ordering is cosmetic under -a, which emits in coordinate order; the comment now says so rather than describing an intent the module discards. Co-Authored-By: Claude Opus 5 (1M context) --- conf/modules.config | 2 +- subworkflows/local/phasing_haplotyping.nf | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 5ca2a6c5..afaeeb06 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -595,7 +595,7 @@ process { } withName: '.*:CLAIR3' { - ext.args = { "--sample_name=${meta.id}" } + // --sample_name is passed by the module itself, from ext.prefix (which defaults to meta.id). publishDir = [ path: { "${params.outdir}/${meta.id}/variants/clair3" }, mode: params.publish_dir_mode, diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index d8f8ce3e..3de385ef 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -163,7 +163,10 @@ workflow PHASING_HAPLOTYPING { tagged_germline_vcf .join(tagged_somatic_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> - def vcfs = [som_vcf, germ_vcf] // somatic first (higher priority in phasing) + // Order here is cosmetic: BCFTOOLS_CONCAT sorts its input file list alphabetically + // (modules/nf-core/bcftools/concat/main.nf), so the germline file is passed first + // regardless. With -a the output is coordinate-ordered either way. + def vcfs = [som_vcf, germ_vcf] def tbis = [som_tbi, germ_tbi] return [ meta, vcfs, tbis] } From d3b45818928f04df4a974c02bc7933055f15d1e0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 22:39:14 +0200 Subject: [PATCH 12/36] Regenerate snapshots for the germline/somatic tagging processes Picks up TAG_GERMLINE and TAG_SOMATIC, and restores the clairsto and lrsomatic_report versions that the previous snapshot run had rolled back. Co-Authored-By: Claude Opus 5 (1M context) --- tests/clair_only.nf.test.snap | 12 +++++-- tests/consensus.nf.test.snap | 49 +++++++++++++++++---------- tests/deep_only.nf.test.snap | 27 +++++++++++---- tests/default.nf.test.snap | 12 +++++-- tests/union.nf.test.snap | 62 +++++++++++++++++++++-------------- 5 files changed, 108 insertions(+), 54 deletions(-) diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index 1a13f798..3c26ccd1 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -28,7 +28,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -57,7 +57,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -111,6 +111,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -804,6 +810,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T16:39:52.362456747" + "timestamp": "2026-09-22T17:07:33.562660027" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index c7b025c3..3bcf37d7 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -33,7 +33,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -59,6 +59,15 @@ "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, + "DS_GERMLINE_SELECT": { + "bcftools": "1.23.1" + }, + "DS_VERDICT_ANNOTATE": { + "bcftools": 1.22 + }, + "DS_VERDICT_QUERY": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -80,7 +89,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -131,6 +140,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -578,38 +593,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,35824329c737d89e70234e0b8e2de3fa", - "sample1_normal.bam.bai:md5,2eccbe13acc6273ebf6332ec5a5da815", - "sample1_tumor.bam:md5,969daba408c88235a28c6ed38d513965", - "sample1_tumor.bam.bai:md5,4face49fc270117ab49d72d1413a7efb", + "sample1_normal.bam:md5,a524e54ab0685eeb79e461926d8c7906", + "sample1_normal.bam.bai:md5,ba98fb39fc3cc8b7319376b003aa3dcd", + "sample1_tumor.bam:md5,db8b0da3dd1a6f1c38420a4942881119", + "sample1_tumor.bam.bai:md5,1b75ed3238520172019817b88d254f1f", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,b85799ad0a512d6c30ff05f8b5dca54a", - "sample1_whatshap_stats.log:md5,5889131d24524deb76e5d585e3a2dd3f", - "sample1_whatshap_stats.tsv:md5,648f4ba86c78fc80d388c95193e3c4f4", + "sample1_whatshap_stats.gtf:md5,30bde8f88b7d4e88b935e88e00997ce7", + "sample1_whatshap_stats.log:md5,51d522df21c6b526909a5cc9de93eaea", + "sample1_whatshap_stats.tsv:md5,8ce67834b3b7471c04151b55585a56bc", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", - "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", - "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", - "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", + "sample2_normal.bam:md5,0b3633a4d389b4467fb3eac83da94882", + "sample2_normal.bam.bai:md5,d96d0071ab25ca8dc2327acba4395515", + "sample2_tumor.bam:md5,5c549642cc7192289222d558dc3a07a7", + "sample2_tumor.bam.bai:md5,1554e8cc8ced05271f5d5f7ac7819410", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", - "sample2_whatshap_stats.log:md5,e7336c45aa5738a67f032898accb305e", - "sample2_whatshap_stats.tsv:md5,e478af5b3adb92a590a485f444a0dabe", + "sample2_whatshap_stats.gtf:md5,2e5ace4cac0b42bb6132513062781e47", + "sample2_whatshap_stats.log:md5,9aa99ee14473f65071366ed1729ab064", + "sample2_whatshap_stats.tsv:md5,13cbc783b1357e1bd2c6bb51ae6f1267", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -636,6 +651,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:28:32.306705325" + "timestamp": "2026-09-22T17:31:36.878403725" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index dd3cba3d..53251031 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -35,6 +35,15 @@ "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, + "DS_GERMLINE_SELECT": { + "bcftools": "1.23.1" + }, + "DS_VERDICT_ANNOTATE": { + "bcftools": 1.22 + }, + "DS_VERDICT_QUERY": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -104,6 +113,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -561,14 +576,14 @@ "read_qual.txt:md5,8b92ff7dc4536188be159b95525511cd", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample3_tumor.bam:md5,965264ef8436cb887ad02c92d40fb50e", - "sample3_tumor.bam.bai:md5,cfc6329667a3c6c66e3c0ca0ace4c6e9", + "sample3_tumor.bam:md5,9449340acf09825767e1f5b4a3f0e5c2", + "sample3_tumor.bam.bai:md5,3be46bae7402c3865f27881457b7a466", "sample3_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", - "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,061ecb622d83678229f5d0a7b0e95d75", - "sample3_whatshap_stats.tsv:md5,7c7453a5eed17e9ccc405844f3bd511a", + "sample3_whatshap_stats.gtf:md5,5a9b20b6ed25ef2ede4271156a83ce88", + "sample3_whatshap_stats.log:md5,a62c149286176f863d4293d0748a6dec", + "sample3_whatshap_stats.tsv:md5,90c20ce05eef472d9f5bb77754636d31", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -581,6 +596,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:36:01.066378821" + "timestamp": "2026-09-22T17:50:37.140724409" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index c0b8e75a..9310ac78 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -18,7 +18,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -47,7 +47,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -95,6 +95,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -582,6 +588,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:44:15.090049384" + "timestamp": "2026-09-22T18:06:32.118064724" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 0663c675..d7a68115 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -30,7 +30,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -56,6 +56,15 @@ "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, + "DS_GERMLINE_SELECT": { + "bcftools": "1.23.1" + }, + "DS_VERDICT_ANNOTATE": { + "bcftools": 1.22 + }, + "DS_VERDICT_QUERY": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -77,7 +86,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -123,14 +132,17 @@ "SORT_POST_NORM": { "bcftools": 1.22 }, - "STANDARDIZE_AF": { - "bcftools": 1.22 - }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -578,52 +590,52 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,271da2f58d613f1f101a12ef06eca766", - "sample1_normal.bam.bai:md5,c51c824e0ac5451f9c10eee78f40cb98", - "sample1_tumor.bam:md5,f5f68131d379181bbc335c9530e9cc7e", - "sample1_tumor.bam.bai:md5,75e62a6480344075038ce6ad2499bfbb", + "sample1_normal.bam:md5,dcb537c163750d41140d0a4da107f18e", + "sample1_normal.bam.bai:md5,12cf13fae8230cfc1cc8888f80102143", + "sample1_tumor.bam:md5,58cf65517c91958f95a1f4be1c75e2df", + "sample1_tumor.bam.bai:md5,e40049da796ad038e7ebeaa9ac070488", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,2b8ff0a9e055c9b4432bc3da9cf98a4d", - "sample1_whatshap_stats.log:md5,70a5270fec94f8b569a4a7bda49751db", - "sample1_whatshap_stats.tsv:md5,1eab3a16f1a51016c3b74f104de152a6", + "sample1_whatshap_stats.gtf:md5,9a7823e6b572036a7e95904b0083b8ca", + "sample1_whatshap_stats.log:md5,05c819627f77305a75d776a27dbf247e", + "sample1_whatshap_stats.tsv:md5,5b9d3f6a6e1e1c67c855f8f9b581bb18", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", - "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", - "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", - "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", + "sample2_normal.bam:md5,74e92562e0cd406a6282bce98ccc93d2", + "sample2_normal.bam.bai:md5,430c3b069a79227396a6b4f1c37a02ef", + "sample2_tumor.bam:md5,0059b8022c674b5daf2fd001ab69a54e", + "sample2_tumor.bam.bai:md5,e236afb5f7a644827e2b38cd36838d98", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", - "sample2_whatshap_stats.log:md5,be94280217c27d05773afa1465152c8a", - "sample2_whatshap_stats.tsv:md5,0c97b6033a149898e88aaf9c632b2de4", + "sample2_whatshap_stats.gtf:md5,8107c5dce892aa11145274fd088788f8", + "sample2_whatshap_stats.log:md5,c2b17319f0f9e6a8ec102be86aa833ba", + "sample2_whatshap_stats.tsv:md5,7028c4a428f5d0fd31472439e287e565", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", "read_qual.txt:md5,8b92ff7dc4536188be159b95525511cd", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample3_tumor.bam:md5,116b6944da4aa833a8d21c46b5f5ecfe", - "sample3_tumor.bam.bai:md5,ea9eca53bbaba26d40b791a2ea1aadf6", + "sample3_tumor.bam:md5,94e7f18f0a22d16f0868780df7975447", + "sample3_tumor.bam.bai:md5,a2cc1013d6fde4fec50608fa9681a7a7", "sample3_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", - "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,496a531c3c19585603fa7fded0c958ba", - "sample3_whatshap_stats.tsv:md5,fba638c8da0d7a9fdacb11734e219b16", + "sample3_whatshap_stats.gtf:md5,5a9b20b6ed25ef2ede4271156a83ce88", + "sample3_whatshap_stats.log:md5,dc9a34518331417b7b5811462b722791", + "sample3_whatshap_stats.tsv:md5,26a07a6db5f1e5bf7844aa0a8c190cb7", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -636,6 +648,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:54:05.409593728" + "timestamp": "2026-09-22T18:43:33.111991495" } } \ No newline at end of file From 25a0191be528d6454b27671e7ec068e9ed2450ec Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 23 Sep 2026 11:53:33 +0200 Subject: [PATCH 13/36] fix report clog --- tests/chm13.nf.test.snap | 15 ++++++-- workflows/lrsomatic.nf | 76 +++++++++++++++++++++++++++++++--------- 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/tests/chm13.nf.test.snap b/tests/chm13.nf.test.snap index a267d392..ad12da16 100644 --- a/tests/chm13.nf.test.snap +++ b/tests/chm13.nf.test.snap @@ -67,6 +67,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -101,6 +104,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -136,9 +145,9 @@ } ], "meta": { - "nf-test": "0.9.0", - "nextflow": "26.04.6" + "nf-test": "0.9.3", + "nextflow": "25.10.4" }, - "timestamp": "2026-09-21T11:19:37.348390574" + "timestamp": "2026-09-23T10:46:28.14524359" } } \ No newline at end of file diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index 37bd1c4c..9704b5ed 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -696,8 +696,10 @@ workflow LRSOMATIC { // ascat_tumoronly_ch: [meta, purityploidy, segments] // All ASCAT files per sample for the report module, which globs by suffix + // groupKey: release each sample on its own three emissions, not when ASCAT finishes for all ch_ascat_files = ASCAT.out.segments_raw .mix(ASCAT.out.purityploidy, ASCAT.out.png) + .map { meta, files -> [groupKey(meta, 3), files] } .groupTuple() .map { meta, files -> [meta, files.flatten()] } // ch_ascat_files: [meta, [file, file, ...]] @@ -1255,8 +1257,10 @@ workflow LRSOMATIC { ) // The WAKHAN outputs the report renders: ranked solutions, heatmap, per-solution plots + // groupKey: release each sample on its own three emissions ch_wakhan_files = WAKHAN.out.solutions_ranks .mix(WAKHAN.out.heatmap_html, WAKHAN.out.solution_dirs) + .map { meta, files -> [groupKey(meta, 3), files] } .groupTuple() .map { meta, files -> [meta, files.flatten()] } // solution_dirs contributes a list // ch_wakhan_files: [meta, [file_or_dir, ...]] @@ -1276,13 +1280,19 @@ workflow LRSOMATIC { .set { report_id_meta } // report_id_meta: [id, meta] - ch_somatic_vep_vcf - .map { meta, vcf -> [meta.id, vcf] } - .set { report_vep_ch } + // A skipped module leaves an empty leg, and remainder: true then defers every sample to + // channel close. One [] per sample keeps each leg matched so samples report independently. + def report_empty_slot = { -> report_id_meta.map { id, _meta -> [id, []] } } - ch_sv_vep_vcf - .map { meta, vcf -> [meta.id, vcf] } - .set { report_sv_vep_ch } + def report_vep_ch = params.skip_vep + ? report_empty_slot() + : ch_somatic_vep_vcf.map { meta, vcf -> [meta.id, vcf] } + // report_vep_ch: [id, vcf] + + def report_sv_vep_ch = params.skip_vep + ? report_empty_slot() + : ch_sv_vep_vcf.map { meta, vcf -> [meta.id, vcf] } + // report_sv_vep_ch: [id, vcf] SEVERUS.out.somatic_vcf .map { meta, vcf -> [meta.id, vcf] } @@ -1292,29 +1302,63 @@ workflow LRSOMATIC { .map { meta, vcf, _tbi -> [meta.id, vcf] } .set { report_somatic_ch } - ch_ascat_files - .map { meta, files -> [meta.id, files] } - .set { report_ascat_ch } + def report_ascat_ch = params.skip_ascat + ? report_empty_slot() + : ch_ascat_files.map { meta, files -> [meta.id, files] } + // report_ascat_ch: [id, [files]] + + def report_wakhan_ch = params.skip_wakhan + ? report_empty_slot() + : ch_wakhan_files.map { meta, files -> [meta.id, files] } + // report_wakhan_ch: [id, [files]] - ch_wakhan_files - .map { meta, files -> [meta.id, files] } - .set { report_wakhan_ch } + // One emission per sample per tool, none optional: mosdepth 2, cramino 1, samtools 2. + // Adding another per-sample emission to either mix below must bump this count. + def qc_files_per_sample = params.skip_qc + ? 0 + : (params.skip_mosdepth ? 0 : 2) + (params.skip_cramino ? 0 : 1) + (params.skip_bamstats ? 0 : 2) // Tumor-side QC, keyed by the sample id (= report id) + // groupKey: emit a sample's bundle on its own files; toString() restores a plain String key ch_mosdepth_summary .mix(ch_mosdepth_global, ch_cramino_post_txt, ch_bam_stats, ch_bam_flagstat) .filter { meta, _f -> meta.type == 'tumor' } - .map { meta, f -> [meta.id, f] } + .map { meta, f -> [groupKey(meta.id, qc_files_per_sample), f] } .groupTuple() - .set { report_qc_tumor_ch } + .map { key, files -> [key.toString(), files] } + .set { report_qc_tumor_grouped } + // report_qc_tumor_grouped: [id, [qc_file, ...]] // Normal-side QC (matched mode): a pair shares meta.id, so already keyed by the report id ch_mosdepth_summary .mix(ch_mosdepth_global, ch_cramino_post_txt, ch_bam_stats, ch_bam_flagstat) .filter { meta, _f -> meta.type == 'normal' } - .map { meta, f -> [meta.id, f] } + .map { meta, f -> [groupKey(meta.id, qc_files_per_sample), f] } .groupTuple() - .set { report_qc_normal_ch } + .map { key, files -> [key.toString(), files] } + .set { report_qc_normal_grouped } + // report_qc_normal_grouped: [id, [qc_file, ...]] -- paired samples only + + def report_qc_tumor_ch = qc_files_per_sample == 0 + ? report_empty_slot() + : report_qc_tumor_grouped + + // Normal-side QC covers paired samples only; meta.paired_data gives the tumor-only arm + // its [] up front instead of waiting out channel close for a match that never arrives. + report_id_meta + .branch { _id, meta -> + paired: meta.paired_data + tumor_only: true + } + .set { report_roster } + + def report_qc_normal_ch = qc_files_per_sample == 0 + ? report_empty_slot() + : report_roster.paired + .join(report_qc_normal_grouped) + .map { id, _meta, files -> [id, files] } + .mix(report_roster.tumor_only.map { id, _meta -> [id, []] }) + // report_qc_normal_ch: [id, [files] | []] -- full roster report_id_meta .join(report_vep_ch, remainder: true) From 0a3024ac5e1be231da08f7dd07d770703c26b687 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 23 Sep 2026 12:03:24 +0200 Subject: [PATCH 14/36] Fill in the PR number on this branch's changelog entries Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64799a46..934f4923 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now runs [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) v1.6.0 from `ghcr.io/ljwharbers/lrsomatic-report:1.6.0` (`oras://ghcr.io/ljwharbers/lrsomatic-report-sif:1.6.0` under Singularity/Apptainer) instead of the source vendored at `assets/lrsomatic_report`, which is deleted; updating the tool is now a tag bump. The image is `linux/amd64` only and **Conda is not supported for this step**: the module errors under `-profile conda`/`mamba`, `conda` is dropped from the nf-test CI matrix, and `--skip_report` gives a conda run (@ljwharbers). @@ -44,7 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3 `src/coverage/processing.py:121` averaged a BAF bin over a slice built from `.index()` lookups; because `.index()` searches by value the slice is empty whenever the position list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError`. The failure was deterministic, so retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3 `src/coverage/processing.py:121` averaged a BAF bin over a slice built from `.index()` lookups; because `.index()` searches by value the slice is empty whenever the position list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError`. The failure was deterministic, so retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). - [#193](https://github.com/IntGenomicsLab/lrsomatic/pull/193) - `--vep_eve https://evemodel.org/api/proteins/bulk/download/` was rejected at launch because the "needs preparing" check keyed on a `.zip` suffix; it now checks whether the value is already a prepared bgzipped file (@AmberVerhasselt). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs a patched modkit 0.6.4 ([ljwharbers/modkit@pacbio-conflict-fix](https://github.com/ljwharbers/modkit/tree/pacbio-conflict-fix)): `ghcr.io/ljwharbers/modkit:0.6.4-pacbiofix-6e0afa2` under Docker and `oras://ghcr.io/ljwharbers/modkit-sif:0.6.4-pacbiofix-6e0afa2` under Singularity/Apptainer. Stock modkit 0.4.3-0.6.4 dropped 32-65 % of reads from recent PacBio HiFi BAMs and returned empty `--cpg` pileups ([nanoporetech/modkit#612](https://github.com/nanoporetech/modkit/issues/612), fix proposed in [nanoporetech/modkit#720](https://github.com/nanoporetech/modkit/pull/720)), and ignored `--phased`/`--modified-bases` for PacBio BAMs with 6mA calls. The image is `linux/amd64` only and Conda is not supported (use `--skip_modkit` there); return to the biocontainer once a release includes the fix (@ljwharbers). From d5e610ebd30703777b1263ac968b6d74c05f0409 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 23 Sep 2026 13:03:18 +0200 Subject: [PATCH 15/36] Satisfy the pre-commit hooks: call the report slot closure explicitly Nextflow's strict syntax resolves `report_empty_slot()` against declared functions, not local closure variables, so it read as undefined. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- workflows/lrsomatic.nf | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c85f93e8..ce751704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#201](https://github.com/IntGenomicsLab/lrsomatic/pull/201) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now pull the fork image from Docker Hub: `oras://docker.io/ljwharbers/clairs-to-sif:0.5.1-verdict-chm13-c0687e8-flat` under Singularity/Apptainer and `docker.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8-flat` otherwise, instead of `ghcr.io/ljwharbers/clairs-to`. The `-cpu` SIF on ghcr failed with `PROTOCOL_ERROR` on slow links: ghcr redirects every blob download to an Azure URL that expires at the next 5-minute mark and resets a stream still open then, and Apptainer resumes neither an `oras://` nor a `docker://` download. Docker Hub's download URLs are valid for 50 minutes and only checked when the request starts. `-flat` is the same software copied into an empty image in a few layers (3.3 GB instead of 7 GB); the software and its outputs are unchanged. `docs/usage.md` describes `pullTimeout`, Docker Hub's anonymous pull limit, pre-pulling, and how to recover the remaining `oras://ghcr.io` SIFs resumably (@ljwharbers). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index 9704b5ed..acf98c0d 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -1285,12 +1285,12 @@ workflow LRSOMATIC { def report_empty_slot = { -> report_id_meta.map { id, _meta -> [id, []] } } def report_vep_ch = params.skip_vep - ? report_empty_slot() + ? report_empty_slot.call() : ch_somatic_vep_vcf.map { meta, vcf -> [meta.id, vcf] } // report_vep_ch: [id, vcf] def report_sv_vep_ch = params.skip_vep - ? report_empty_slot() + ? report_empty_slot.call() : ch_sv_vep_vcf.map { meta, vcf -> [meta.id, vcf] } // report_sv_vep_ch: [id, vcf] @@ -1303,12 +1303,12 @@ workflow LRSOMATIC { .set { report_somatic_ch } def report_ascat_ch = params.skip_ascat - ? report_empty_slot() + ? report_empty_slot.call() : ch_ascat_files.map { meta, files -> [meta.id, files] } // report_ascat_ch: [id, [files]] def report_wakhan_ch = params.skip_wakhan - ? report_empty_slot() + ? report_empty_slot.call() : ch_wakhan_files.map { meta, files -> [meta.id, files] } // report_wakhan_ch: [id, [files]] @@ -1340,7 +1340,7 @@ workflow LRSOMATIC { // report_qc_normal_grouped: [id, [qc_file, ...]] -- paired samples only def report_qc_tumor_ch = qc_files_per_sample == 0 - ? report_empty_slot() + ? report_empty_slot.call() : report_qc_tumor_grouped // Normal-side QC covers paired samples only; meta.paired_data gives the tumor-only arm @@ -1353,7 +1353,7 @@ workflow LRSOMATIC { .set { report_roster } def report_qc_normal_ch = qc_files_per_sample == 0 - ? report_empty_slot() + ? report_empty_slot.call() : report_roster.paired .join(report_qc_normal_grouped) .map { id, _meta, files -> [id, files] } From 13765572e3196a66b0a2a1d665b0b4bff9b5e933 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Thu, 24 Sep 2026 09:55:30 +0200 Subject: [PATCH 16/36] Add ecDNA analysis: CoRAL reconstruction and AmpliconClassifier Folds the standalone Downstream pipeline's amplicon analysis into lrsomatic as an ECDNA subworkflow, on by default for tumour samples with ASCAT calls. CoRAL seeds from ASCAT's cnvs.txt, which the nf-core ASCAT module already emits and nothing consumed. bin/ascat_to_coral_bed.py converts it to the headerless BED CoRAL wants, respelling contigs to match the reference: ASCAT writes "1" where the BAM may say "chr1", and CoRAL builds chromosome sizes from the BAM header. Zero-length and inverted segments are dropped, which CoRAL's own parser would otherwise reject. The two tools were previously local .sif files built from uncommitted working trees, so the images could not be rebuilt by anyone. They are now built from pinned commits of public forks: - docker.io/robertaforsyth/coral:3.0.0-chm13-847f3d4 - docker.io/robertaforsyth/ampliconclassifier:2.0.0-chm13-cdeaa63 with Dockerfiles under containers/. AmpliconClassifier moves from the v1.5.2 fork to upstream v2.0.0, the first release with official CoRAL support; the CHM13 patches applied unchanged. On B2096749 the ecDNA call is reproduced exactly, and two amplicons v1.5.2 reported as "No amp/Invalid" now resolve as Complex-non-cyclic and Linear. CoRAL defaults to the open-source SCIP solver rather than Gurobi. Across the 1102 solver logs from the previous cohort the largest model was 620 rows by 438 columns and the slowest solve took 0.49 s, so a licensed solver buys nothing at this scale. --coral_solver gurobi_direct with --gurobi_license remains available, and serialises reconstruction because WLS licences cap concurrent sessions. Also fixes three bugs carried over from the Downstream modules: publishDir paths were doubled by mkdir'ing a directory named after the publish target, the empty-seed filter staged files just to measure them, and errorStrategy 'ignore' hid a real reconstruction failure without surfacing it. Not included: an ecDNA section in LRSOMATICREPORT. That module renders through render_report.R inside ghcr.io/ljwharbers/lrsomatic-report, so displaying ecDNA results needs a change in ljwharbers/lrsomatic_report first. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 3 + CITATIONS.md | 8 + bin/ascat_to_coral_bed.py | 83 +++++++++ conf/igenomes.config | 9 + conf/modules.config | 83 +++++++++ conf/test.config | 2 + containers/ampliconclassifier.Dockerfile | 48 +++++ containers/coral.Dockerfile | 55 ++++++ docs/output.md | 48 +++++ docs/usage.md | 51 ++++++ modules/local/ampliconclassifier/main.nf | 55 ++++++ modules/local/ampliconclassifier/meta.yml | 133 ++++++++++++++ .../ampliconclassifier/tests/main.nf.test | 35 ++++ .../tests/main.nf.test.snap | 167 ++++++++++++++++++ modules/local/ascattocoralbed/environment.yml | 7 + modules/local/ascattocoralbed/main.nf | 38 ++++ modules/local/ascattocoralbed/meta.yml | 70 ++++++++ .../ascattocoralbed/tests/fixtures/chr.fai | 3 + .../tests/fixtures/sample1.cnvs.txt | 6 + .../local/ascattocoralbed/tests/main.nf.test | 94 ++++++++++ .../ascattocoralbed/tests/main.nf.test.snap | 60 +++++++ modules/local/coral/cycle/main.nf | 46 +++++ modules/local/coral/cycle/meta.yml | 74 ++++++++ modules/local/coral/cycle/tests/main.nf.test | 36 ++++ .../local/coral/cycle/tests/main.nf.test.snap | 67 +++++++ modules/local/coral/plot/main.nf | 42 +++++ modules/local/coral/plot/meta.yml | 76 ++++++++ modules/local/coral/plot/tests/main.nf.test | 39 ++++ .../local/coral/plot/tests/main.nf.test.snap | 49 +++++ modules/local/coral/reconstruct/main.nf | 53 ++++++ modules/local/coral/reconstruct/meta.yml | 112 ++++++++++++ .../coral/reconstruct/tests/main.nf.test | 42 +++++ .../coral/reconstruct/tests/main.nf.test.snap | 117 ++++++++++++ modules/local/coral/seed/main.nf | 42 +++++ modules/local/coral/seed/meta.yml | 81 +++++++++ modules/local/coral/seed/tests/main.nf.test | 41 +++++ .../local/coral/seed/tests/main.nf.test.snap | 43 +++++ nextflow.config | 17 ++ nextflow_schema.json | 126 ++++++++++++- subworkflows/local/ecdna.nf | 142 +++++++++++++++ subworkflows/local/prepare_aa_data_repo.nf | 43 +++++ subworkflows/local/tests/ecdna.nf.test | 121 +++++++++++++ tests/.nftignore | 1 + workflows/lrsomatic.nf | 62 +++++++ 44 files changed, 2523 insertions(+), 7 deletions(-) create mode 100755 bin/ascat_to_coral_bed.py create mode 100644 containers/ampliconclassifier.Dockerfile create mode 100644 containers/coral.Dockerfile create mode 100644 modules/local/ampliconclassifier/main.nf create mode 100644 modules/local/ampliconclassifier/meta.yml create mode 100644 modules/local/ampliconclassifier/tests/main.nf.test create mode 100644 modules/local/ampliconclassifier/tests/main.nf.test.snap create mode 100644 modules/local/ascattocoralbed/environment.yml create mode 100644 modules/local/ascattocoralbed/main.nf create mode 100644 modules/local/ascattocoralbed/meta.yml create mode 100644 modules/local/ascattocoralbed/tests/fixtures/chr.fai create mode 100644 modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt create mode 100644 modules/local/ascattocoralbed/tests/main.nf.test create mode 100644 modules/local/ascattocoralbed/tests/main.nf.test.snap create mode 100644 modules/local/coral/cycle/main.nf create mode 100644 modules/local/coral/cycle/meta.yml create mode 100644 modules/local/coral/cycle/tests/main.nf.test create mode 100644 modules/local/coral/cycle/tests/main.nf.test.snap create mode 100644 modules/local/coral/plot/main.nf create mode 100644 modules/local/coral/plot/meta.yml create mode 100644 modules/local/coral/plot/tests/main.nf.test create mode 100644 modules/local/coral/plot/tests/main.nf.test.snap create mode 100644 modules/local/coral/reconstruct/main.nf create mode 100644 modules/local/coral/reconstruct/meta.yml create mode 100644 modules/local/coral/reconstruct/tests/main.nf.test create mode 100644 modules/local/coral/reconstruct/tests/main.nf.test.snap create mode 100644 modules/local/coral/seed/main.nf create mode 100644 modules/local/coral/seed/meta.yml create mode 100644 modules/local/coral/seed/tests/main.nf.test create mode 100644 modules/local/coral/seed/tests/main.nf.test.snap create mode 100644 subworkflows/local/ecdna.nf create mode 100644 subworkflows/local/prepare_aa_data_repo.nf create mode 100644 subworkflows/local/tests/ecdna.nf.test diff --git a/CHANGELOG.md b/CHANGELOG.md index ce751704..1cb50a6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Added` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Added ecDNA and focal amplification analysis: CoRAL reconstructs amplicon structures from each tumour BAM, seeded by ASCAT's copy-number calls, and AmpliconClassifier labels each amplicon as ecDNA, BFB or linear. Runs by default on tumour samples with ASCAT calls; turn it off with `--skip_coral`, or keep reconstruction without classification with `--skip_ampliconclassifier`. CoRAL's optimisation defaults to the open-source SCIP solver rather than Gurobi: across the 1102 solver logs from the previous standalone cohort the largest model was 620 rows by 438 columns and the slowest solve took 0.49 s, so a licensed solver buys nothing at this scale, though `--coral_solver gurobi_direct` with `--gurobi_license` remains available. AmpliconClassifier's AmpliconArchitect data repository is downloaded automatically for GRCh38 and must be supplied with `--aa_data_repo` for CHM13, which has no published repository (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Added `ascat_to_coral_bed.py` and the `ASCAT_TO_CORAL_BED` module, converting ASCAT's `cnvs.txt` into the headerless BED CoRAL seeds from. Contigs are respelled to match the reference, since ASCAT writes `1` where the BAM may say `chr1` and CoRAL builds its chromosome sizes from the BAM header; zero-length and inverted segments are dropped, which CoRAL's own parser would otherwise reject (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Added stub nf-tests for the `ECDNA` subworkflow and `ASCAT_TO_CORAL_BED` (tag `small`), covering the empty-seed branch, the opt-in cycle re-extraction and the `--skip_ampliconclassifier` path (@robert-a-forsyth). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - Added CHM13 support for ClairS-TO's Verdict module, which tags tumour-only calls as germline, somatic or subclonal somatic; its resources were GRCh38-only, so on CHM13 germline variants leaked into `somatic.vcf.gz`. With `--genome CHM13 --skip_ascat` the pipeline builds a CHM13 resource set from the ASCAT files it already downloads and passes it as `--cna_resource_dir`; a prepared directory can be given with `--clairsto_cna_resources` (validated at launch). Without `--skip_ascat` tagging comes from ASCAT's own tables instead (next entry) (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - Added `CLAIRSTO_VERDICT_TAG`: when ASCAT is in the run, Verdict's germline tagging is computed from ASCAT's purity, ploidy and segments instead of Verdict's own estimate, so `CLAIRSTO` runs with `--disable_verdict` and ASCAT runs before small variant calling. Output names are unchanged. The tables the tags were computed from are published as `_Tumor_Purity_Ploidy.txt` and `_Tumor_CNA.txt`, also on `--skip_ascat` runs (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - Added a stub nf-test for `TUMORONLY_SMALLVAR` covering both germline tagging paths (tag `small`) (@ljwharbers). diff --git a/CITATIONS.md b/CITATIONS.md index a1403be7..cbb41940 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -18,6 +18,10 @@ > Cheng J, Novati G, Pan J, Bycroft C, Žemgulytė A, Applebaum T, Pritzel A, Wong LH, Zielinski M, Sargeant T, Schneider RG, Senior AW, Jumper J, Hassabis D, Kohli P, Avsec Ž. Accurate proteome-wide missense variant effect prediction with AlphaMissense. Science. 2023 Sep 22;381(6664):eadg7492. doi: 10.1126/science.adg7492. +- [AmpliconClassifier](https://doi.org/10.1038/s41586-023-05937-5) + + > Luebeck, J., Ng, A.W.T., Galipeau, P.C. et al. Extrachromosomal DNA in the cancerous transformation of Barrett's oesophagus. Nature 616, 798-805 (2023). https://doi.org/10.1038/s41586-023-05937-5 + - [ASCAT](https://pubmed.ncbi.nlm.nih.gov/20837533/) > Van Loo P, Nordgard SH, Lingjærde OC, Russnes HG, Rye IH, Sun W, Weigman VJ, Marynen P, Zetterberg A, Naume B, Perou CM, Børresen-Dale AL, Kristensen VN. Allele-specific copy number analysis of tumors. Proc Natl Acad Sci U S A. 2010 Sep 28;107(39):16910-5. doi: 10.1073/pnas.1009843107. Epub 2010 Sep 13. PubMed PMID: 20837533; PubMed Central PMCID: PMC2947907. @@ -46,6 +50,10 @@ > Landrum MJ, Lee JM, Benson M, Brown GR, Chao C, Chitipiralla S, Gu B, Hart J, Hoffman D, Jang W, Karapetyan K, Katz K, Liu C, Maddipatla Z, Malheiro A, McDaniel K, Ovetsky M, Riley G, Zhou G, Holmes JB, Kattman BL, Maglott DR. ClinVar: improving access to variant interpretations and supporting evidence. Nucleic Acids Res. 2018 Jan 4;46(D1):D1062-D1067. doi: 10.1093/nar/gkx1153. +- [CoRAL](https://doi.org/10.1101/gr.279131.124) + + > Zhu, K., Jones, M.G., Luebeck, J. et al. CoRAL accurately resolves extrachromosomal DNA genome structures with long-read sequencing. Genome Res. 34, 1344-1354 (2024). https://doi.org/10.1101/gr.279131.124 + - [cramino](https://github.com/wdecoster/cramino) > De Coster W. cramino: A fast and simple tool for quality control of long read sequencing data [Software]. GitHub. https://github.com/wdecoster/cramino diff --git a/bin/ascat_to_coral_bed.py b/bin/ascat_to_coral_bed.py new file mode 100755 index 00000000..2e7e7d21 --- /dev/null +++ b/bin/ascat_to_coral_bed.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Convert ASCAT's cnvs.txt into the headerless BED CoRAL's --cn-seg expects. + +CoRAL reads total copy number from the last column and builds chromosome sizes from +the BAM header, so contigs are respelled to match the reference (chr1 vs 1). +""" +import argparse +import sys + +REQUIRED = ('chr', 'startpos', 'endpos', 'nMajor', 'nMinor') + + +def fai_contigs(path): + if not path: + return [] + with open(path) as fp: + return [line.split('\t', 1)[0] for line in fp if line.strip()] + + +def spell_like_reference(chrom, contigs): + """ASCAT's '1' becomes 'chr1' when that is what the reference calls it, and the reverse.""" + if not contigs or chrom in contigs: + return chrom + if chrom.startswith('chr') and chrom[3:] in contigs: + return chrom[3:] + if 'chr' + chrom in contigs: + return 'chr' + chrom + return chrom + + +def sort_key(chrom): + """Natural contig order: 1-22, then X, Y, then anything else alphabetically.""" + bare = chrom[3:] if chrom.startswith('chr') else chrom + if bare.isdigit(): + return (0, int(bare), '') + if bare in ('X', 'Y'): + return (1, 'XY'.index(bare), '') + return (2, 0, bare) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--cnvs', required=True, help="ASCAT .cnvs.txt") + parser.add_argument('--fai', help="Reference .fai, to respell contigs to match the BAM") + parser.add_argument('--output', required=True, help="BED4 written for CoRAL --cn-seg") + args = parser.parse_args() + + contigs = fai_contigs(args.fai) + + with open(args.cnvs) as fp: + header = fp.readline().rstrip('\n').split('\t') + missing = [c for c in REQUIRED if c not in header] + if missing: + sys.exit(f"ERROR: {args.cnvs} is missing required columns: {', '.join(missing)}") + idx = {c: header.index(c) for c in REQUIRED} + + rows, dropped = [], 0 + for line in fp: + if not line.strip(): + continue + fields = line.rstrip('\n').split('\t') + start, end = int(fields[idx['startpos']]), int(fields[idx['endpos']]) + # CoRAL's segment parser rejects these, so drop them here with a count + if start >= end: + dropped += 1 + continue + total_cn = round(float(fields[idx['nMajor']])) + round(float(fields[idx['nMinor']])) + rows.append((spell_like_reference(fields[idx['chr']], contigs), start, end, total_cn)) + + if dropped: + print(f"WARNING: dropped {dropped} segments with start >= end", file=sys.stderr) + + rows.sort(key=lambda r: (sort_key(r[0]), r[1])) + with open(args.output, 'w') as out: + for chrom, start, end, total_cn in rows: + out.write(f"{chrom}\t{start}\t{end}\t{total_cn}\n") + + if not rows: + print(f"WARNING: {args.output} is empty; CoRAL will find no seeds", file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/conf/igenomes.config b/conf/igenomes.config index 71ff9f59..bdeea004 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -24,6 +24,11 @@ params.genomes = [ vep_species : "homo_sapiens", savana_contigs : "https://raw.githubusercontent.com/cortes-ciriano-lab/savana/main/example/contigs.chr.hg38.txt", savana_g1000_vcf : "1000g_hg38", + coral_ref : "hg38", + ac_ref : "GRCh38", + // Plain build, not GRCh38_indexed: the extra BWA index is for AmpliconArchitect's + // alignment step, which AmpliconClassifier never runs. + aa_data_repo_url : "https://refs.ampliconrepository.org/data/module_support_files/AmpliconArchitect/GRCh38.tar.gz", vep_alphamissense : "https://storage.googleapis.com/dm_alphamissense/AlphaMissense_hg38.tsv.gz", vep_alphamissense_tbi : "https://g-608c0c.273595.03c0.data.globus.org/VEP_plugins/AlphaMissense_hg38.tsv.gz.tbi", // A dated release rather than the rolling vcf_GRCh38/clinvar.vcf.gz, whose VCF and @@ -54,6 +59,10 @@ params.genomes = [ vep_species : "homo_sapiens_gca009914755v4", savana_contigs : "https://raw.githubusercontent.com/IntGenomicsLab/test-datasets/main/references/savana/contigs.chr.chm13.txt", savana_g1000_vcf : "1000g_t2t", + coral_ref : "t2t", + ac_ref : "CHM13", + // No published AA data repo for CHM13; supply one with --aa_data_repo + aa_data_repo_url : null, vep_alphamissense_aa : "https://g-608c0c.273595.03c0.data.globus.org/VEP_plugins/alphamissense_protein_v2023_uniprot-2026_03.tsv.gz", vep_alphamissense_aa_tbi : "https://g-608c0c.273595.03c0.data.globus.org/VEP_plugins/alphamissense_protein_v2023_uniprot-2026_03.tsv.gz.tbi", // Pinned to a release rather than current_variation/, which moves at every Ensembl release diff --git a/conf/modules.config b/conf/modules.config index 9372bad1..043ba201 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -665,6 +665,89 @@ process { ] } + withName: '.*:ASCAT_TO_CORAL_BED' { + publishDir = [ + path: { "${params.outdir}/${meta.id}/ecdna" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: '.*:CORAL_.*' { + // Gurobi licences are user-supplied and mounted; SCIP, the default, needs nothing. + containerOptions = { + params.coral_solver == 'gurobi_direct' && params.gurobi_license + ? "--bind ${params.gurobi_license}:/opt/gurobi/gurobi.lic:ro --env GRB_LICENSE_FILE=/opt/gurobi/gurobi.lic" + : null + } + publishDir = [ + path: { "${params.outdir}/${meta.id}/ecdna/coral" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: '.*:CORAL_SEED' { + ext.args = { + [ + "--gain ${params.coral_gain}", + "--min-seed-size ${params.coral_min_seed_size}", + "--max-seg-gap ${params.coral_max_seg_gap}", + ].join(' ') + } + } + + withName: '.*:CORAL_RECONSTRUCT' { + // Gurobi WLS licences cap concurrent solver sessions, so serialise under Gurobi. + // 1000 stands in for "no extra limit"; the executor's queueSize already bounds this. + maxForks = params.coral_solver == 'gurobi_direct' ? 1 : 1000 + ext.args = { + [ + "--solver ${params.coral_solver}", + "--solver-threads ${params.coral_solver_threads}", + "--solver-time-limit ${params.coral_solver_time_limit}", + "--global-time-limit ${params.coral_global_time_limit}", + "--min-bp-support ${params.coral_min_bp_support}", + ].join(' ') + } + } + + withName: '.*:CORAL_CYCLE' { + // Gurobi WLS licences cap concurrent solver sessions, so serialise under Gurobi. + // 1000 stands in for "no extra limit"; the executor's queueSize already bounds this. + maxForks = params.coral_solver == 'gurobi_direct' ? 1 : 1000 + ext.args = { + [ + "--solver ${params.coral_solver}", + "--threads ${params.coral_solver_threads}", + "--solver-time-limit ${params.coral_solver_time_limit}", + "--global-time-limit ${params.coral_global_time_limit}", + "--alpha ${params.coral_cycle_decomp_alpha}", + ].join(' ') + } + publishDir = [ + path: { "${params.outdir}/${meta.id}/ecdna/coral/cycles" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: '.*:CORAL_PLOT' { + publishDir = [ + path: { "${params.outdir}/${meta.id}/ecdna/coral/plots" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: '.*:AMPLICONCLASSIFIER' { + publishDir = [ + path: { "${params.outdir}/${meta.id}/ecdna/amplicon_classifier" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName : '.*:WAKHAN' { ext.args = { [ diff --git a/conf/test.config b/conf/test.config index 1829a02e..3de55ac9 100644 --- a/conf/test.config +++ b/conf/test.config @@ -69,6 +69,8 @@ params { vep_species = "caenorhabditis_elegans" skip_wakhan = true skip_ascat = true + // CoRAL seeds from ASCAT, which is skipped here, and the chr19 slice has no amplification + skip_coral = true skip_modkit = true savana_chromosomes = "19" // SAVANA's het-SNP coverage/mapq floors (--allele_min_reads default 10, --allele_mapq diff --git a/containers/ampliconclassifier.Dockerfile b/containers/ampliconclassifier.Dockerfile new file mode 100644 index 00000000..7fa4545d --- /dev/null +++ b/containers/ampliconclassifier.Dockerfile @@ -0,0 +1,48 @@ +# AmpliconClassifier v2.0.0 with T2T-CHM13 support, from a pinned fork commit. +# +# v2.0.0 is the first release with official CoRAL support. Installed with +# `pip install .` rather than a PATH symlink so the pinned BFBArchitect +# dependency, the console entry points and the bundled CHM13 lncRNA GFF3 +# resource all land correctly. +FROM docker.io/library/python:3.11-slim + +ARG AC_REPO=https://github.com/robert-a-forsyth/AmpliconClassifier.git +ARG AC_REF=cdeaa63 +ARG AC_VERSION=2.0.0 + +LABEL org.opencontainers.image.title="AmpliconClassifier" \ + org.opencontainers.image.description="AmpliconClassifier 2.0.0, T2T-CHM13 fork" \ + org.opencontainers.image.source="${AC_REPO}" \ + org.opencontainers.image.revision="${AC_REF}" \ + org.opencontainers.image.version="${AC_VERSION}" \ + org.opencontainers.image.licenses="BSD-2-Clause" + +ENV LANG=C.UTF-8 \ + PYTHONDONTWRITEBYTECODE=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc g++ git procps ca-certificates \ + zlib1g-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libssl-dev \ + coinor-cbc \ + && rm -rf /var/lib/apt/lists/* + +# BFBArchitect==1.0.1 comes in as a pinned dependency and pulls PuLP, CNVkit, +# pysam and matplotlib. gurobipy arrives too but needs no licence: BFBArchitect +# falls back Gurobi -> MOSEK -> CBC, so the image runs licence-free. +RUN pip install --no-cache-dir --upgrade pip \ + && git clone "${AC_REPO}" /opt/AmpliconClassifier \ + && git -C /opt/AmpliconClassifier checkout "${AC_REF}" \ + && pip install --no-cache-dir /opt/AmpliconClassifier \ + && rm -rf /opt/AmpliconClassifier/.git + +# Mount point only. The AA data repo is ~1.1 GB of third-party-derived +# annotation with no stated licence, so it is staged at runtime. +RUN mkdir -p /opt/data_repo +ENV AA_DATA_REPO=/opt/data_repo + +RUN amplicon_classifier.py --version \ + && python -c "import ampclasslib.ac_util as u; \ +p = u.get_ncrna_file_loc('CHM13'); \ +import os; assert os.path.exists(p), p; print('CHM13 lncRNA resource OK')" + +CMD ["amplicon_classifier.py", "--help"] diff --git a/containers/coral.Dockerfile b/containers/coral.Dockerfile new file mode 100644 index 00000000..612e6ad5 --- /dev/null +++ b/containers/coral.Dockerfile @@ -0,0 +1,55 @@ +# CoRAL with T2T-CHM13 support, built from a pinned commit of the fork. +# +# micromamba rather than python:slim because SCIP -- the open-source global MINLP +# solver CoRAL needs for its non-convex MIQCP -- is only packaged on conda-forge. +# Everything is installed into the base env and PATH is set explicitly, so the +# image works without shell activation (Nextflow/Singularity run no entrypoint). +FROM docker.io/mambaorg/micromamba:2.0.5-debian12-slim + +ARG CORAL_REPO=https://github.com/robert-a-forsyth/CoRAL.git +ARG CORAL_REF=847f3d4 +ARG CORAL_VERSION=3.0.0 + +LABEL org.opencontainers.image.title="CoRAL" \ + org.opencontainers.image.description="CoRAL amplicon reconstruction, T2T-CHM13 fork" \ + org.opencontainers.image.source="${CORAL_REPO}" \ + org.opencontainers.image.revision="${CORAL_REF}" \ + org.opencontainers.image.version="${CORAL_VERSION}" \ + org.opencontainers.image.licenses="BSD-3-Clause" + +USER root +ENV PATH=/opt/conda/bin:$PATH \ + LANG=C.UTF-8 \ + PYTHONDONTWRITEBYTECODE=1 + +# Build toolchain and the headers pysam and cvxopt fail without (upstream README) +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc g++ git make pkg-config curl procps ca-certificates \ + libhdf5-dev libbz2-dev liblzma-dev zlib1g-dev \ + libcurl4-openssl-dev libssl-dev libffi-dev \ + libsuitesparse-dev \ + && rm -rf /var/lib/apt/lists/* + +# scip provides bin/scip, whose built-in AMPL reader is what Pyomo's SCIPAMPL +# plugin shells out to. pyscipopt is deliberately not installed -- it is unused. +# htslib is not needed: pysam installs from a manylinux wheel. +RUN micromamba install -y -n base -c conda-forge python=3.12 scip=10.1.0 \ + && micromamba clean --all --yes + +# CPU-only torch first, or pomegranate/cnvkit pull the multi-GB CUDA wheel +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu + +RUN git clone "${CORAL_REPO}" /opt/CoRAL \ + && git -C /opt/CoRAL checkout "${CORAL_REF}" \ + && pip install --no-cache-dir /opt/CoRAL \ + && rm -rf /opt/CoRAL/.git + +# Mount point only. Gurobi licences are user-supplied and never baked in. +RUN mkdir -p /opt/gurobi +ENV GRB_LICENSE_FILE=/opt/gurobi/gurobi.lic + +# No `| head`: SCIP dies on SIGPIPE when the reader closes early (exit 141) +RUN coral --help > /dev/null && scip -v > /dev/null + +CMD ["coral", "--help"] diff --git a/docs/output.md b/docs/output.md index 495ab48c..6fa881ac 100644 --- a/docs/output.md +++ b/docs/output.md @@ -358,6 +358,54 @@ The germline/somatic split comes from a panel of normals and from ClairS-TO's Ve | `read_qual.txt` | file containing quality statistics about identified segements | | `severus.log` | log file | +#### `ecdna` + +Extrachromosomal DNA and focal amplification. CoRAL reconstructs amplicon structures from the tumour +BAM, seeded by ASCAT's copy-number segments, and AmpliconClassifier labels each reconstructed +amplicon (ecDNA, BFB, linear, and so on). Runs on every tumour sample with ASCAT calls; disable with +`--skip_coral`, or keep reconstruction and drop classification with `--skip_ampliconclassifier`. + +A sample with no segment above `--coral_gain` produces an empty seed BED and is skipped with a log +message rather than failing. CoRAL defaults to the open-source SCIP solver; `--coral_solver +gurobi_direct` is faster but needs `--gurobi_license`. AmpliconClassifier needs an AmpliconArchitect +data repository, downloaded automatically for GRCh38 and supplied with `--aa_data_repo` for CHM13. + +``` +├── ecdna +│ ├── sample_coral_cn.bed +│ ├── coral +│ │ ├── sample_CNV_SEEDS.bed +│ │ ├── reconstruct +│ │ │ ├── sample_amplicon1_graph.txt +│ │ │ ├── sample_amplicon1_cycles.txt +│ │ │ ├── sample_summary.txt +│ │ │ └── sample_reconstruct.log +│ │ └── plots +│ │ ├── sample_amplicon1_graph.png +│ │ └── sample_amplicon1_cycles.png +│ └── amplicon_classifier +│ ├── sample_amplicon_classification_profiles.tsv +│ ├── sample_gene_list.tsv +│ ├── sample_ecDNA_counts.tsv +│ ├── sample_result_table.tsv +│ └── sample_classification_bed_files/ +``` + +| File | Description | +| ----------------------------------------------- | ------------------------------------------------------------------------------ | +| `sample_coral_cn.bed` | ASCAT's copy number as the BED CoRAL seeds from | +| `sample_CNV_SEEDS.bed` | Amplified intervals above `--coral_gain`; empty means no amplicons | +| `sample_amplicon_graph.txt` | Breakpoint graph per amplicon, in AmpliconArchitect format | +| `sample_amplicon_cycles.txt` | Decomposed cycles and paths per amplicon | +| `sample_summary.txt` | Per-run amplicon summary; written even when no amplicon is found | +| `sample_reconstruct.log` | CoRAL reconstruction log, including solver output | +| `sample_amplicon_{graph,cycles}.png` | Per-amplicon copy-number and cycle plots | +| `sample_amplicon_classification_profiles.tsv` | The headline call per amplicon: ecDNA+, BFB+, decomposition class | +| `sample_gene_list.tsv` | Genes intersecting each classified amplicon | +| `sample_ecDNA_counts.tsv` | Number of distinct ecDNA species detected | +| `sample_result_table.tsv` | Combined per-sample table, the format AmpliconRepository ingests | +| `sample_classification_bed_files/` | Per-feature BED intervals for each classified amplicon | + #### `savana` SAVANA structural variant and copy-number calling. Runs alongside Severus/ASCAT rather than replacing diff --git a/docs/usage.md b/docs/usage.md index 00a0c28e..9715fb32 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -680,6 +680,57 @@ Two of these predictors get there anyway, because they score _proteins_ rather t > for `--custom` files, but a name it cannot map annotates nothing rather than raising an error, so > confirm that `CLNSIG` values appear in an annotated VCF before trusting them. +## ecDNA and focal amplification + +CoRAL reconstructs amplicon structures and AmpliconClassifier labels them. Both run by default on +every tumour sample that has ASCAT calls. CoRAL seeds from ASCAT's copy number, so `--skip_ascat` +together with CoRAL is rejected at launch: use `--skip_coral` as well, or keep ASCAT. + +### Solver + +CoRAL's cycle decomposition is a non-convex mixed-integer quadratically-constrained problem. The +pipeline defaults to `--coral_solver scip`, which is open-source, shipped in the image and needs no +licence. Gurobi is available with: + +```bash +--coral_solver gurobi_direct --gurobi_license /path/to/gurobi.lic +``` + +The licence is bind-mounted into the CoRAL tasks; it is never baked into the image. Under Gurobi the +reconstruction steps are serialised (`maxForks = 1`), because a Web License Service licence caps +concurrent solver sessions. + +Gurobi is faster, but on the models this pipeline produces that has not mattered: across 1102 solver +logs from the earlier standalone cohort, the largest model was 620 rows by 438 columns and the +slowest single solve took 0.49 s, with no model reaching the time limit. Prefer the default unless +you have measured a reason not to. + +### AmpliconClassifier reference data + +AmpliconClassifier reads an AmpliconArchitect data repository at runtime. For `--genome GRCh38` it is +downloaded automatically (about 1.1 GB) and needs no configuration. For `--genome CHM13` there is no +published repository, so one must be supplied: + +```bash +--genome CHM13 --aa_data_repo /path/to/AA_DATA_REPO +``` + +The directory must contain a `CHM13/` subdirectory. Alternatively run reconstruction alone with +`--skip_ampliconclassifier`, which needs no reference data. + +### Tuning + +`--coral_gain` (default 6.0) sets the total copy number a segment must reach to seed an amplicon. A +sample with nothing above it produces an empty seed file and is skipped with a log message rather +than failing. + +`--coral_min_bp_support` defaults to 1.75, the value validated on this lab's cohort. CoRAL's own +documentation recommends a considerably higher value for WGS, around 10.0; raise it if you see +spurious breakpoints. + +`--coral_run_cycle` re-extracts cycles with `coral cycle_all` after reconstruction and classifies +those instead of the originals. It is off by default. + ## Core Nextflow arguments > [!NOTE] diff --git a/modules/local/ampliconclassifier/main.nf b/modules/local/ampliconclassifier/main.nf new file mode 100644 index 00000000..6060babe --- /dev/null +++ b/modules/local/ampliconclassifier/main.nf @@ -0,0 +1,55 @@ +process AMPLICONCLASSIFIER { + tag "$meta.id" + label 'process_medium' + + // No conda: bioconda's `ampliconclassifier` recipe is stuck at 0.4.14 (2023) and + // predates CoRAL support; this image carries the CHM13 fork of v2.0.0. See meta.yml + container "docker.io/robertaforsyth/ampliconclassifier:2.0.0-chm13-cdeaa63" + + input: + tuple val(meta), path(reconstruction) + tuple val(meta2), path(data_repo, stageAs: 'aa_data_repo') + val(ac_ref) + + output: + tuple val(meta), path("*_amplicon_classification_profiles.tsv"), emit: classification, optional: true + tuple val(meta), path("*_gene_list.tsv"), emit: gene_list, optional: true + tuple val(meta), path("*_ecDNA_counts.tsv"), emit: ecdna_counts, optional: true + tuple val(meta), path("*_result_table.tsv"), emit: result_table, optional: true + tuple val(meta), path("*_classification_bed_files", type: 'dir'), emit: bed_files, optional: true + tuple val(meta), path("*_SV_summaries", type: 'dir'), emit: sv_summaries, optional: true + tuple val(meta), path("*_annotated_cycles_files", type: 'dir'), emit: annotated_cycles, optional: true + tuple val(meta), path("*.log"), emit: log, optional: true + tuple val("${task.process}"), val('ampliconclassifier'), eval("amplicon_classifier.py --version"), topic: versions, emit: versions_ampliconclassifier + + when: + task.ext.when == null || task.ext.when + + script: + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "AMPLICONCLASSIFIER does not support Conda. Please use Docker / Singularity / Apptainer instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + export AA_DATA_REPO=\$(readlink -f aa_data_repo) + + amplicon_classifier.py \\ + --ref ${ac_ref} \\ + --AA_results ${reconstruction} \\ + -o ${prefix} \\ + ${args} \\ + > ${prefix}_classifier.log 2>&1 + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}_amplicon_classification_profiles.tsv + touch ${prefix}_gene_list.tsv + touch ${prefix}_ecDNA_counts.tsv + touch ${prefix}_result_table.tsv + touch ${prefix}_classifier.log + mkdir -p ${prefix}_classification_bed_files ${prefix}_SV_summaries ${prefix}_annotated_cycles_files + """ +} diff --git a/modules/local/ampliconclassifier/meta.yml b/modules/local/ampliconclassifier/meta.yml new file mode 100644 index 00000000..7b4f7b66 --- /dev/null +++ b/modules/local/ampliconclassifier/meta.yml @@ -0,0 +1,133 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "ampliconclassifier" +description: Classify reconstructed amplicons as ecDNA, BFB, linear or no amplification. +keywords: + - ecdna + - amplicon + - classification + - bfb +tools: + - "ampliconclassifier": + description: "Classifies focal amplifications from AmpliconArchitect or CoRAL output." + homepage: "https://github.com/AmpliconSuite/AmpliconClassifier" + documentation: "https://github.com/AmpliconSuite/AmpliconClassifier" + doi: "10.1038/s41586-023-05937-5" + licence: ["BSD-2-Clause"] + identifier: "" + +## No environment.yml: bioconda's `ampliconclassifier` recipe is stuck at 0.4.14 (2023) and +## predates CoRAL support, which arrived in v2.0.0. Built from a fork of v2.0.0 adding +## T2T-CHM13 support (https://github.com/robert-a-forsyth/AmpliconClassifier, branch +## chm13-support). See containers/ampliconclassifier.Dockerfile. + +input: + - - meta: + type: map + description: Groovy Map containing sample information + - reconstruction: + type: directory + description: | + A CoRAL output directory holding matching *_graph.txt, *_cycles.txt and + *_summary.txt files. AmpliconClassifier pairs them by shared prefix. + - - meta2: + type: map + description: Groovy Map for the data repository + - data_repo: + type: directory + description: | + AmpliconArchitect data repository, exported as $AA_DATA_REPO. Staged under a + fixed name so the tool never writes into the shared reference directory. + - - ac_ref: + type: string + description: Reference genome name, one of hg19, GRCh37, GRCh38, mm10, CHM13 + +output: + classification: + - meta: + type: map + description: Groovy Map containing sample information + - "*_amplicon_classification_profiles.tsv": + type: file + description: Per-amplicon class, ecDNA+ and BFB+ calls + pattern: "*_amplicon_classification_profiles.tsv" + gene_list: + - meta: + type: map + description: Groovy Map containing sample information + - "*_gene_list.tsv": + type: file + description: Genes intersecting each classified amplicon + pattern: "*_gene_list.tsv" + ecdna_counts: + - meta: + type: map + description: Groovy Map containing sample information + - "*_ecDNA_counts.tsv": + type: file + description: Number of distinct ecDNA species detected + pattern: "*_ecDNA_counts.tsv" + result_table: + - meta: + type: map + description: Groovy Map containing sample information + - "*_result_table.tsv": + type: file + description: Combined per-sample table, the AmpliconRepository ingest format + pattern: "*_result_table.tsv" + bed_files: + - meta: + type: map + description: Groovy Map containing sample information + - "*_classification_bed_files": + type: directory + description: Per-feature BED intervals for each classified amplicon + sv_summaries: + - meta: + type: map + description: Groovy Map containing sample information + - "*_SV_summaries": + type: directory + description: Per-amplicon structural variant summaries + annotated_cycles: + - meta: + type: map + description: Groovy Map containing sample information + - "*_annotated_cycles_files": + type: directory + description: Cycles files annotated with classification + log: + - meta: + type: map + description: Groovy Map containing sample information + - "*.log": + type: file + description: Classifier log + pattern: "*.log" + versions_ampliconclassifier: + - - ${task.process}: + type: string + description: Process name + - ampliconclassifier: + type: string + description: Tool name + - version: + type: string + description: Tool version + +topics: + versions: + - - ${task.process}: + type: string + description: Process name + - ampliconclassifier: + type: string + description: Tool name + - version: + type: string + description: Tool version + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/ampliconclassifier/tests/main.nf.test b/modules/local/ampliconclassifier/tests/main.nf.test new file mode 100644 index 00000000..c6e26e88 --- /dev/null +++ b/modules/local/ampliconclassifier/tests/main.nf.test @@ -0,0 +1,35 @@ +nextflow_process { + + name "Test Process AMPLICONCLASSIFIER" + script "../main.nf" + process "AMPLICONCLASSIFIER" + + tag "modules" + tag "modules_local" + tag "ampliconclassifier" + tag "small" + + // Stub only: the real tool needs a BAM, a solver and, for the classifier, a ~1 GB data repo. + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [ id:'sample1' ], file('reconstruct') ] + input[1] = [ [:], file('aa_data_repo') ] + input[2] = 'CHM13' + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.classification[0][1].toString().endsWith('_amplicon_classification_profiles.tsv') }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/ampliconclassifier/tests/main.nf.test.snap b/modules/local/ampliconclassifier/tests/main.nf.test.snap new file mode 100644 index 00000000..39b9dce2 --- /dev/null +++ b/modules/local/ampliconclassifier/tests/main.nf.test.snap @@ -0,0 +1,167 @@ +{ + "stub": { + "content": [ + { + "0": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon_classification_profiles.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "sample1" + }, + "sample1_gene_list.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + [ + { + "id": "sample1" + }, + "sample1_ecDNA_counts.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "sample1" + }, + "sample1_result_table.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + [ + { + "id": "sample1" + }, + [ + + ] + ] + ], + "5": [ + [ + { + "id": "sample1" + }, + [ + + ] + ] + ], + "6": [ + [ + { + "id": "sample1" + }, + [ + + ] + ] + ], + "7": [ + [ + { + "id": "sample1" + }, + "sample1_classifier.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "8": [ + [ + "AMPLICONCLASSIFIER", + "ampliconclassifier", + "2.0.0" + ] + ], + "annotated_cycles": [ + [ + { + "id": "sample1" + }, + [ + + ] + ] + ], + "bed_files": [ + [ + { + "id": "sample1" + }, + [ + + ] + ] + ], + "classification": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon_classification_profiles.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "ecdna_counts": [ + [ + { + "id": "sample1" + }, + "sample1_ecDNA_counts.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "gene_list": [ + [ + { + "id": "sample1" + }, + "sample1_gene_list.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "log": [ + [ + { + "id": "sample1" + }, + "sample1_classifier.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "result_table": [ + [ + { + "id": "sample1" + }, + "sample1_result_table.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "sv_summaries": [ + [ + { + "id": "sample1" + }, + [ + + ] + ] + ], + "versions_ampliconclassifier": [ + [ + "AMPLICONCLASSIFIER", + "ampliconclassifier", + "2.0.0" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:50:03.780875635" + } +} \ No newline at end of file diff --git a/modules/local/ascattocoralbed/environment.yml b/modules/local/ascattocoralbed/environment.yml new file mode 100644 index 00000000..c9b822b2 --- /dev/null +++ b/modules/local/ascattocoralbed/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "conda-forge::python=3.12" diff --git a/modules/local/ascattocoralbed/main.nf b/modules/local/ascattocoralbed/main.nf new file mode 100644 index 00000000..ee75800a --- /dev/null +++ b/modules/local/ascattocoralbed/main.nf @@ -0,0 +1,38 @@ +process ASCAT_TO_CORAL_BED { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/python:3.12': + 'biocontainers/python:3.12' }" + + input: + tuple val(meta), path(cnvs) + tuple val(meta2), path(fai) + + output: + tuple val(meta), path("*_coral_cn.bed"), emit: bed + tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //'"), topic: versions, emit: versions_python + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def fai_arg = fai ? "--fai ${fai}" : '' + """ + ascat_to_coral_bed.py \\ + --cnvs ${cnvs} \\ + ${fai_arg} \\ + --output ${prefix}_coral_cn.bed \\ + ${args} + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}_coral_cn.bed + """ +} diff --git a/modules/local/ascattocoralbed/meta.yml b/modules/local/ascattocoralbed/meta.yml new file mode 100644 index 00000000..e81343f8 --- /dev/null +++ b/modules/local/ascattocoralbed/meta.yml @@ -0,0 +1,70 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "ascat_to_coral_bed" +description: Convert ASCAT copy-number calls into the BED CoRAL seeds from. +keywords: + - copy number + - ascat + - ecdna +tools: + - "python": + description: "Python, running bin/ascat_to_coral_bed.py from this pipeline." + homepage: "https://www.python.org" + documentation: "https://docs.python.org" + licence: ["PSF-2.0"] + identifier: "" + +input: + - - meta: + type: map + description: Groovy Map containing sample information + - cnvs: + type: file + description: ASCAT cnvs.txt with chr/startpos/endpos/nMajor/nMinor columns + pattern: "*.cnvs.txt" + - - meta2: + type: map + description: Groovy Map for the reference + - fai: + type: file + description: | + Reference index, used to respell contigs. ASCAT writes "1" where the BAM may + say "chr1", and CoRAL builds chromosome sizes from the BAM header. Optional. + pattern: "*.fai" + +output: + bed: + - meta: + type: map + description: Groovy Map containing sample information + - "*_coral_cn.bed": + type: file + description: Headerless BED4 with total copy number in the last column + pattern: "*_coral_cn.bed" + versions_python: + - - ${task.process}: + type: string + description: Process name + - python: + type: string + description: Tool name + - version: + type: string + description: Tool version + +topics: + versions: + - - ${task.process}: + type: string + description: Process name + - python: + type: string + description: Tool name + - version: + type: string + description: Tool version + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/ascattocoralbed/tests/fixtures/chr.fai b/modules/local/ascattocoralbed/tests/fixtures/chr.fai new file mode 100644 index 00000000..a862f050 --- /dev/null +++ b/modules/local/ascattocoralbed/tests/fixtures/chr.fai @@ -0,0 +1,3 @@ +chr1 248956422 0 60 61 +chr2 248956422 0 60 61 +chrX 248956422 0 60 61 diff --git a/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt b/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt new file mode 100644 index 00000000..40394b02 --- /dev/null +++ b/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt @@ -0,0 +1,6 @@ +chr startpos endpos nMajor nMinor +1 100000 200000 1 1 +1 250000 400000 5 3 +1 500000 500000 2 1 +2 100000 300000 2 0 +X 100000 200000 1 0 diff --git a/modules/local/ascattocoralbed/tests/main.nf.test b/modules/local/ascattocoralbed/tests/main.nf.test new file mode 100644 index 00000000..560fac5c --- /dev/null +++ b/modules/local/ascattocoralbed/tests/main.nf.test @@ -0,0 +1,94 @@ +nextflow_process { + + name "Test Process ASCAT_TO_CORAL_BED" + script "../main.nf" + process "ASCAT_TO_CORAL_BED" + + tag "modules" + tag "modules_local" + tag "ascat_to_coral_bed" + tag "small" + + // Fixture: an ASCAT cnvs.txt calling its contigs "1"/"2"/"X", with one zero-length + // segment that CoRAL's own parser would reject, and a .fai that spells them "chr1" etc. + test("converts ASCAT cnvs.txt to CoRAL BED and respells contigs") { + + when { + process { + """ + input[0] = [ + [ id:'sample1' ], + file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) + ] + input[1] = [ + [:], + file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/chr.fai", checkIfExists: true) + ] + """ + } + } + + then { + def bed = path(process.out.bed[0][1]).readLines() + + assertAll( + { assert process.success }, + // The zero-length segment (chr1:500000-500000) is dropped + { assert bed.size() == 4 }, + // Contigs respelled to match the reference + { assert bed.every { it.startsWith('chr') } }, + // Total CN is nMajor + nMinor, in the last column + { assert bed[0].split('\t')[3] == '2' }, + { assert bed[1].split('\t')[3] == '8' }, + // Natural contig order: chr1, chr2, then chrX + { assert bed.collect { it.split('\t')[0] } == ['chr1', 'chr1', 'chr2', 'chrX'] }, + { assert snapshot(process.out.bed).match() } + ) + } + } + + test("no reference index - contigs are left as ASCAT spelled them") { + + when { + process { + """ + input[0] = [ + [ id:'sample1' ], + file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) + ] + input[1] = [ [:], [] ] + """ + } + } + + then { + def bed = path(process.out.bed[0][1]).readLines() + + assertAll( + { assert process.success }, + { assert bed.collect { it.split('\t')[0] } == ['1', '1', '2', 'X'] } + ) + } + } + + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [ id:'sample1' ], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) ] + input[1] = [ [:], [] ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/ascattocoralbed/tests/main.nf.test.snap b/modules/local/ascattocoralbed/tests/main.nf.test.snap new file mode 100644 index 00000000..70007849 --- /dev/null +++ b/modules/local/ascattocoralbed/tests/main.nf.test.snap @@ -0,0 +1,60 @@ +{ + "converts ASCAT cnvs.txt to CoRAL BED and respells contigs": { + "content": [ + [ + [ + { + "id": "sample1" + }, + "sample1_coral_cn.bed:md5,771563ffb2c54c0fddbe3aafb288e984" + ] + ] + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:36:07.311149784" + }, + "stub": { + "content": [ + { + "0": [ + [ + { + "id": "sample1" + }, + "sample1_coral_cn.bed:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + "ASCAT_TO_CORAL_BED", + "python", + "3.12.2" + ] + ], + "bed": [ + [ + { + "id": "sample1" + }, + "sample1_coral_cn.bed:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions_python": [ + [ + "ASCAT_TO_CORAL_BED", + "python", + "3.12.2" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:36:16.961617365" + } +} \ No newline at end of file diff --git a/modules/local/coral/cycle/main.nf b/modules/local/coral/cycle/main.nf new file mode 100644 index 00000000..2ca506cc --- /dev/null +++ b/modules/local/coral/cycle/main.nf @@ -0,0 +1,46 @@ +process CORAL_CYCLE { + tag "$meta.id" + label 'process_high' + + errorStrategy { task.exitStatus in 130..145 ? 'retry' : 'ignore' } + + container "docker.io/robertaforsyth/coral:3.0.0-chm13-847f3d4" + + input: + tuple val(meta), path(reconstruction) + + output: + tuple val(meta), path("cycles"), emit: reconstruction + tuple val(meta), path("cycles/*_amplicon*_cycles.txt"), emit: cycles, optional: true + tuple val("${task.process}"), val('coral'), eval("python -c 'import importlib.metadata as m; print(m.version(\"CoRAL\"))'"), topic: versions, emit: versions_coral + + when: + task.ext.when == null || task.ext.when + + script: + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CORAL_CYCLE does not support Conda. Please use Docker / Singularity / Apptainer instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // Graphs are copied alongside the re-extracted cycles so AmpliconClassifier + // still finds a matching graph/cycles/summary set in one directory. + """ + mkdir -p cycles + cp ${reconstruction}/*_graph.txt ${reconstruction}/*_summary.txt cycles/ 2>/dev/null || true + + coral cycle_all \\ + --bp-dir ${reconstruction} \\ + --output-prefix cycles/${prefix} \\ + ${args} + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + mkdir -p cycles + touch cycles/${prefix}_amplicon1_cycles.txt + touch cycles/${prefix}_amplicon1_graph.txt + touch cycles/${prefix}_summary.txt + """ +} diff --git a/modules/local/coral/cycle/meta.yml b/modules/local/coral/cycle/meta.yml new file mode 100644 index 00000000..d45cca19 --- /dev/null +++ b/modules/local/coral/cycle/meta.yml @@ -0,0 +1,74 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "coral_cycle" +description: Re-extract cycle decompositions from existing breakpoint graphs. +keywords: + - ecdna + - amplicon + - cycle decomposition +tools: + - "coral": + description: "Reconstructs extrachromosomal DNA structures from long-read sequencing." + homepage: "https://github.com/AmpliconSuite/CoRAL" + documentation: "https://github.com/AmpliconSuite/CoRAL" + doi: "10.1101/gr.279131.124" + licence: ["BSD-3-Clause"] + identifier: "" + +## No environment.yml: CoRAL is not on bioconda -- the `coral` recipe there is an unrelated +## RNA-seq read-bridging tool. Built from a fork adding T2T-CHM13 support and two SCIP fixes +## (https://github.com/robert-a-forsyth/CoRAL, branch chm13-support). + +input: + - - meta: + type: map + description: Groovy Map containing sample information + - reconstruction: + type: directory + description: Output directory from `coral reconstruct` + +output: + reconstruction: + - meta: + type: map + description: Groovy Map containing sample information + - "cycles": + type: directory + description: | + Re-extracted cycles alongside the graph and summary files copied from the + reconstruction, so AmpliconClassifier still sees a complete set. + cycles: + - meta: + type: map + description: Groovy Map containing sample information + - "cycles/*_amplicon*_cycles.txt": + type: file + description: Re-extracted per-amplicon cycles + pattern: "*_amplicon*_cycles.txt" + versions_coral: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +topics: + versions: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/coral/cycle/tests/main.nf.test b/modules/local/coral/cycle/tests/main.nf.test new file mode 100644 index 00000000..5402f2e7 --- /dev/null +++ b/modules/local/coral/cycle/tests/main.nf.test @@ -0,0 +1,36 @@ +nextflow_process { + + name "Test Process CORAL_CYCLE" + script "../main.nf" + process "CORAL_CYCLE" + + tag "modules" + tag "modules_local" + tag "coral_cycle" + tag "small" + + // Stub only: the real tool needs a BAM, a solver and, for the classifier, a ~1 GB data repo. + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'sample1' ], + file('reconstruct') + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/coral/cycle/tests/main.nf.test.snap b/modules/local/coral/cycle/tests/main.nf.test.snap new file mode 100644 index 00000000..c4ed4f81 --- /dev/null +++ b/modules/local/coral/cycle/tests/main.nf.test.snap @@ -0,0 +1,67 @@ +{ + "stub": { + "content": [ + { + "0": [ + [ + { + "id": "sample1" + }, + [ + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_amplicon1_graph.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + [ + "CORAL_CYCLE", + "coral", + "3.0.0" + ] + ], + "cycles": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "reconstruction": [ + [ + { + "id": "sample1" + }, + [ + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_amplicon1_graph.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions_coral": [ + [ + "CORAL_CYCLE", + "coral", + "3.0.0" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:50:09.602318674" + } +} \ No newline at end of file diff --git a/modules/local/coral/plot/main.nf b/modules/local/coral/plot/main.nf new file mode 100644 index 00000000..dc5d023b --- /dev/null +++ b/modules/local/coral/plot/main.nf @@ -0,0 +1,42 @@ +process CORAL_PLOT { + tag "$meta.id" + label 'process_medium' + + // Plotting is cosmetic: never let it fail a run that reconstructed successfully + errorStrategy { task.exitStatus in 130..145 ? 'retry' : 'ignore' } + + container "docker.io/robertaforsyth/coral:3.0.0-chm13-847f3d4" + + input: + tuple val(meta), path(reconstruction), path(bam), path(bai) + val(coral_ref) + + output: + tuple val(meta), path("*_amplicon*.{pdf,png}"), emit: plots, optional: true + tuple val("${task.process}"), val('coral'), eval("python -c 'import importlib.metadata as m; print(m.version(\"CoRAL\"))'"), topic: versions, emit: versions_coral + + when: + task.ext.when == null || task.ext.when + + script: + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CORAL_PLOT does not support Conda. Please use Docker / Singularity / Apptainer instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + coral plot_all \\ + --ref ${coral_ref} \\ + --reconstruction-dir ${reconstruction} \\ + --bam ${bam} \\ + --output-prefix ${prefix} \\ + ${args} + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}_amplicon1_graph.png + touch ${prefix}_amplicon1_cycles.png + """ +} diff --git a/modules/local/coral/plot/meta.yml b/modules/local/coral/plot/meta.yml new file mode 100644 index 00000000..754fe5b0 --- /dev/null +++ b/modules/local/coral/plot/meta.yml @@ -0,0 +1,76 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "coral_plot" +description: Render per-amplicon copy-number and cycle plots. +keywords: + - ecdna + - amplicon + - visualisation +tools: + - "coral": + description: "Reconstructs extrachromosomal DNA structures from long-read sequencing." + homepage: "https://github.com/AmpliconSuite/CoRAL" + documentation: "https://github.com/AmpliconSuite/CoRAL" + doi: "10.1101/gr.279131.124" + licence: ["BSD-3-Clause"] + identifier: "" + +## No environment.yml: CoRAL is not on bioconda -- the `coral` recipe there is an unrelated +## RNA-seq read-bridging tool. Built from a fork adding T2T-CHM13 support and two SCIP fixes +## (https://github.com/robert-a-forsyth/CoRAL, branch chm13-support). + +input: + - - meta: + type: map + description: Groovy Map containing sample information + - reconstruction: + type: directory + description: Output directory from `coral reconstruct` + - bam: + type: file + description: Tumour BAM, for the coverage track + pattern: "*.bam" + - bai: + type: file + description: BAM index + pattern: "*.bai" + - - coral_ref: + type: string + description: Reference genome name, one of hg19, hg38, mm10, t2t + +output: + plots: + - meta: + type: map + description: Groovy Map containing sample information + - "*_amplicon*.{pdf,png}": + type: file + description: Per-amplicon graph and cycle plots + pattern: "*_amplicon*.{pdf,png}" + versions_coral: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +topics: + versions: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/coral/plot/tests/main.nf.test b/modules/local/coral/plot/tests/main.nf.test new file mode 100644 index 00000000..0e0a8c94 --- /dev/null +++ b/modules/local/coral/plot/tests/main.nf.test @@ -0,0 +1,39 @@ +nextflow_process { + + name "Test Process CORAL_PLOT" + script "../main.nf" + process "CORAL_PLOT" + + tag "modules" + tag "modules_local" + tag "coral_plot" + tag "small" + + // Stub only: the real tool needs a BAM, a solver and, for the classifier, a ~1 GB data repo. + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'sample1' ], + file('reconstruct'), + file('sample1.bam'), + file('sample1.bam.bai') + ] + input[1] = 't2t' + """ + } + } + + then { + assertAll( + { assert process.success }, + + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/coral/plot/tests/main.nf.test.snap b/modules/local/coral/plot/tests/main.nf.test.snap new file mode 100644 index 00000000..608bb693 --- /dev/null +++ b/modules/local/coral/plot/tests/main.nf.test.snap @@ -0,0 +1,49 @@ +{ + "stub": { + "content": [ + { + "0": [ + [ + { + "id": "sample1" + }, + [ + "sample1_amplicon1_cycles.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_amplicon1_graph.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + [ + "CORAL_PLOT", + "coral", + "3.0.0" + ] + ], + "plots": [ + [ + { + "id": "sample1" + }, + [ + "sample1_amplicon1_cycles.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_amplicon1_graph.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions_coral": [ + [ + "CORAL_PLOT", + "coral", + "3.0.0" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:50:15.950672699" + } +} \ No newline at end of file diff --git a/modules/local/coral/reconstruct/main.nf b/modules/local/coral/reconstruct/main.nf new file mode 100644 index 00000000..fab0a1b7 --- /dev/null +++ b/modules/local/coral/reconstruct/main.nf @@ -0,0 +1,53 @@ +process CORAL_RECONSTRUCT { + tag "$meta.id" + label 'process_high' + + // A single unsolvable amplicon should not fail a whole cohort; the subworkflow + // warns on the missing output rather than letting the report slot go silent. + errorStrategy { task.exitStatus in 130..145 ? 'retry' : 'ignore' } + + container "docker.io/robertaforsyth/coral:3.0.0-chm13-847f3d4" + + input: + tuple val(meta), path(seeds), path(cn_seg), path(bam), path(bai) + + output: + tuple val(meta), path("reconstruct"), emit: reconstruction + tuple val(meta), path("reconstruct/*_amplicon*_graph.txt"), emit: graphs, optional: true + tuple val(meta), path("reconstruct/*_amplicon*_cycles.txt"), emit: cycles, optional: true + tuple val(meta), path("reconstruct/*_summary.txt"), emit: summary, optional: true + tuple val(meta), path("reconstruct/*_reconstruct.log"), emit: log, optional: true + tuple val("${task.process}"), val('coral'), eval("python -c 'import importlib.metadata as m; print(m.version(\"CoRAL\"))'"), topic: versions, emit: versions_coral + + when: + task.ext.when == null || task.ext.when + + script: + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CORAL_RECONSTRUCT does not support Conda. Please use Docker / Singularity / Apptainer instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // CoRAL derives its output directory by splitting --output-prefix on '/', and + // AmpliconClassifier pairs graph/cycles/summary by prefix within one directory. + """ + mkdir -p reconstruct + + coral reconstruct \\ + --lr-bam ${bam} \\ + --cnv-seed ${seeds} \\ + --cn-seg ${cn_seg} \\ + --output-prefix reconstruct/${prefix} \\ + ${args} + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + mkdir -p reconstruct + touch reconstruct/${prefix}_amplicon1_graph.txt + touch reconstruct/${prefix}_amplicon1_cycles.txt + touch reconstruct/${prefix}_summary.txt + touch reconstruct/${prefix}_reconstruct.log + """ +} diff --git a/modules/local/coral/reconstruct/meta.yml b/modules/local/coral/reconstruct/meta.yml new file mode 100644 index 00000000..f9373390 --- /dev/null +++ b/modules/local/coral/reconstruct/meta.yml @@ -0,0 +1,112 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "coral_reconstruct" +description: Reconstruct amplicon breakpoint graphs and decompose them into cycles. +keywords: + - ecdna + - amplicon + - structural variation + - long-read +tools: + - "coral": + description: "Reconstructs extrachromosomal DNA structures from long-read sequencing." + homepage: "https://github.com/AmpliconSuite/CoRAL" + documentation: "https://github.com/AmpliconSuite/CoRAL" + doi: "10.1101/gr.279131.124" + licence: ["BSD-3-Clause"] + identifier: "" + +## No environment.yml: CoRAL is not on bioconda -- the `coral` recipe there is an unrelated +## RNA-seq read-bridging tool. Built from a fork adding T2T-CHM13 support and two SCIP fixes +## (https://github.com/robert-a-forsyth/CoRAL, branch chm13-support). + +input: + - - meta: + type: map + description: Groovy Map containing sample information + - seeds: + type: file + description: Seed intervals from `coral seed` + pattern: "*_CNV_SEEDS.bed" + - cn_seg: + type: file + description: Copy-number segments with total CN in the last column + pattern: "*.{bed,cns}" + - bam: + type: file + description: Tumour BAM + pattern: "*.bam" + - bai: + type: file + description: BAM index + pattern: "*.bai" + +output: + reconstruction: + - meta: + type: map + description: Groovy Map containing sample information + - "reconstruct": + type: directory + description: | + Graph, cycles and summary files together. AmpliconClassifier pairs them by + shared prefix, so nothing inside may be renamed. + graphs: + - meta: + type: map + description: Groovy Map containing sample information + - "reconstruct/*_amplicon*_graph.txt": + type: file + description: Per-amplicon breakpoint graph, AmpliconArchitect format + pattern: "*_amplicon*_graph.txt" + cycles: + - meta: + type: map + description: Groovy Map containing sample information + - "reconstruct/*_amplicon*_cycles.txt": + type: file + description: Per-amplicon decomposed cycles and paths + pattern: "*_amplicon*_cycles.txt" + summary: + - meta: + type: map + description: Groovy Map containing sample information + - "reconstruct/*_summary.txt": + type: file + description: Per-run amplicon summary, written even with no amplicons + pattern: "*_summary.txt" + log: + - meta: + type: map + description: Groovy Map containing sample information + - "reconstruct/*_reconstruct.log": + type: file + description: Reconstruction log including solver output + pattern: "*_reconstruct.log" + versions_coral: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +topics: + versions: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/coral/reconstruct/tests/main.nf.test b/modules/local/coral/reconstruct/tests/main.nf.test new file mode 100644 index 00000000..c746a5d5 --- /dev/null +++ b/modules/local/coral/reconstruct/tests/main.nf.test @@ -0,0 +1,42 @@ +nextflow_process { + + name "Test Process CORAL_RECONSTRUCT" + script "../main.nf" + process "CORAL_RECONSTRUCT" + + tag "modules" + tag "modules_local" + tag "coral_reconstruct" + tag "small" + + // Stub only: the real tool needs a BAM, a solver and, for the classifier, a ~1 GB data repo. + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'sample1' ], + file('sample1_CNV_SEEDS.bed'), + file('sample1_coral_cn.bed'), + file('sample1.bam'), + file('sample1.bam.bai') + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + // AmpliconClassifier pairs graph/cycles/summary by prefix in one directory + { assert process.out.graphs[0][1].toString().endsWith('_amplicon1_graph.txt') }, + { assert process.out.cycles[0][1].toString().endsWith('_amplicon1_cycles.txt') }, + { assert process.out.summary[0][1].toString().endsWith('_summary.txt') }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/coral/reconstruct/tests/main.nf.test.snap b/modules/local/coral/reconstruct/tests/main.nf.test.snap new file mode 100644 index 00000000..0d60bf14 --- /dev/null +++ b/modules/local/coral/reconstruct/tests/main.nf.test.snap @@ -0,0 +1,117 @@ +{ + "stub": { + "content": [ + { + "0": [ + [ + { + "id": "sample1" + }, + [ + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_amplicon1_graph.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_reconstruct.log:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon1_graph.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "sample1" + }, + "sample1_summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + [ + { + "id": "sample1" + }, + "sample1_reconstruct.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "5": [ + [ + "CORAL_RECONSTRUCT", + "coral", + "3.0.0" + ] + ], + "cycles": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "graphs": [ + [ + { + "id": "sample1" + }, + "sample1_amplicon1_graph.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "log": [ + [ + { + "id": "sample1" + }, + "sample1_reconstruct.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "reconstruction": [ + [ + { + "id": "sample1" + }, + [ + "sample1_amplicon1_cycles.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_amplicon1_graph.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_reconstruct.log:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1_summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "summary": [ + [ + { + "id": "sample1" + }, + "sample1_summary.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions_coral": [ + [ + "CORAL_RECONSTRUCT", + "coral", + "3.0.0" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:50:23.471598843" + } +} \ No newline at end of file diff --git a/modules/local/coral/seed/main.nf b/modules/local/coral/seed/main.nf new file mode 100644 index 00000000..dc489a9d --- /dev/null +++ b/modules/local/coral/seed/main.nf @@ -0,0 +1,42 @@ +process CORAL_SEED { + tag "$meta.id" + label 'process_low' + + // No conda: CoRAL is not packaged on bioconda (the `coral` recipe there is an + // unrelated RNA-seq tool), and this image carries the CHM13 fork. See meta.yml + container "docker.io/robertaforsyth/coral:3.0.0-chm13-847f3d4" + + input: + tuple val(meta), path(cn_seg), path(bam), path(bai) + val(coral_ref) + + output: + tuple val(meta), path("*_CNV_SEEDS.bed"), emit: seeds + tuple val("${task.process}"), val('coral'), eval("python -c 'import importlib.metadata as m; print(m.version(\"CoRAL\"))'"), topic: versions, emit: versions_coral + + when: + task.ext.when == null || task.ext.when + + script: + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "CORAL_SEED does not support Conda. Please use Docker / Singularity / Apptainer instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + coral seed \\ + --cn-seg ${cn_seg} \\ + --ref ${coral_ref} \\ + --lr-bam ${bam} \\ + --output-prefix ${prefix} \\ + ${args} + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + // A non-empty seed, so the stub exercises the reconstruct path rather than the + // empty-seed branch. Use --coral_gain to force the empty case in a test. + """ + printf 'chr1\\t100000\\t400000\\t8\\n' > ${prefix}_CNV_SEEDS.bed + """ +} diff --git a/modules/local/coral/seed/meta.yml b/modules/local/coral/seed/meta.yml new file mode 100644 index 00000000..b75e9118 --- /dev/null +++ b/modules/local/coral/seed/meta.yml @@ -0,0 +1,81 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "coral_seed" +description: Identify amplified intervals that seed CoRAL amplicon reconstruction. +keywords: + - ecdna + - amplicon + - copy number + - long-read +tools: + - "coral": + description: "Reconstructs extrachromosomal DNA structures from long-read sequencing." + homepage: "https://github.com/AmpliconSuite/CoRAL" + documentation: "https://github.com/AmpliconSuite/CoRAL" + doi: "10.1101/gr.279131.124" + licence: ["BSD-3-Clause"] + identifier: "" + +## No environment.yml: CoRAL is not on bioconda -- the `coral` recipe there is an unrelated +## RNA-seq read-bridging tool. The container is built from a fork adding T2T-CHM13 support +## (https://github.com/robert-a-forsyth/CoRAL, branch chm13-support), which also fixes two +## upstream bugs that made `--solver scip` fail. See containers/coral.Dockerfile. + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1' ]` + - cn_seg: + type: file + description: Copy-number segments with total CN in the last column + pattern: "*.{bed,cns}" + - bam: + type: file + description: Tumour BAM; CoRAL reads chromosome sizes from its header + pattern: "*.bam" + - bai: + type: file + description: BAM index + pattern: "*.bai" + - - coral_ref: + type: string + description: Reference genome name, one of hg19, hg38, mm10, t2t + +output: + seeds: + - meta: + type: map + description: Groovy Map containing sample information + - "*_CNV_SEEDS.bed": + type: file + description: Amplified seed intervals; empty when nothing exceeds --gain + pattern: "*_CNV_SEEDS.bed" + versions_coral: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +topics: + versions: + - - ${task.process}: + type: string + description: Process name + - coral: + type: string + description: Tool name + - version: + type: string + description: Tool version + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/coral/seed/tests/main.nf.test b/modules/local/coral/seed/tests/main.nf.test new file mode 100644 index 00000000..6f3610b2 --- /dev/null +++ b/modules/local/coral/seed/tests/main.nf.test @@ -0,0 +1,41 @@ +nextflow_process { + + name "Test Process CORAL_SEED" + script "../main.nf" + process "CORAL_SEED" + + tag "modules" + tag "modules_local" + tag "coral" + tag "coral_seed" + tag "small" + + // Stub only: seeding needs a real BAM, since CoRAL reads chromosome sizes from its header. + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'sample1' ], + file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true), + file('sample1.bam'), + file('sample1.bam.bai') + ] + input[1] = 't2t' + """ + } + } + + then { + assertAll( + { assert process.success }, + // Non-empty, so the subworkflow's empty-seed branch is not taken + { assert path(process.out.seeds[0][1]).readLines().size() == 1 }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/coral/seed/tests/main.nf.test.snap b/modules/local/coral/seed/tests/main.nf.test.snap new file mode 100644 index 00000000..bcca1d64 --- /dev/null +++ b/modules/local/coral/seed/tests/main.nf.test.snap @@ -0,0 +1,43 @@ +{ + "stub": { + "content": [ + { + "0": [ + [ + { + "id": "sample1" + }, + "sample1_CNV_SEEDS.bed:md5,f679880d9757f33af43363b38254f3f0" + ] + ], + "1": [ + [ + "CORAL_SEED", + "coral", + "3.0.0" + ] + ], + "seeds": [ + [ + { + "id": "sample1" + }, + "sample1_CNV_SEEDS.bed:md5,f679880d9757f33af43363b38254f3f0" + ] + ], + "versions_coral": [ + [ + "CORAL_SEED", + "coral", + "3.0.0" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-24T09:50:31.018971049" + } +} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 4230dd67..5bc6f220 100644 --- a/nextflow.config +++ b/nextflow.config @@ -76,6 +76,21 @@ params { sigprofiler_matrix_args = '--plot' sigprofiler_assignment_args = null + // CoRAL / ecDNA options + coral_gain = 6.0 + coral_min_seed_size = 100000 + coral_max_seg_gap = 300000 + coral_solver = 'scip' + coral_solver_threads = -1 + coral_solver_time_limit = 7200 + coral_global_time_limit = 21600 + coral_min_bp_support = 1.75 + coral_run_cycle = false + coral_cycle_decomp_alpha = 0.01 + coral_plot = true + gurobi_license = null + aa_data_repo = null + // Skip options skip_qc = false skip_cramino = false @@ -96,6 +111,8 @@ params { skip_whatshapstats = false skip_signatures = false skip_report = false + skip_coral = false + skip_ampliconclassifier = false // minimap2 options minimap2_ont_model = null diff --git a/nextflow_schema.json b/nextflow_schema.json index a4efeb21..42579ed4 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,7 +10,10 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": ["input", "outdir"], + "required": [ + "input", + "outdir" + ], "properties": { "input": { "type": "string", @@ -81,25 +84,37 @@ "type": "string", "description": "When two germline callers are used, specifies how to combine them. 'consensus' keeps only variants called by both callers; 'all' keeps all variants from both callers.", "default": "all", - "enum": ["consensus", "all"] + "enum": [ + "consensus", + "all" + ] }, "somatic_var_combine": { "type": "string", "description": "When two somatic callers are used, specifies how to combine them. 'consensus' keeps only variants called by both callers; 'all' keeps all variants from both callers.", "default": "all", - "enum": ["consensus", "all"] + "enum": [ + "consensus", + "all" + ] }, "prioritize_caller_germline": { "type": "string", "description": "When both germline callers are used, specifies which caller's format to use for variants called by both. Must be [deepvariant, clair].", "default": "clair", - "enum": ["deepvariant", "clair"] + "enum": [ + "deepvariant", + "clair" + ] }, "prioritize_caller_somatic": { "type": "string", "description": "When both somatic callers are used, specifies which caller's format to use for variants called by both. Must be [deepsomatic, clair].", "default": "clair", - "enum": ["deepsomatic", "clair"] + "enum": [ + "deepsomatic", + "clair" + ] }, "smallvar_filter_pass": { "type": "boolean", @@ -155,7 +170,10 @@ "description": "Name of iGenomes reference.", "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details.", - "enum": ["GRCh38", "CHM13"] + "enum": [ + "GRCh38", + "CHM13" + ] }, "igenomes_ignore": { "type": "boolean", @@ -590,6 +608,14 @@ "skip_report": { "type": "boolean", "description": "Skip the final per-sample HTML report" + }, + "skip_coral": { + "type": "boolean", + "description": "Skip CoRAL amplicon reconstruction and AmpliconClassifier." + }, + "skip_ampliconclassifier": { + "type": "boolean", + "description": "Run CoRAL but skip AmpliconClassifier, which avoids needing an AA data repository." } } }, @@ -660,7 +686,14 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], "hidden": true }, "email_on_fail": { @@ -742,6 +775,82 @@ "description": "Display hidden parameters in the help message (only works when --help or --help_full are provided)." } } + }, + "coral_options": { + "title": "CoRAL / ecDNA options", + "type": "object", + "description": "Amplicon reconstruction with CoRAL and classification with AmpliconClassifier.", + "default": "", + "properties": { + "coral_gain": { + "type": "number", + "default": 6.0, + "description": "Minimum total copy number for a segment to seed an amplicon (CoRAL's --gain)." + }, + "coral_min_seed_size": { + "type": "integer", + "default": 100000, + "description": "Minimum seed interval size in bp (CoRAL's --min-seed-size)." + }, + "coral_max_seg_gap": { + "type": "integer", + "default": 300000, + "description": "Largest gap in bp that CoRAL will bridge when merging seed segments (--max-seg-gap)." + }, + "coral_solver": { + "type": "string", + "default": "scip", + "enum": [ + "scip", + "gurobi_direct" + ], + "description": "Optimiser for CoRAL's non-convex MIQCP cycle decomposition. SCIP is open-source and needs no licence; gurobi_direct is faster but requires --gurobi_license." + }, + "coral_solver_threads": { + "type": "integer", + "default": -1, + "description": "Solver threads; -1 lets the solver decide (CoRAL's --solver-threads)." + }, + "coral_solver_time_limit": { + "type": "integer", + "default": 7200, + "description": "Per-amplicon solver time limit in seconds (CoRAL's --solver-time-limit)." + }, + "coral_global_time_limit": { + "type": "integer", + "default": 21600, + "description": "Per-sample time limit in seconds across all amplicons (CoRAL's --global-time-limit)." + }, + "coral_min_bp_support": { + "type": "number", + "default": 1.75, + "description": "Minimum breakpoint read support (CoRAL's --min-bp-support). CoRAL's own README recommends a much larger value such as 10.0 for WGS; 1.75 matches the setting validated on this cohort." + }, + "coral_run_cycle": { + "type": "boolean", + "description": "Re-extract cycles with `coral cycle_all` after reconstruction, and classify those instead." + }, + "coral_cycle_decomp_alpha": { + "type": "number", + "default": 0.01, + "description": "Cycle decomposition alpha, used only with --coral_run_cycle (CoRAL's --alpha)." + }, + "coral_plot": { + "type": "boolean", + "default": true, + "description": "Render CoRAL's per-amplicon graph and cycle plots. Plot failures never fail the run." + }, + "gurobi_license": { + "type": "string", + "format": "file-path", + "description": "Path to a gurobi.lic, mounted into the CoRAL tasks. Required with --coral_solver gurobi_direct." + }, + "aa_data_repo": { + "type": "string", + "format": "directory-path", + "description": "AmpliconArchitect data repository for AmpliconClassifier. Downloaded automatically for GRCh38; required for CHM13, which has no published repo." + } + } } }, "allOf": [ @@ -790,6 +899,9 @@ { "$ref": "#/$defs/report_options" }, + { + "$ref": "#/$defs/coral_options" + }, { "$ref": "#/$defs/skip_options" }, diff --git a/subworkflows/local/ecdna.nf b/subworkflows/local/ecdna.nf new file mode 100644 index 00000000..d4d34b07 --- /dev/null +++ b/subworkflows/local/ecdna.nf @@ -0,0 +1,142 @@ +// +// ecDNA and focal amplification: CoRAL reconstruction, AmpliconClassifier classification +// + +// IMPORT MODULES +include { ASCAT_TO_CORAL_BED } from '../../modules/local/ascattocoralbed/main' +include { CORAL_SEED } from '../../modules/local/coral/seed/main' +include { CORAL_RECONSTRUCT } from '../../modules/local/coral/reconstruct/main' +include { CORAL_CYCLE } from '../../modules/local/coral/cycle/main' +include { CORAL_PLOT } from '../../modules/local/coral/plot/main' +include { AMPLICONCLASSIFIER } from '../../modules/local/ampliconclassifier/main' + +workflow ECDNA { + + take: + tumor_bam // [meta, bam, bai] -- tumour BAMs only + ascat_cnvs // [meta, cnvs_txt] -- ASCAT.out.cnvs + fai // [[:], fai] + data_repo // [[:], dir] -- AA data repo, empty channel with --skip_ampliconclassifier + coral_ref // val 'hg38' | 't2t' + ac_ref // val 'GRCh38' | 'CHM13' + + main: + // + // MODULE: ASCAT_TO_CORAL_BED (label: process_single) + // ASCAT writes cnvs.txt as chr/startpos/endpos/nMajor/nMinor; CoRAL wants + // headerless BED with total CN last, spelled like the BAM's contigs. + // Input: [meta, cnvs_txt], [[:], fai] + // Output: .bed -- [meta, bed] + // + ASCAT_TO_CORAL_BED ( + ascat_cnvs, + fai.first() + ) + + // + // MODULE: CORAL_SEED (label: process_low) + // Input: [meta, cn_seg_bed, bam, bai] + // Output: .seeds -- [meta, bed] -- amplified intervals above --gain + // + // multiMap, not two reads of one channel: a channel feeding both a process and an + // operator is not allowed, and reconstruct needs the same cn_seg/bam that seed used. + ASCAT_TO_CORAL_BED.out.bed + .join(tumor_bam, failOnMismatch: true, failOnDuplicate: true) + .multiMap { meta, cn_seg, bam, bai -> + seed: [meta, cn_seg, bam, bai] + reconstruct: [meta, cn_seg, bam, bai] + plot: [meta, bam, bai] + } + .set { coral_inputs } + // coral_inputs.seed / .reconstruct: [meta, cn_seg_bed, bam, bai]; .plot: [meta, bam, bai] + + CORAL_SEED ( + coral_inputs.seed, + coral_ref + ) + + // + // A sample with no amplification above --gain yields an empty seed BED, and + // CoRAL reconstruct errors on one. Size, not countLines(), so the file is + // not staged just to be measured. + // + CORAL_SEED.out.seeds + .branch { _meta, bed -> + seeded: bed.toFile().length() > 0 + unseeded: true + } + .set { branched_seeds } + + branched_seeds.unseeded + .subscribe { meta, _bed -> log.info("No amplified intervals found for ${meta.id}: skipping ecDNA reconstruction.") } + + // + // MODULE: CORAL_RECONSTRUCT (label: process_high) + // Input: [meta, seeds, cn_seg_bed, bam, bai] + // Output: .reconstruction -- [meta, dir] -- graph/cycles/summary, named as AC expects + // + // remainder: false and no failOnMismatch: an unseeded sample is dropped here by design + branched_seeds.seeded + .join(coral_inputs.reconstruct, failOnDuplicate: true) + .set { coral_reconstruct_input } + // coral_reconstruct_input: [meta, seeds, cn_seg_bed, bam, bai] + + CORAL_RECONSTRUCT ( + coral_reconstruct_input + ) + + // + // MODULE: CORAL_CYCLE (label: process_high) -- opt-in cycle re-extraction + // Input: [meta, reconstruction_dir] + // Output: .reconstruction -- [meta, dir] -- re-extracted cycles beside the copied graphs + // + if (params.coral_run_cycle) { + CORAL_CYCLE ( + CORAL_RECONSTRUCT.out.reconstruction + ) + ch_for_classifier = CORAL_CYCLE.out.reconstruction + } + else { + ch_for_classifier = CORAL_RECONSTRUCT.out.reconstruction + } + // ch_for_classifier: [meta, dir] + + // + // MODULE: CORAL_PLOT (label: process_medium) + // Runs beside the classifier rather than in front of it: plotting is cosmetic + // and must never gate classification. + // + ch_plots = channel.empty() + if (params.coral_plot) { + CORAL_RECONSTRUCT.out.reconstruction + .join(coral_inputs.plot, failOnDuplicate: true) + .set { coral_plot_input } + // coral_plot_input: [meta, reconstruction_dir, bam, bai] + + CORAL_PLOT ( + coral_plot_input, + coral_ref + ) + ch_plots = CORAL_PLOT.out.plots + } + + // + // MODULE: AMPLICONCLASSIFIER (label: process_medium) + // Input: [meta, reconstruction_dir], [[:], data_repo], ac_ref + // Output: .classification -- [meta, tsv] -- amplicon_classification_profiles.tsv + // + ch_classification = channel.empty() + if (!params.skip_ampliconclassifier) { + AMPLICONCLASSIFIER ( + ch_for_classifier, + data_repo.first(), + ac_ref + ) + ch_classification = AMPLICONCLASSIFIER.out.classification + } + + emit: + classification = ch_classification // [meta, tsv] + reconstruction = CORAL_RECONSTRUCT.out.reconstruction // [meta, dir] + plots = ch_plots // [meta, [files]] +} diff --git a/subworkflows/local/prepare_aa_data_repo.nf b/subworkflows/local/prepare_aa_data_repo.nf new file mode 100644 index 00000000..da5c05b9 --- /dev/null +++ b/subworkflows/local/prepare_aa_data_repo.nf @@ -0,0 +1,43 @@ +// +// Stage the AmpliconArchitect data repository AmpliconClassifier reads at runtime +// + +include { WGET as WGET_AA_DATA_REPO } from '../../modules/nf-core/wget/main' +include { UNTAR as UNTAR_AA_DATA_REPO } from '../../modules/nf-core/untar/main' + +workflow PREPARE_AA_DATA_REPO { + + take: + data_repo // path or null -- params.aa_data_repo + repo_url // URL or null -- genome attribute aa_data_repo_url + + main: + ch_versions = channel.empty() + + if (data_repo) { + ch_data_repo = channel.value([ [ id: 'aa_data_repo' ], file(data_repo, checkIfExists: true) ]) + } + else if (repo_url) { + // + // MODULES: WGET_AA_DATA_REPO -> UNTAR_AA_DATA_REPO (labels: process_single) + // ~1.1 GB tarball; the plain build, not GRCh38_indexed, whose extra BWA index AC never reads + // + WGET_AA_DATA_REPO ( + channel.value([ [ id: 'aa_data_repo' ], repo_url ]) + ) + + UNTAR_AA_DATA_REPO ( + WGET_AA_DATA_REPO.out.outfile + ) + + ch_data_repo = UNTAR_AA_DATA_REPO.out.untar + ch_versions = ch_versions.mix(WGET_AA_DATA_REPO.out.versions, UNTAR_AA_DATA_REPO.out.versions) + } + else { + error("AmpliconClassifier needs an AmpliconArchitect data repository, which is not published for ${params.genome}. Set --aa_data_repo , or use --skip_ampliconclassifier.") + } + + emit: + data_repo = ch_data_repo // [[id:'aa_data_repo'], dir] + versions = ch_versions +} diff --git a/subworkflows/local/tests/ecdna.nf.test b/subworkflows/local/tests/ecdna.nf.test new file mode 100644 index 00000000..99df521e --- /dev/null +++ b/subworkflows/local/tests/ecdna.nf.test @@ -0,0 +1,121 @@ +nextflow_workflow { + + name "Test Subworkflow ECDNA" + script "../ecdna.nf" + workflow "ECDNA" + + tag "subworkflows" + tag "subworkflows_local" + tag "ecdna" + tag "small" + + // Stub-only: CoRAL and AmpliconClassifier need real BAMs, a solver and a ~1 GB data + // repo. What these prove is the wiring -- that the joins pair, that the empty-seed + // branch drops a sample before reconstruct, and that the params gate the optional steps. + test("-stub - reconstructs and classifies a seeded sample") { + + options "-stub" + + when { + params { + coral_run_cycle = false + coral_plot = true + skip_ampliconclassifier = false + } + workflow { + """ + input[0] = channel.of([ [ id:'sample1' ], file('sample1.bam'), file('sample1.bam.bai') ]) + input[1] = channel.of([ [ id:'sample1' ], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) ]) + input[2] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/chr.fai", checkIfExists: true) ]) + input[3] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures") ]) + input[4] = 't2t' + input[5] = 'CHM13' + """ + } + } + + then { + def names = workflow.trace.tasks().collect { it.name.split(' ')[0] } + + assertAll( + { assert workflow.success }, + { assert names.contains('ECDNA:ASCAT_TO_CORAL_BED') }, + { assert names.contains('ECDNA:CORAL_SEED') }, + { assert names.contains('ECDNA:CORAL_RECONSTRUCT') }, + { assert names.contains('ECDNA:CORAL_PLOT') }, + { assert names.contains('ECDNA:AMPLICONCLASSIFIER') }, + // Cycle re-extraction is opt-in + { assert !names.contains('ECDNA:CORAL_CYCLE') }, + { assert workflow.out.classification.size() == 1 } + ) + } + } + + test("-stub - coral_run_cycle classifies the re-extracted cycles") { + + options "-stub" + + when { + params { + coral_run_cycle = true + coral_plot = false + skip_ampliconclassifier = false + } + workflow { + """ + input[0] = channel.of([ [ id:'sample1' ], file('sample1.bam'), file('sample1.bam.bai') ]) + input[1] = channel.of([ [ id:'sample1' ], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) ]) + input[2] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/chr.fai", checkIfExists: true) ]) + input[3] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures") ]) + input[4] = 't2t' + input[5] = 'CHM13' + """ + } + } + + then { + def names = workflow.trace.tasks().collect { it.name.split(' ')[0] } + + assertAll( + { assert workflow.success }, + { assert names.contains('ECDNA:CORAL_CYCLE') }, + // Plotting is off, and must not gate the classifier + { assert !names.contains('ECDNA:CORAL_PLOT') }, + { assert names.contains('ECDNA:AMPLICONCLASSIFIER') } + ) + } + } + + test("-stub - skip_ampliconclassifier stops after reconstruction") { + + options "-stub" + + when { + params { + coral_run_cycle = false + coral_plot = false + skip_ampliconclassifier = true + } + workflow { + """ + input[0] = channel.of([ [ id:'sample1' ], file('sample1.bam'), file('sample1.bam.bai') ]) + input[1] = channel.of([ [ id:'sample1' ], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) ]) + input[2] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/chr.fai", checkIfExists: true) ]) + input[3] = channel.empty() + input[4] = 't2t' + input[5] = 'CHM13' + """ + } + } + + then { + def names = workflow.trace.tasks().collect { it.name.split(' ')[0] } + + assertAll( + { assert workflow.success }, + { assert names.contains('ECDNA:CORAL_RECONSTRUCT') }, + { assert !names.contains('ECDNA:AMPLICONCLASSIFIER') } + ) + } + } +} diff --git a/tests/.nftignore b/tests/.nftignore index dfc63dec..c835ed89 100644 --- a/tests/.nftignore +++ b/tests/.nftignore @@ -28,6 +28,7 @@ pipeline_info/*.{html,json,txt,yml} */variants/deepsomatic/*.{vcf.gz,vcf.gz.tbi} */variants/deepvariant/*.{vcf.gz,vcf.gz.tbi} */variants/savana/* +*/ecdna/* */signatures/matrices/** */signatures/assignment/** */report/*.html diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index acf98c0d..c0133f6d 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -28,6 +28,8 @@ include { NANOPLOT as NANOPLOT_PRE } from '../modules/nf-core/nanoplot/ include { NANOPLOT as NANOPLOT_POST } from '../modules/nf-core/nanoplot/main' include { MOSDEPTH } from '../modules/nf-core/mosdepth/main' include { ASCAT } from '../modules/nf-core/ascat/main' +include { ECDNA } from '../subworkflows/local/ecdna' +include { PREPARE_AA_DATA_REPO } from '../subworkflows/local/prepare_aa_data_repo' include { SEVERUS } from '../modules/nf-core/severus/main.nf' include { METAEXTRACT } from '../modules/local/metaextract/main' include { CLAIRSTO_CNA_RESOURCES } from '../modules/local/clairsto/cna_resources/main' @@ -115,6 +117,9 @@ workflow LRSOMATIC { params.vep_species = getGenomeAttribute('vep_species') params.sigprofiler_genome = getGenomeAttribute('sigprofiler_genome') params.sigprofiler_genome_url = getGenomeAttribute('sigprofiler_genome_url') + params.coral_ref = getGenomeAttribute('coral_ref') + params.ac_ref = getGenomeAttribute('ac_ref') + params.aa_data_repo_url = getGenomeAttribute('aa_data_repo_url') // Resolved once here to avoid a HEAD request per default plugin URL, and passed straight to // the VEP tasks: conf/modules.config closures do not see a param assigned here. @@ -193,6 +198,15 @@ workflow LRSOMATIC { // CHM13 has no ascat_loci_rt attribute, so the built set is GC-only by construction build_clairsto_cna = clairsto_cna_dir == null && params.genome == 'CHM13' && params.skip_ascat + // CoRAL seeds from ASCAT's copy number, so it cannot run without it + if (!params.skip_coral && params.skip_ascat) { + error("CoRAL seeds from ASCAT's copy-number segments. Remove --skip_ascat, or add --skip_coral.") + } + // Gurobi needs a licence file; SCIP, the default, needs nothing + if (!params.skip_coral && params.coral_solver == 'gurobi_direct' && !params.gurobi_license) { + error("--coral_solver gurobi_direct needs a licence: pass --gurobi_license , or use the default --coral_solver scip.") + } + // A missing set would leave the join below waiting forever, so CLAIRSTO would silently never run if (build_clairsto_cna) { def missing_ascat = ['ascat_alleles': params.ascat_allele_files, @@ -649,6 +663,8 @@ workflow LRSOMATIC { ch_ascat_files = channel.empty() ascat_tumoronly_ch = channel.empty() + ch_ascat_cnvs = channel.empty() + ch_ecdna_tumor_bam = channel.empty() if (!params.skip_ascat) { branched_minimap.tumor_only @@ -703,6 +719,17 @@ workflow LRSOMATIC { .groupTuple() .map { meta, files -> [meta, files.flatten()] } // ch_ascat_files: [meta, [file, file, ...]] + + // CoRAL seeds from ASCAT's total copy number. Built from ascat_ch so the meta + // key matches ASCAT's output exactly and the joins in ECDNA pair. + ch_ascat_cnvs = ASCAT.out.cnvs + // ch_ascat_cnvs: [meta, cnvs_txt] + + ch_ecdna_tumor_bam = ascat_ch + .map { meta, _normal_bam, _normal_bai, tumor_bam, tumor_bai -> + return [meta, tumor_bam, tumor_bai] + } + // ch_ecdna_tumor_bam: [meta, tumor_bam, tumor_bai] } // SUBWORKFLOW: TUMORONLY_SMALLVAR @@ -1266,6 +1293,41 @@ workflow LRSOMATIC { // ch_wakhan_files: [meta, [file_or_dir, ...]] } + // + // SUBWORKFLOW: ECDNA -- CoRAL amplicon reconstruction, then AmpliconClassifier + // Input: ch_ecdna_tumor_bam -- [meta, tumor_bam, tumor_bai] + // ch_ascat_cnvs -- [meta, cnvs_txt] + // ch_fai -- [[:], fai] + // Output: .classification -- [meta, tsv] -- amplicon_classification_profiles.tsv + // + + ch_ecdna_classification = channel.empty() + + if (!params.skip_coral) { + // The data repo is only fetched when the classifier will actually use it + ch_aa_data_repo = channel.empty() + if (!params.skip_ampliconclassifier) { + PREPARE_AA_DATA_REPO ( + params.aa_data_repo, + params.aa_data_repo_url + ) + ch_aa_data_repo = PREPARE_AA_DATA_REPO.out.data_repo + ch_versions = ch_versions.mix(PREPARE_AA_DATA_REPO.out.versions) + } + + ECDNA ( + ch_ecdna_tumor_bam, + ch_ascat_cnvs, + ch_fai, + ch_aa_data_repo, + params.coral_ref, + params.ac_ref + ) + + ch_ecdna_classification = ECDNA.out.classification + // ch_ecdna_classification: [meta, tsv] + } + // // MODULE: LRSOMATICREPORT -- per-sample HTML report; all inputs optional, so joins use remainder: true on the tumor id // From 001926c4f8e29d302fd96d19d21bf790ad1df249 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 10:46:46 +0200 Subject: [PATCH 17/36] fixes ecDNA subworkflow --- subworkflows/local/ecdna.nf | 8 ++++---- subworkflows/local/prepare_aa_data_repo.nf | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/subworkflows/local/ecdna.nf b/subworkflows/local/ecdna.nf index d4d34b07..66c94197 100644 --- a/subworkflows/local/ecdna.nf +++ b/subworkflows/local/ecdna.nf @@ -15,8 +15,8 @@ workflow ECDNA { take: tumor_bam // [meta, bam, bai] -- tumour BAMs only ascat_cnvs // [meta, cnvs_txt] -- ASCAT.out.cnvs - fai // [[:], fai] - data_repo // [[:], dir] -- AA data repo, empty channel with --skip_ampliconclassifier + fai // [[:], fai] -- value channel, reused by every sample + data_repo // [[:], dir] -- value channel; empty with --skip_ampliconclassifier coral_ref // val 'hg38' | 't2t' ac_ref // val 'GRCh38' | 'CHM13' @@ -30,7 +30,7 @@ workflow ECDNA { // ASCAT_TO_CORAL_BED ( ascat_cnvs, - fai.first() + fai ) // @@ -129,7 +129,7 @@ workflow ECDNA { if (!params.skip_ampliconclassifier) { AMPLICONCLASSIFIER ( ch_for_classifier, - data_repo.first(), + data_repo, ac_ref ) ch_classification = AMPLICONCLASSIFIER.out.classification diff --git a/subworkflows/local/prepare_aa_data_repo.nf b/subworkflows/local/prepare_aa_data_repo.nf index da5c05b9..822457fc 100644 --- a/subworkflows/local/prepare_aa_data_repo.nf +++ b/subworkflows/local/prepare_aa_data_repo.nf @@ -30,7 +30,9 @@ workflow PREPARE_AA_DATA_REPO { WGET_AA_DATA_REPO.out.outfile ) - ch_data_repo = UNTAR_AA_DATA_REPO.out.untar + // .first(): UNTAR emits a queue channel, and every sample's classifier task + // needs the same repo -- without this only the first sample would get it. + ch_data_repo = UNTAR_AA_DATA_REPO.out.untar.first() ch_versions = ch_versions.mix(WGET_AA_DATA_REPO.out.versions, UNTAR_AA_DATA_REPO.out.versions) } else { From 27728f812ab733876e1804451cdc9046820ee8aa Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:33:33 +0200 Subject: [PATCH 18/36] Move the INFO/SOMATIC filter of local BCFTOOLS_VIEW into ext.args Drop the hard-coded -i from the module script and set it in the PHASING_HAPLOTYPING:BCFTOOLS_VIEW config block. Refresh meta.yml: drop the removed targets inputs, generic description, document versions output/topic. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 1 + modules/local/bcftools/view/main.nf | 1 - modules/local/bcftools/view/meta.yml | 35 +++++++++++++++++++--------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 9372bad1..ca462202 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -460,6 +460,7 @@ process { } withName: '.*:PHASING_HAPLOTYPING:BCFTOOLS_VIEW' { ext.prefix = { "somatic_smallvariants" } + ext.args = { "-i 'INFO/SOMATIC=1'" } publishDir = [ path: { "${params.outdir}/${meta.id}/variants/phased" }, mode: params.publish_dir_mode, diff --git a/modules/local/bcftools/view/main.nf b/modules/local/bcftools/view/main.nf index ae81084e..63905dd2 100644 --- a/modules/local/bcftools/view/main.nf +++ b/modules/local/bcftools/view/main.nf @@ -23,7 +23,6 @@ process BCFTOOLS_VIEW { def prefix = task.ext.prefix ?: "${meta.id}" """ bcftools view \\ - -i 'INFO/SOMATIC=1' \\ -Oz \\ -W=tbi \\ ${args} \\ diff --git a/modules/local/bcftools/view/meta.yml b/modules/local/bcftools/view/meta.yml index 28c0fc48..29e3646b 100644 --- a/modules/local/bcftools/view/meta.yml +++ b/modules/local/bcftools/view/meta.yml @@ -1,5 +1,5 @@ name: bcftools_view -description: Filter VCF to positions defined by a targets file using bcftools view -T +description: Filter a VCF with bcftools view, with the filter expression supplied via ext.args; outputs a bgzipped VCF and tbi index keywords: - filtering - VCF @@ -25,14 +25,6 @@ input: type: file description: Tabix index of the input VCF pattern: "*.tbi" - - targets: - type: file - description: VCF file used as position filter (-T) - pattern: "*.{vcf.gz,vcf,bcf}" - - targets_tbi: - type: file - description: Tabix index of the targets VCF - pattern: "*.tbi" output: vcf: - - meta: @@ -50,7 +42,28 @@ output: type: file description: Tabix index of filtered VCF pattern: "*.tbi" + versions_bcftools: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version | sed '1!d; s/^.*bcftools //'": + type: string + description: The command used to generate the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version | sed '1!d; s/^.*bcftools //'": + type: string + description: The command used to generate the version of the tool authors: - - "@rforsyth" + - "@robert-a-forsyth" maintainers: - - "@rforsyth" + - "@robert-a-forsyth" From c691a0cffa1a5f18ec1719fea697349ecb482570 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:35:21 +0200 Subject: [PATCH 19/36] Leave FILTER untouched in VCFTAG VCFTAG now only stamps the provenance flag, so --smallvar_filter_pass false keeps each caller's FILTER. ORIG_FILTER is stamped only on the ClairS-TO germline split, the one arm rewritten to PASS, and SIGNATURES_BCFTOOLS_VIEW goes back to --apply-filters PASS with --exclude 'ALT="*"'. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 +-- conf/modules.config | 10 ++----- docs/usage.md | 21 ++++++------- modules/local/vcfsplit/main.nf | 34 +++++---------------- modules/local/vcftag/main.nf | 25 ++-------------- modules/local/vcftag/tests/main.nf.test | 35 ++++++++++------------ subworkflows/local/phasing_haplotyping.nf | 4 +-- tests/fixtures/vcftag_input.vcf | 2 ++ tests/fixtures/vcftag_input.vcf.gz | Bin 505 -> 546 bytes tests/fixtures/vcftag_input.vcf.gz.tbi | Bin 112 -> 112 bytes 10 files changed, 42 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce751704..c810c559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` records each ClairS-TO germline record's original `FILTER` in `INFO/ORIG_FILTER` before rewriting it to `PASS`. Germline calls remain published in full (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#201](https://github.com/IntGenomicsLab/lrsomatic/pull/201) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now pull the fork image from Docker Hub: `oras://docker.io/ljwharbers/clairs-to-sif:0.5.1-verdict-chm13-c0687e8-flat` under Singularity/Apptainer and `docker.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8-flat` otherwise, instead of `ghcr.io/ljwharbers/clairs-to`. The `-cpu` SIF on ghcr failed with `PROTOCOL_ERROR` on slow links: ghcr redirects every blob download to an Azure URL that expires at the next 5-minute mark and resets a stream still open then, and Apptainer resumes neither an `oras://` nor a `docker://` download. Docker Hub's download URLs are valid for 50 minutes and only checked when the request starts. `-flat` is the same software copied into an empty image in a few layers (3.3 GB instead of 7 GB); the software and its outputs are unchanged. `docs/usage.md` describes `pullTimeout`, Docker Hub's anonymous pull limit, pre-pulling, and how to recover the remaining `oras://ghcr.io` SIFs resumably (@ljwharbers). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index ca462202..6cd1516f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -753,14 +753,8 @@ process { } withName : '.*:SIGNATURES_BCFTOOLS_VIEW' { - // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only. - // The PASS test reads INFO/ORIG_FILTER, not FILTER: VCFTAG normalises FILTER to PASS on - // every record before phasing, so --apply-filters PASS here could never remove anything and - // non-PASS records reached SigProfiler whenever --smallvar_filter_pass was false. - // ORIG_FILTER carries the caller's own verdict, so the filter bites again. "." is kept - // because an unfiltered record is not a rejected one. bcftools allows only one of -i/-e, - // so the ALT="*" exclusion is folded into the same expression. - ext.args = { "-i '(INFO/ORIG_FILTER=\"PASS\" || INFO/ORIG_FILTER=\".\") && ALT!=\"*\"' --types snps,mnps,indels --output-type v" } + // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only + ext.args = { "--apply-filters PASS --types snps,mnps,indels --exclude 'ALT=\"*\"' --output-type v" } ext.prefix = { "${meta.id}_somatic_pass" } publishDir = [ enabled: false diff --git a/docs/usage.md b/docs/usage.md index 00a0c28e..b9ab4571 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -425,13 +425,10 @@ rather than `PASS`-filtered. The per-caller VCFs published under `//variants/` are never filtered, so no calls are lost from the results directory. -Set it to `false` to restore the previous unfiltered behaviour. `VCFTAG` -normalises `FILTER` to `PASS` on both arms before phasing, so the caller's own -verdict is preserved in `INFO/ORIG_FILTER` and the signature input is filtered on -that field rather than on `FILTER`; without this the `PASS` filter in front of -SigProfiler could never remove anything once `FILTER` had been rewritten. Every -published VCF downstream of phasing therefore reads `FILTER=PASS`, with the -original value available in `INFO/ORIG_FILTER`. +Set it to `false` to restore the previous unfiltered behaviour: each caller's +records are passed on with their original `FILTER`. Only the ClairS-TO germline +split is normalised to `PASS`, with its original value kept in +`INFO/ORIG_FILTER`. `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only @@ -453,11 +450,11 @@ every record. Three INFO fields carry this provenance: -| Field | Meaning | -| ------------- | ---------------------------------------------------------------------------------------------------------------- | -| `SOMATIC` | Record came from the somatic call set | -| `GERMLINE` | Record came from the germline call set | -| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS`. Multiple filters are joined with `,` | +| Field | Meaning | +| ------------- | --------------------------------------------------------------------------------------------------------------------- | +| `SOMATIC` | Record came from the somatic call set | +| `GERMLINE` | Record came from the germline call set | +| `ORIG_FILTER` | Original `FILTER` of ClairS-TO germline records, before normalisation to `PASS`. Multiple filters are joined with `,` | Germline calls dropped from `variants/phased/somatic_smallvariants.vcf.gz` are not lost: they remain in `variants/phased/germline_smallvariants.vcf.gz`, in diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index d773494a..2b627223 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -28,21 +28,7 @@ process VCFSPLIT { bcftools view -i 'FILTER="PASS"' $snv_vcf | bgzip -c > snv_pass.vcf.gz tabix -p vcf indels_pass.vcf.gz tabix -p vcf snv_pass.vcf.gz - bcftools concat -a -Oz -o somatic_tmp.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz - tabix -p vcf somatic_tmp.vcf.gz - - # Record the caller's original FILTER in INFO. These records are already PASS, but stamping - # both splits keeps them symmetric and self-describing alongside the germline arm below. - # The header line's double quotes arrive via -v q. Escaped quotes inside a Nextflow script - # block are fragile, and losing the escape silently produces an unparseable VCF header. - # FILTER is ";"-delimited but ";" separates INFO fields, so it is stored as ",". - bcftools view somatic_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} - /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } - { of = \$7; gsub(/;/, ",", of) - \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of - print } - ' | bgzip -c > somatic.vcf.gz + bcftools concat -a -Oz -o somatic.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz tabix -p vcf somatic.vcf.gz bcftools view -i 'FILTER~"NonSomatic" || INFO/Verdict_Germline=1' $indel_vcf | bgzip -c > indels_filtered.vcf.gz @@ -52,15 +38,11 @@ process VCFSPLIT { bcftools concat -a -Oz -o germline_tmp.vcf.gz indels_filtered.vcf.gz snv_filtered.vcf.gz tabix -p vcf germline_tmp.vcf.gz - # FILTER is normalised to PASS so that downstream tools which filter on PASS -- implicitly or - # otherwise -- see every germline record. The caller's original FILTER (typically NonSomatic) - # would otherwise be destroyed here, which is what previously made germline records - # indistinguishable from somatic ones once the two sets were merged for phasing; it is kept in - # INFO/ORIG_FILTER instead. Germline/somatic provenance itself is stamped later, by - # PHASING_HAPLOTYPING:TAG_GERMLINE / TAG_SOMATIC, which covers callers that bypass VCFSPLIT. + # Normalise FILTER to PASS and keep the original in INFO/ORIG_FILTER (";" stored as ","). + # The header's double quotes arrive via -v q, avoiding fragile escapes in the script block. bcftools view germline_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } + /^#CHROM/ { print "##INFO="; print; next } { of = \$7; gsub(/;/, ",", of) \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of \$7 = "PASS" @@ -68,15 +50,13 @@ process VCFSPLIT { ' | bgzip -c > germline.vcf.gz tabix -p vcf germline.vcf.gz - # Read both headers back. tabix will happily index a VCF whose header htslib cannot parse, so - # without this a malformed header surfaces as a confusing failure in a later process instead - # of here. set -e is in effect, so a bad header fails this task. + # Fail here, not downstream, if either header does not parse. bcftools view -h somatic.vcf.gz > /dev/null bcftools view -h germline.vcf.gz > /dev/null # Cleanup intermediate files - rm indels_pass.vcf.gz snv_pass.vcf.gz somatic_tmp.vcf.gz - rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi somatic_tmp.vcf.gz.tbi + rm indels_pass.vcf.gz snv_pass.vcf.gz + rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi """ stub: diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf index a7d81868..390122c6 100644 --- a/modules/local/vcftag/main.nf +++ b/modules/local/vcftag/main.nf @@ -23,41 +23,22 @@ process VCFTAG { script: prefix = task.ext.prefix ?: "${meta.id}_${flag.toLowerCase()}" """ - # Stamp a constant INFO flag recording which arm this record came from, and normalise FILTER - # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and - # filter on FILTER, so a FILTER-based label does not survive to where it is needed. + # Stamp a constant INFO flag marking the call set; FILTER is left as the caller emitted it. # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. - # - # The caller's own FILTER is preserved in INFO/ORIG_FILTER before it is overwritten, so the - # verdict is not destroyed by the normalisation. FILTER is ";"-delimited but ";" separates INFO - # fields, so it is stored as ",". VCFSPLIT already stamps ORIG_FILTER on the ClairS-TO arm; both - # the header line and the per-record field are therefore added only when not already present, - # since a duplicate INFO key would make the record unparseable. - bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"; seen_orig=0} - /^##INFO=" - if (!seen_orig) { - print "##INFO=" - } print next } { - if (\$8 !~ /(^|;)ORIG_FILTER=/) { - of = \$7 - gsub(/;/, ",", of) - \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of - } - \$7 = "PASS" \$8 = (\$8 == "." || \$8 == "") ? flag : \$8 ";" flag print } ' | bgzip -c > ${prefix}.vcf.gz - # tabix will index a VCF whose header htslib cannot parse, so read it back here rather than - # letting a malformed header surface as a confusing failure in a later process. + # Fail here, not downstream, if the header does not parse. bcftools view -h ${prefix}.vcf.gz > /dev/null tabix -p vcf ${prefix}.vcf.gz diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test index 3f4a658f..c0ec94a6 100644 --- a/modules/local/vcftag/tests/main.nf.test +++ b/modules/local/vcftag/tests/main.nf.test @@ -10,7 +10,7 @@ nextflow_process { // Runs for real (no -stub): the tagging is an awk program embedded in the Nextflow script // block, so the escaping only holds if it is actually executed. - test("stamps the flag, declares its header and normalises FILTER") { + test("stamps the flag and declares its header without touching FILTER") { when { process { @@ -34,29 +34,24 @@ nextflow_process { def lines = all.findAll { line -> !line.startsWith('#') } def header = all.findAll { line -> line.startsWith('##') }.join('\n') + def filters = lines.collectEntries { l -> def f = l.split('\t'); [ (f[1]): f[6] ] } + def infos = lines.collectEntries { l -> def f = l.split('\t'); [ (f[1]): f[7] ] } + assertAll( // every input record survives -- tagging must not filter - { assert lines.size() == 4 }, + { assert lines.size() == 5 }, // the flag is declared, so bcftools can query it downstream { assert header.contains('ID=SOMATIC') }, - // every record carries the flag, including the one whose INFO was '.' - { assert lines.every { it.contains('SOMATIC') } }, - // FILTER is normalised to PASS on all records, including NonSomatic and RefCall - { assert lines.every { it.split('\t')[6] == 'PASS' } }, - // a pre-existing INFO field is preserved rather than overwritten - { assert lines.any { it.contains('EXISTING') && it.contains('CALLER=clairs-to') } }, - // the record whose INFO was '.' gets no leading '.', just the stamped fields - { assert lines.any { it.split('\t')[7] == 'ORIG_FILTER=RefCall;SOMATIC' } }, - // the caller's FILTER is preserved before being normalised away - { assert header.contains('ID=ORIG_FILTER') }, - { assert lines.every { it.contains('ORIG_FILTER=') } }, - { assert lines.any { it.contains('ORIG_FILTER=PASS') } }, - { assert lines.any { it.contains('ORIG_FILTER=NonSomatic') } }, - { assert lines.any { it.contains('ORIG_FILTER=RefCall') } }, - // a record whose FILTER was '.' records '.' rather than being skipped - { assert lines.any { it.contains('ORIG_FILTER=.') } }, - // ORIG_FILTER is stamped exactly once per record - { assert lines.every { it.split('\t')[7].count('ORIG_FILTER=') == 1 } } + // the flag is stamped exactly once per record, including the one whose INFO was '.' + { assert infos.values().every { it.split(';').count('SOMATIC') == 1 } }, + { assert infos['300'] == 'SOMATIC' }, + // FILTER is left exactly as the caller emitted it + { assert filters == [ '100':'PASS', '200':'NonSomatic', '300':'RefCall', '400':'.', '500':'LowQual;RefCall' ] }, + // pre-existing INFO is preserved, with the flag appended + { assert infos['200'] == 'CALLER=clairs-to;EXISTING;SOMATIC' }, + { assert infos['500'] == 'CALLER=clair3;EXISTING;SOMATIC' }, + // VCFTAG no longer adds ORIG_FILTER + { assert !all.any { it.contains('ORIG_FILTER') } } ) } } diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 3de385ef..40fb011e 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -144,8 +144,8 @@ workflow PHASING_HAPLOTYPING { // MODULE: VCFTAG (label: process_single), aliased TAG_SOMATIC / TAG_GERMLINE // Stamp each arm with an INFO provenance flag before the merge. This is the only point where // germline-vs-somatic origin is unambiguous for every caller: GERMLINE_CONSENSUS can emit - // records that never passed through VCFSPLIT, so tagging earlier would leave holes. After the - // merge the two populations are otherwise indistinguishable -- both carry FILTER=PASS. + // records that never passed through VCFSPLIT, so tagging earlier would leave holes. FILTER + // cannot tell the arms apart after the merge, so the flag carries provenance. // LongPhase preserves custom INFO keys, so the flags survive phasing (verified on v2.0.1). // TAG_SOMATIC ( somatic_vcf, 'SOMATIC' ) diff --git a/tests/fixtures/vcftag_input.vcf b/tests/fixtures/vcftag_input.vcf index 34fda4cc..a1945635 100644 --- a/tests/fixtures/vcftag_input.vcf +++ b/tests/fixtures/vcftag_input.vcf @@ -2,6 +2,7 @@ ##INFO= ##INFO= ##FILTER= +##FILTER= ##FILTER= ##contig= #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT testsample @@ -9,3 +10,4 @@ chr1 100 . A C 30 PASS CALLER=clairs-to GT:DP 0/1:20 chr1 200 . G T 20 NonSomatic CALLER=clairs-to;EXISTING GT:DP 0/1:18 chr1 300 . T A 10 RefCall . GT:DP 0/0:12 chr1 400 . C G 40 . CALLER=deepsomatic GT:DP 1/1:25 +chr1 500 . G A 5 LowQual;RefCall CALLER=clair3;EXISTING GT:DP 0/1:9 diff --git a/tests/fixtures/vcftag_input.vcf.gz b/tests/fixtures/vcftag_input.vcf.gz index 988819c22c88bc153d91397e8db80263c9cbe483..c956c3c5d4ce5c621969ebe1ab7248a6926ccd8f 100644 GIT binary patch literal 546 zcmb2|=3rp}f&Xj_PR>jW?u@Y)4(2sE2(Uf){7Xc7=c<)Q%yNT#IGR))S2*~t*XG;t za+7ztyKesd2Gf;uo*Deyf5v`VXc6PmOQz20&sm=zJtG$xCc)nMS!$ik`CcXaxkr{) z@!y*1|7jbey7Spo)5u0mll%$VFMG;;iq$5H`EXo#{Iveaf1b@YJ=e{jJv_zI#5=+5 zut?nwM?LK*uCT9u0&Y9!e+ycZvMj4@&kp`$UVdjw9naTKS6HoOaiP{*TFo&aH0b76 zk+c@0YjFYZ7YWl`v{yF%@Q#)IPn(GGpG~WLH5~TLGPDk}{ecSJ= U?1oZT&Pri%y)*+eI8lHI0I~Ds^Z)<= literal 505 zcmb2|=3rp}f&Xj_PR>jWCXBHc0<)V91lT`3{}SBtE9{_W^?fZ6S z+fkQE(UJTAUdY?k-QWv4HmA{{%kZ*k5#{^G^y?=~%1eqZH8YXkG$`I+ZUHL{{s zwcAhK9qOH-yzYwWPR861clbT6iWyu#YJ6QT-L}|IDd6RbRxzCyY|j?ol;{oWNh{vM z9p5Nck#g|U$Cv*SC0SN?itn9rblM&<{|5TzIE<&hk1xcXQ_i2S;UPlT%R{p`otT--6BcUf0b|oU_Md!IY&7=E{q;aZGA# zR=D!{^11#z>A&-lz`W};9Sw1slNjynB(jR?) h|F!eEB^Ifge7nt;pScx%4H&2B?v!R=2FEps008J@+w1@U diff --git a/tests/fixtures/vcftag_input.vcf.gz.tbi b/tests/fixtures/vcftag_input.vcf.gz.tbi index 3e8c3efd859874550e714bc5ac9aa7e101d7798f..75130768821ae4097ed26672d63124118d62a5d6 100644 GIT binary patch delta 34 qcmXRYm|!OLfN`P3jRUXtsvR_0)V0}Y0aM6c0S1PU?tIRP0SW*L?G51o delta 34 qcmXRYm|!NAz_n1~T~p?IPlMhSk92reFof!}Ffc^zjTE04pa1~jl?&_u From 75036a0e7e44db77439b08b0d88e1b1345c6a586 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:37:24 +0200 Subject: [PATCH 20/36] Prefilter DS_VERDICT_QUERY to the non-PASS/RefCall DeepSomatic rows Only GERMLINE/PON verdicts are consumed downstream, so the verdict table no longer carries the ~9M RefCall and PASS rows (13.4M -> 4.3M on B1975944). The filter excludes PASS/RefCall rather than including GERMLINE/PON, because bcftools errors on a FILTER ID the header does not declare. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 11 +++++------ subworkflows/local/tumor_only/tumoronly_smallvar.nf | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 6cd1516f..c1031053 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -168,10 +168,10 @@ process { // alias, so they do not disturb the generic BCFTOOLS_QUERY/ANNOTATE/VIEW entries below. withName: '.*:DS_VERDICT_QUERY' { ext.prefix = { "${meta.id}.ds_verdict" } - // FILTER is single-valued in DeepSomatic output (RefCall/GERMLINE/PON/PASS), so it cannot - // inject the ";" that would break INFO parsing once transferred. + // Keep only non-PASS/RefCall rows (GERMLINE/PON in practice). Excludes rather than includes, + // since naming an undeclared FILTER (PON is absent without a PON) makes bcftools fail. ext.args = { - "-f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" + "-e 'FILTER=\"PASS\" || FILTER=\"RefCall\"' -f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" } publishDir = [ enabled: false @@ -181,7 +181,7 @@ process { withName: '.*:DS_VERDICT_ANNOTATE' { ext.prefix = { "${meta.id}.deepvariant_verdict" } ext.args = { - '''-h <(echo '##INFO=') \ + '''-h <(echo '##INFO=') \ -c CHROM,POS,REF,ALT,INFO/DS_VERDICT \ -Oz \ -W=tbi''' @@ -193,8 +193,7 @@ process { withName: '.*:DS_GERMLINE_SELECT' { ext.prefix = { "${meta.id}.deepvariant_germline" } - // Keep only positively-adjudicated germline sites. RefCall and sites DeepSomatic never - // evaluated (DS_VERDICT absent) are dropped rather than assumed germline. + // Keep only DeepSomatic-adjudicated germline sites; all others lack DS_VERDICT or fail this test. ext.args = { "-i 'INFO/DS_VERDICT=\"GERMLINE\" || INFO/DS_VERDICT=\"PON\"' --output-type z --write-index=tbi" } diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index cb89b849..fefab476 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -231,8 +231,8 @@ workflow TUMORONLY_SMALLVAR { // (57,684) PASS -- the last being real somatic calls that must not be published as germline. // // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and - // unevaluated sites are dropped rather than assumed germline. The verdict stays in - // INFO/DS_VERDICT so the decision is auditable in the published VCF. + // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts + // are transferred to INFO/DS_VERDICT. // // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would @@ -240,7 +240,7 @@ workflow TUMORONLY_SMALLVAR { // // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter - // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER + // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER, non-PASS/RefCall rows only // DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) From 18023f2bbad64f05939bdfacb13d240cd3d6e865 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:39:10 +0200 Subject: [PATCH 21/36] Rejoin split multi-allelics after the caller consensus BCFTOOLS_NORM splits multi-allelics (-m -any) so isec can match per ALT, which left two records at one POS in the VCF handed to LongPhase and Wakhan. BCFTOOLS_NORM_REJOIN (-m +any) now rejoins them at the end of SMALL_VARIANT_CONSENSUS on both the consensus and 'all' paths. The final file keeps its _{germline,somatic}_sorted name; the sort steps now write *_split_sorted. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 20 ++++++++++--- subworkflows/local/small_variant_consensus.nf | 28 +++++++++++++++++-- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c1031053..a71c5c53 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -126,6 +126,7 @@ process { // A>G never intersects: in consensus mode the variant is dropped, in union mode it appears // twice. ClairS-TO emits no multi-allelic records at all while DeepVariant emits ~2.5%, so // every DeepVariant multi-allelic site was systematically excluded from the consensus. + // BCFTOOLS_NORM_REJOIN rejoins the split records after isec, before phasing. ext.args = { "-m -any -Oz" } @@ -133,6 +134,17 @@ process { enabled: false ] } + // Rejoin split multi-allelics; LongPhase keys variants by position and cannot take two records at one POS. + withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_NORM_REJOIN' { + ext.prefix = { "${meta.id}_germline_sorted" } + ext.args = { '-m +any --output-type z --write-index=tbi' } + publishDir = [ enabled: false ] + } + withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_NORM_REJOIN' { + ext.prefix = { "${meta.id}_somatic_sorted" } + ext.args = { '-m +any --output-type z --write-index=tbi' } + publishDir = [ enabled: false ] + } withName: '.*:BCFTOOLS_ISEC' { ext.prefix = { "${meta.id}_isec" } @@ -553,26 +565,26 @@ process { ] } withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_SORT' { - ext.prefix = { "${meta.id}_germline_sorted" } + ext.prefix = { "${meta.id}_germline_split_sorted" } ext.args = {'-Oz -W=tbi'} publishDir = [ enabled: false ] } withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_SORT' { - ext.prefix = { "${meta.id}_somatic_sorted" } + ext.prefix = { "${meta.id}_somatic_split_sorted" } ext.args = {'-Oz -W=tbi'} publishDir = [ enabled: false ] } withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_SORT_CONSENSUS' { - ext.prefix = { "${meta.id}_germline_sorted" } + ext.prefix = { "${meta.id}_germline_split_sorted" } ext.args = { '-Oz -W=tbi' } publishDir = [ enabled: false ] } withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_SORT_CONSENSUS' { - ext.prefix = { "${meta.id}_somatic_sorted" } + ext.prefix = { "${meta.id}_somatic_split_sorted" } ext.args = { '-Oz -W=tbi' } publishDir = [ enabled: false ] } diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index d50ba7f7..4717cb69 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -1,4 +1,5 @@ include { BCFTOOLS_NORM } from '../../modules/nf-core/bcftools/norm/main' +include { BCFTOOLS_NORM as BCFTOOLS_NORM_REJOIN } from '../../modules/nf-core/bcftools/norm/main' include { BCFTOOLS_ISEC } from '../../modules/nf-core/bcftools/isec/main' include { BCFTOOLS_QUERY } from '../../modules/nf-core/bcftools/query/main' include { BCFTOOLS_ANNOTATE } from '../../modules/nf-core/bcftools/annotate/main' @@ -21,7 +22,7 @@ workflow SMALL_VARIANT_CONSENSUS { main: // - // MODULE: BCFTOOLS_NORM (label: process_medium) -- left-align and normalise; sorted after, since left-alignment can reorder records + // MODULE: BCFTOOLS_NORM (label: process_medium) -- left-align and split multi-allelics for isec; rejoined before phasing // Input: [meta, vcf, tbi] -- per-caller VCF // Output: .vcf -- [meta, vcf] -- left-aligned, normalised VCF (unsorted) // @@ -54,6 +55,8 @@ workflow SMALL_VARIANT_CONSENSUS { // Renaming cannot fix that and `annotate -h` cannot override an existing FORMAT definition, // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record // reaching here carries one ALT and one AF value, making the two declarations equivalent. + // After BCFTOOLS_NORM_REJOIN, consensus files keep the source caller's own header (lossless round + // trip); 'all' mode takes the Number=A header of isec's 0000.vcf.gz, which concat lists first. // // The rename is carried out by BCFTOOLS_ANNOTATE below rather than by a second annotate call: // meta.rename_to selects the --rename-annots file in conf/modules.config. Only 'all' mode needs @@ -266,8 +269,27 @@ workflow SMALL_VARIANT_CONSENSUS { error("combine_method must be 'consensus' or 'all', got '${combine_method}'") } + // + // MODULE: BCFTOOLS_NORM_REJOIN (BCFTOOLS_NORM alias) -- rejoin split sites (-m +any) so LongPhase and Wakhan see one record per position + // Input: [meta, vcf, tbi] -- sorted consensus/union VCF + // Output: .vcf -- [meta, vcf.gz] + // .tbi -- [meta, tbi] + // + BCFTOOLS_NORM_REJOIN( + vcf.join(tbi, failOnMismatch: true, failOnDuplicate: true), + fasta + ) + + BCFTOOLS_NORM_REJOIN.out.vcf + .join(BCFTOOLS_NORM_REJOIN.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .multiMap { meta, rejoined_vcf, rejoined_tbi -> + vcf: [meta, rejoined_vcf] + tbi: [meta, rejoined_tbi] + } + .set { rejoined } + emit: - vcf // [meta, vcf] -- final consensus/combined VCF - tbi // [meta, tbi] + vcf = rejoined.vcf // [meta, vcf] -- final consensus/combined VCF, multi-allelics rejoined + tbi = rejoined.tbi // [meta, tbi] } From 6a3e297208bb874fe60dcce485400c5bfdd0bf07 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:41:27 +0200 Subject: [PATCH 22/36] Join ASCAT and Wakhan report files instead of counting with groupKey groupKey(meta, 3) counted optional outputs, so a sample missing one was silently dropped at channel close. ASCAT now joins its required outputs and adds segments_raw with remainder: true. Wakhan always writes a solution_* link for every row of solutions_ranks.tsv, so solution_dirs is now required and the three outputs are joined. Co-Authored-By: Claude Opus 5.5 (1M context) --- modules/local/wakhan/main.nf | 2 +- workflows/lrsomatic.nf | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/local/wakhan/main.nf b/modules/local/wakhan/main.nf index b461baaf..517ae995 100644 --- a/modules/local/wakhan/main.nf +++ b/modules/local/wakhan/main.nf @@ -38,7 +38,7 @@ process WAKHAN { tuple val(meta), path("solutions_ranks.tsv") , emit: solutions_ranks // Whole directories, not the plots inside: every solution's plot has the same basename, // and LRSOMATICREPORT resolves them by solution_/ path - tuple val(meta), path("solution_*", type: 'dir') , emit: solution_dirs, optional: true + tuple val(meta), path("solution_*", type: 'dir') , emit: solution_dirs // WARN: Manually update version information as tool does not provide on CLI tuple val("${task.process}"), val('wakhan'), val("0.4.4"), topic: versions, emit: versions_wakhan diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index acf98c0d..03517069 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -696,12 +696,11 @@ workflow LRSOMATIC { // ascat_tumoronly_ch: [meta, purityploidy, segments] // All ASCAT files per sample for the report module, which globs by suffix - // groupKey: release each sample on its own three emissions, not when ASCAT finishes for all - ch_ascat_files = ASCAT.out.segments_raw - .mix(ASCAT.out.purityploidy, ASCAT.out.png) - .map { meta, files -> [groupKey(meta, 3), files] } - .groupTuple() - .map { meta, files -> [meta, files.flatten()] } + // Joined per sample; segments_raw is optional, so it arrives as null when absent + ch_ascat_files = ASCAT.out.purityploidy + .join(ASCAT.out.png) + .join(ASCAT.out.segments_raw, remainder: true) + .map { meta, purityploidy, png, segments_raw -> [meta, [purityploidy, png, segments_raw ?: []].flatten()] } // ch_ascat_files: [meta, [file, file, ...]] } @@ -1257,12 +1256,11 @@ workflow LRSOMATIC { ) // The WAKHAN outputs the report renders: ranked solutions, heatmap, per-solution plots - // groupKey: release each sample on its own three emissions + // Joined per sample; all three outputs are required ch_wakhan_files = WAKHAN.out.solutions_ranks - .mix(WAKHAN.out.heatmap_html, WAKHAN.out.solution_dirs) - .map { meta, files -> [groupKey(meta, 3), files] } - .groupTuple() - .map { meta, files -> [meta, files.flatten()] } // solution_dirs contributes a list + .join(WAKHAN.out.heatmap_html) + .join(WAKHAN.out.solution_dirs) + .map { meta, ranks, heatmap, dirs -> [meta, [ranks, heatmap, dirs].flatten()] } // dirs may be a list // ch_wakhan_files: [meta, [file_or_dir, ...]] } From d653bcbd7a6eb19ecc8629efb88f5b32a2c51fa4 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:43:32 +0200 Subject: [PATCH 23/36] Inline the PASS filter as BCFTOOLS_VIEW aliases at each caller Drop the one-module VCF_PASS_FILTER subworkflow; each call site now picks between the filtered copy and its input on params.smallvar_filter_pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 42 ++++--------------- .../local/paired/paired_smallvar_germline.nf | 34 ++++++++------- .../local/paired/paired_smallvar_somatic.nf | 34 ++++++++------- .../local/tumor_only/tumoronly_smallvar.nf | 38 ++++++++--------- subworkflows/local/vcf_pass_filter.nf | 41 ------------------ tests/union.nf.test | 5 +-- 6 files changed, 63 insertions(+), 131 deletions(-) delete mode 100644 subworkflows/local/vcf_pass_filter.nf diff --git a/conf/modules.config b/conf/modules.config index a71c5c53..58d2116f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -772,47 +772,19 @@ process { ] } - // VCF_PASS_FILTER copies, one alias per small variant caller. DeepVariant and - // DeepSomatic emit every evaluated site rather than only their calls, and Clair3/ClairS - // keep their LowQual and NonSomatic records; these filtered copies are what the - // consensus, phasing, VEP and report steps consume. The per-caller VCFs under - // variants// are published from the caller processes and are left untouched. - // ClairS-TO has no entry here: VCFSPLIT already restricts its SOMATIC split to PASS. - // Note this is not true of the germline split, whose FILTER is rewritten to PASS rather - // than filtered; germline/somatic origin is carried in INFO by VCFTAG instead. - // --write-index is required because the module's index output is optional, and the - // downstream join() would otherwise drop every sample. - withName: '.*:DEEPVARIANT_PASS_FILTER:PASS_FILTER' { + // PASS-only per-caller copies fed to consensus/phasing/VEP; ClairS-TO is covered by VCFSPLIT. + // --write-index=tbi is required: the module's index output is optional and the join needs it. + withName: '.*:(CLAIR3|CLAIRS|DEEPVARIANT|DEEPSOMATIC)_PASS_FILTER' { ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_deepvariant_pass" } publishDir = [ enabled: false ] } - withName: '.*:DEEPSOMATIC_PASS_FILTER:PASS_FILTER' { - ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_deepsomatic_pass" } - publishDir = [ - enabled: false - ] - } - - withName: '.*:CLAIR3_PASS_FILTER:PASS_FILTER' { - ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_clair3_pass" } - publishDir = [ - enabled: false - ] - } - - withName: '.*:CLAIRS_PASS_FILTER:PASS_FILTER' { - ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_clairs_pass" } - publishDir = [ - enabled: false - ] - } + withName: '.*:CLAIR3_PASS_FILTER' { ext.prefix = { "${meta.id}_clair3_pass" } } + withName: '.*:CLAIRS_PASS_FILTER' { ext.prefix = { "${meta.id}_clairs_pass" } } + withName: '.*:DEEPVARIANT_PASS_FILTER' { ext.prefix = { "${meta.id}_deepvariant_pass" } } + withName: '.*:DEEPSOMATIC_PASS_FILTER' { ext.prefix = { "${meta.id}_deepsomatic_pass" } } withName : '.*:SIGPROFILER_MATRIXGENERATOR' { ext.args = { params.sigprofiler_matrix_args ?: '' } diff --git a/subworkflows/local/paired/paired_smallvar_germline.nf b/subworkflows/local/paired/paired_smallvar_germline.nf index 798e5b3f..3164e33b 100644 --- a/subworkflows/local/paired/paired_smallvar_germline.nf +++ b/subworkflows/local/paired/paired_smallvar_germline.nf @@ -1,11 +1,11 @@ // IMPORT MODULES +include { BCFTOOLS_VIEW as CLAIR3_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' include { CLAIR3 } from '../../../modules/local/clair3/main.nf' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' -include { VCF_PASS_FILTER as CLAIR3_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' -include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_GERMLINE { @@ -75,14 +75,15 @@ workflow PAIRED_SMALLVAR_GERMLINE { fai ) - // Clair3's merge_output.vcf.gz keeps LowQual and RefCall records; restrict them the - // same way DeepVariant's output is restricted so both callers enter the union on - // equal terms. The VCF published under variants/clair3/ is unaffected. - CLAIR3_PASS_FILTER ( - CLAIR3.out.vcf.join(CLAIR3.out.tbi) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def clair3_vcf = CLAIR3.out.vcf.join(CLAIR3.out.tbi) + if (params.smallvar_filter_pass) { + CLAIR3_PASS_FILTER ( clair3_vcf, [], [], [] ) + clair3_vcf = CLAIR3_PASS_FILTER.out.vcf + .join(CLAIR3_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - CLAIR3_PASS_FILTER.out.vcf + clair3_vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clair3'] return [new_meta, vcf, tbi] @@ -127,14 +128,15 @@ workflow PAIRED_SMALLVAR_GERMLINE { [[:],[]] // GFF annotation (not used) ) - // DeepVariant emits a record for every site it evaluates, not just its calls, so - // most records are RefCall. Without this the union is "every site DeepVariant - // looked at". The VCF published under variants/deepvariant/ is unaffected. - DEEPVARIANT_PASS_FILTER ( - DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepvariant_vcf = DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_vcf, [], [], [] ) + deepvariant_vcf = DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DEEPVARIANT_PASS_FILTER.out.vcf + deepvariant_vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/paired/paired_smallvar_somatic.nf b/subworkflows/local/paired/paired_smallvar_somatic.nf index 953ed01d..ab296b99 100644 --- a/subworkflows/local/paired/paired_smallvar_somatic.nf +++ b/subworkflows/local/paired/paired_smallvar_somatic.nf @@ -1,4 +1,6 @@ // IMPORT MODULES +include { BCFTOOLS_VIEW as CLAIRS_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' include { CLAIRS } from '../../../modules/local/clairs/main.nf' include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat' include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort' @@ -6,8 +8,6 @@ include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/so // IMPORT SUBWORKFLOWS include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' -include { VCF_PASS_FILTER as CLAIRS_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' -include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_SOMATIC { @@ -73,14 +73,15 @@ workflow PAIRED_SMALLVAR_SOMATIC { BCFTOOLS_CONCAT.out.vcf ) - // The concatenated ClairS VCF still carries its LowQual and NonSomatic calls; - // restrict them the same way DeepSomatic's output is restricted so both callers - // enter the union on equal terms. variants/clairs/ is unaffected. - CLAIRS_PASS_FILTER ( - BCFTOOLS_SORT.out.vcf.join(BCFTOOLS_SORT.out.tbi) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def clairs_vcf = BCFTOOLS_SORT.out.vcf.join(BCFTOOLS_SORT.out.tbi) + if (params.smallvar_filter_pass) { + CLAIRS_PASS_FILTER ( clairs_vcf, [], [], [] ) + clairs_vcf = CLAIRS_PASS_FILTER.out.vcf + .join(CLAIRS_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - CLAIRS_PASS_FILTER.out.vcf + clairs_vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clairs'] return [new_meta, vcf, tbi] @@ -117,14 +118,15 @@ workflow PAIRED_SMALLVAR_SOMATIC { ds_pon_channel ) - // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), - // not just its calls. Without this the union is "every site DeepSomatic looked - // at". The VCF published under variants/deepsomatic/ is unaffected. - DEEPSOMATIC_PASS_FILTER ( - DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepsomatic_vcf = DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_vcf, [], [], [] ) + deepsomatic_vcf = DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DEEPSOMATIC_PASS_FILTER.out.vcf + deepsomatic_vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index fefab476..e4f90f5c 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -1,4 +1,6 @@ // IMPORT MODULES +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' include { CLAIRSTO } from '../../../modules/local/clairsto/main.nf' include { CLAIRSTO_VERDICT_TAG } from '../../../modules/local/clairsto/verdict_tag/main.nf' include { VCFSPLIT } from '../../../modules/local/vcfsplit/main.nf' @@ -8,8 +10,6 @@ include { DEEPVARIANT } from '../../../subwork include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' -include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' -include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' // Germline verdict transfer: DeepSomatic adjudicates DeepVariant's tumor-derived germline calls. // Three independent bcftools invocations, so three aliased instances of the upstream modules. @@ -165,15 +165,15 @@ workflow TUMORONLY_SMALLVAR { [[:],[]], // GZI (empty if FASTA is uncompressed) ds_pon_channel ) - // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), - // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT - // already restricts it to PASS. The VCF published under variants/deepsomatic/ is - // unaffected. - DEEPSOMATIC_PASS_FILTER ( - DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepsomatic_vcf = DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_vcf, [], [], [] ) + deepsomatic_vcf = DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DEEPSOMATIC_PASS_FILTER.out.vcf + deepsomatic_vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] @@ -208,15 +208,13 @@ workflow TUMORONLY_SMALLVAR { [[:],[]] // GFF annotation (not used) ) - // DeepVariant emits a record for every site it evaluates, not just its calls, so - // most records are RefCall. ClairS-TO needs no equivalent step here because - // VCFSPLIT already restricts its SOMATIC split to PASS -- note that its GERMLINE split - // is not PASS-filtered but PASS-rewritten, so a PASS filter would not reduce it and - // germline/somatic origin is carried in INFO by VCFTAG instead. The VCF published under - // variants/deepvariant/ is unaffected. - DEEPVARIANT_PASS_FILTER ( - DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepvariant_vcf = DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_vcf, [], [], [] ) + deepvariant_vcf = DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } // GERMLINE VERDICT TRANSFER (tumor-only, deep family) // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is @@ -248,7 +246,7 @@ workflow TUMORONLY_SMALLVAR { // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. // - DEEPVARIANT_PASS_FILTER.out.vcf + deepvariant_vcf .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) .map { meta, vcf, tbi, annotations, annotations_index -> diff --git a/subworkflows/local/vcf_pass_filter.nf b/subworkflows/local/vcf_pass_filter.nf deleted file mode 100644 index f52e0cb9..00000000 --- a/subworkflows/local/vcf_pass_filter.nf +++ /dev/null @@ -1,41 +0,0 @@ -// IMPORT MODULES -// Aliased so the software-versions report keeps this separate from the local -// BCFTOOLS_VIEW used by PHASING_HAPLOTYPING, which is a different bcftools build. -include { BCFTOOLS_VIEW as PASS_FILTER } from '../../modules/nf-core/bcftools/view/main' - -// -// SUBWORKFLOW: VCF_PASS_FILTER -// Restrict a per-caller VCF to its PASS records before it is handed to the caller -// consensus, phasing, VEP and the report. DeepVariant and DeepSomatic emit a record for -// every site they evaluate (RefCall/GERMLINE/PON), and Clair3/ClairS keep their LowQual -// and NonSomatic calls, so without this the union is "every site every caller looked at". -// The per-caller VCFs published under variants// are produced elsewhere and are -// never filtered, so no calls are lost from the results directory. -// -// Include once per caller under an alias so conf/modules.config can give each one its own -// prefix, e.g. include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER }. -// -workflow VCF_PASS_FILTER { - - take: - vcfs // [meta, vcf, tbi] - - main: - if (params.smallvar_filter_pass) { - // The module declares `emit: index, optional: true`, so the index only exists because - // conf/modules.config puts --write-index=tbi in ext.args for every alias. failOnMismatch - // turns a missing index into an immediate error instead of silently dropping the sample. - PASS_FILTER ( vcfs, [], [], [] ) - - PASS_FILTER.out.vcf - .join(PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) - .set{ filtered } - } - else { - vcfs - .set{ filtered } - } - - emit: - vcf = filtered // [meta, vcf, tbi] -} diff --git a/tests/union.nf.test b/tests/union.nf.test index aa14800b..8acb4956 100644 --- a/tests/union.nf.test +++ b/tests/union.nf.test @@ -134,12 +134,11 @@ nextflow_pipeline { { assert workflow.success }, // ── No PASS filter process may run ─────────────────────────── - // VCF_PASS_FILTER passes its input straight through when the param is - // false, so PASS_FILTER never executes and never reports a version. + // With the param false no _PASS_FILTER process runs or reports a version. { def versions = file("$outputDir/pipeline_info/lrsomatic_software_mqc_versions.yml") assert versions.exists() - assert !versions.text.contains('PASS_FILTER') + assert !versions.text.contains('_PASS_FILTER') }, // ── Phased VCFs still exist and have data ──────────────────── From c91a23e70fd4ca9b82d4291ca9e2deaf5cc64ac9 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:45:10 +0200 Subject: [PATCH 24/36] Add VCFTAG meta.yml and fix var_keep defaults in usage.md Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/usage.md | 4 +- modules/local/vcftag/meta.yml | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 modules/local/vcftag/meta.yml diff --git a/docs/usage.md b/docs/usage.md index b9ab4571..afe0e4ee 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -401,8 +401,8 @@ These options control how variants from multiple callers are filtered and merged | Parameter | Description | | ------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| `--germline_var_keep` | Expression or threshold for retaining germline variants after calling. Default = `null` | -| `--somatic_var_keep` | Expression or threshold for retaining somatic variants after calling. Default = `null` | +| `--germline_var_keep` | Comma-separated germline callers to run: `deepvariant`, `clair`. Default = `clair` | +| `--somatic_var_keep` | Comma-separated somatic callers to run: `deepsomatic`, `clair`. Default = `clair` | | `--germline_var_combine` | How to combine germline caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | | `--somatic_var_combine` | How to combine somatic caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | | `--prioritize_caller_germline` | Whose record to use for variants called by both germline callers: `deepvariant` or `clair`. Default = `clair` | diff --git a/modules/local/vcftag/meta.yml b/modules/local/vcftag/meta.yml new file mode 100644 index 00000000..648ca689 --- /dev/null +++ b/modules/local/vcftag/meta.yml @@ -0,0 +1,75 @@ +name: vcftag +description: Stamp a constant INFO flag, named by the flag input, on every record of a VCF; FILTER is left untouched +keywords: + - vcf + - annotation + - INFO + - variant calling +tools: + - bcftools: + description: Tools for variant calling and manipulating VCFs and BCFs + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + tool_dev_url: https://github.com/samtools/bcftools + doi: "10.1093/gigascience/giab008" + licence: ["MIT"] + identifier: biotools:bcftools +input: + - - meta: + type: map + description: Groovy Map containing sample information e.g. [ id:'test' ] + - vcf: + type: file + description: Input VCF file to tag + pattern: "*.vcf.gz" + - tbi: + type: file + description: Tabix index of the input VCF + pattern: "*.tbi" + - flag: + type: string + description: | + Name of the INFO flag added to every record (e.g. `GERMLINE`); + its lowercase form is also used in the default output prefix +output: + vcf: + - - meta: + type: map + description: Groovy Map containing sample information + - ${prefix}.vcf.gz: + type: file + description: VCF with the INFO flag added to every record + pattern: "*.vcf.gz" + tbi: + - - meta: + type: map + description: Groovy Map containing sample information + - ${prefix}.vcf.gz.tbi: + type: file + description: Tabix index of the tagged VCF + pattern: "*.vcf.gz.tbi" + versions_bcftools: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version |& sed '1!d ; s/bcftools //'": + type: string + description: The command used to generate the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version |& sed '1!d ; s/bcftools //'": + type: string + description: The command used to generate the version of the tool +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" From 5fae9328ba7a34cd02a1c091813342cd08c0a687 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:46:56 +0200 Subject: [PATCH 25/36] Allow deepvariant without deepsomatic; skip the verdict transfer then Tumour-only DeepVariant germline calls are adjudicated with DeepSomatic's verdict only when deepsomatic is selected; otherwise they pass through. Drops validateDeepCallerPairing, which rejected valid paired-only runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- docs/usage.md | 8 +- .../local/tumor_only/tumoronly_smallvar.nf | 108 +++++++++--------- .../utils_nfcore_lrsomatic_pipeline/main.nf | 24 ---- 4 files changed, 62 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c810c559..4fded0de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. The verdict is applied only when `deepsomatic` is also selected; with `deepvariant` alone the germline calls are used unfiltered by verdict and may include somatic variants. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` records each ClairS-TO germline record's original `FILTER` in `INFO/ORIG_FILTER` before rewriting it to `PASS`. Germline calls remain published in full (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). diff --git a/docs/usage.md b/docs/usage.md index afe0e4ee..0c686ce9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -138,7 +138,7 @@ For tumour-only small variants, ClairS-TO separates germline from somatic calls When `--germline_var_keep` includes `deepvariant`, the tumour-only germline arm runs DeepVariant on the **tumour** BAM. DeepVariant is a germline caller with no somatic discrimination, so on its own those calls mix germline and clonal somatic -variants. The pipeline therefore transfers DeepSomatic's verdict onto them: +variants. When DeepSomatic also runs, the pipeline transfers its verdict onto them: DeepSomatic evaluates the same sites and labels each `GERMLINE`, `PON`, `RefCall` or `PASS`, and that label is recorded in `INFO/DS_VERDICT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; @@ -146,9 +146,9 @@ adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; germline. On a 30x tumour-only sample this keeps about 83% of DeepVariant's `PASS` calls and removes roughly 1% that DeepSomatic positively calls somatic. -Because of this, `deepvariant` and `deepsomatic` must be enabled together: -`--germline_var_keep deepvariant` without `deepsomatic` in `--somatic_var_keep` is -rejected at launch. Note that even after adjudication the tumour-only germline arm +This restriction to sites DeepSomatic calls `GERMLINE` or `PON` applies only when +`deepsomatic` is also in `--somatic_var_keep`; without it, the DeepVariant germline +calls are used without any verdict filter and may include somatic variants. Note that even after adjudication the tumour-only germline arm is a tumour-derived proxy, not a call set from normal tissue, and should not be used for secondary findings without that caveat. diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index e4f90f5c..5ec14050 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -216,66 +216,72 @@ workflow TUMORONLY_SMALLVAR { .join(DEEPVARIANT_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) } - // GERMLINE VERDICT TRANSFER (tumor-only, deep family) - // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is - // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its - // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the - // germline VCF therefore carries most of the somatic call set. - // - // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic - // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO - // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 - // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% - // (57,684) PASS -- the last being real somatic calls that must not be published as germline. - // - // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and - // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts - // are transferred to INFO/DS_VERDICT. - // - // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified - // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would - // break INFO parsing. - // - // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) - // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter - // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER, non-PASS/RefCall rows only - // - DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) + // Keep only DeepSomatic-adjudicated germline sites; skipped when deepsomatic isn't selected. + def deepvariant_germline = deepvariant_vcf + if (somatic_var_keep.contains('deepsomatic')) { + // GERMLINE VERDICT TRANSFER (tumor-only, deep family) + // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is + // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its + // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the + // germline VCF therefore carries most of the somatic call set. + // + // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic + // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO + // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 + // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% + // (57,684) PASS -- the last being real somatic calls that must not be published as germline. + // + // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and + // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts + // are transferred to INFO/DS_VERDICT. + // + // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified + // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would + // break INFO parsing. + // + // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) + // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter + // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER, non-PASS/RefCall rows only + // + DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) - // - // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) - // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. - // - deepvariant_vcf - .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) - .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) - .map { meta, vcf, tbi, annotations, annotations_index -> - def columns = [] // no extra column specs - def header_lines = [] // no extra header lines - def rename_chrs = [] // no chromosome renaming - return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] - } - .set{ ds_verdict_annotate_input } + // + // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) + // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. + // + deepvariant_vcf + .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) + .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) + .map { meta, vcf, tbi, annotations, annotations_index -> + def columns = [] // no extra column specs + def header_lines = [] // no extra header lines + def rename_chrs = [] // no chromosome renaming + return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] + } + .set{ ds_verdict_annotate_input } - DS_VERDICT_ANNOTATE ( ds_verdict_annotate_input ) + DS_VERDICT_ANNOTATE ( ds_verdict_annotate_input ) - // - // MODULE: DS_GERMLINE_SELECT (BCFTOOLS_VIEW alias, label: process_medium) - // Keeps only the positively-adjudicated germline records (see ext.args in conf/modules.config). - // - DS_GERMLINE_SELECT ( - DS_VERDICT_ANNOTATE.out.vcf.join(DS_VERDICT_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true), - [], [], [] - ) + // + // MODULE: DS_GERMLINE_SELECT (BCFTOOLS_VIEW alias, label: process_medium) + // Keeps only the positively-adjudicated germline records (see ext.args in conf/modules.config). + // + DS_GERMLINE_SELECT ( + DS_VERDICT_ANNOTATE.out.vcf.join(DS_VERDICT_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true), + [], [], [] + ) + + deepvariant_germline = DS_GERMLINE_SELECT.out.vcf + .join(DS_GERMLINE_SELECT.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DS_GERMLINE_SELECT.out.vcf - .join(DS_GERMLINE_SELECT.out.index, failOnMismatch: true, failOnDuplicate: true) + deepvariant_germline .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] } .set{deepvariant_ch} - // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] -- germline-adjudicated only + // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] -- germline-adjudicated if deepsomatic ran } // COMBINE GERMLINE VARIANTS diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index 072a9d5f..a64c405c 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -239,30 +239,6 @@ workflow PIPELINE_COMPLETION { def validateInputParameters() { genomeExistsError() validateReportGenePanels() - validateDeepCallerPairing() -} - -// -// The tumor-only germline arm runs DeepVariant on the TUMOR BAM, which cannot separate germline -// from clonal somatic calls on its own. TUMORONLY_SMALLVAR resolves that by transferring -// DeepSomatic's FILTER verdict (GERMLINE/PON/RefCall/PASS) onto those calls, so DeepVariant -// without DeepSomatic would leave the germline arm unadjudicated -- and would reference -// DEEPSOMATIC.out before it exists. Require the deep family to be fully on or fully off. -// -def callerList(value) { - if (value instanceof List) { - return value - } - return value.toString().tokenize(',').collect { token -> token.trim() } -} - -def validateDeepCallerPairing() { - def germline = callerList(params.germline_var_keep) - def somatic = callerList(params.somatic_var_keep) - - if (germline.contains('deepvariant') && !somatic.contains('deepsomatic')) { - error("--germline_var_keep includes 'deepvariant' but --somatic_var_keep does not include 'deepsomatic'. In tumor-only mode the DeepVariant germline calls are adjudicated with DeepSomatic's verdict, so the two must be enabled together. Add 'deepsomatic' to --somatic_var_keep, or drop 'deepvariant' from --germline_var_keep.") - } } // From 00a9312d32cacd5881ea3246af145bd8e62e2ea0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:50:44 +0200 Subject: [PATCH 26/36] Cut long comments, sample numbers and CHANGELOG write-ups to one or two lines Addresses review point 9 on #200. Comments in the consensus, verdict transfer, phasing, VCFSPLIT and modules.config blocks now say what each step does in a line or two; sample IDs and cohort counts are removed from source and docs. The #200 CHANGELOG entries are rewritten to match the length of the others, with short Fixed lines for the multi-allelic rejoin and the per-sample report. No code changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +++-- conf/modules.config | 27 ++++------- docs/usage.md | 48 +++++++------------ modules/local/vcfsplit/main.nf | 3 +- subworkflows/local/phasing_haplotyping.nf | 16 ++----- subworkflows/local/small_variant_consensus.nf | 33 +++---------- .../local/tumor_only/tumoronly_smallvar.nf | 21 +------- 7 files changed, 44 insertions(+), 116 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fded0de..51bfabd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. The verdict is applied only when `deepsomatic` is also selected; with `deepvariant` alone the germline calls are used unfiltered by verdict and may include somatic variants. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` records each ClairS-TO germline record's original `FILTER` in `INFO/ORIG_FILTER` before rewriting it to `PASS`. Germline calls remain published in full (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - In tumour-only mode with both `deepvariant` and `deepsomatic` selected, DeepVariant's germline calls are now kept only where DeepSomatic's verdict is `GERMLINE` or `PON` (`INFO/DS_VERDICT`); with `deepvariant` alone they are unfiltered and may include somatic variants (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The phased somatic VCF is now split from the merged germline+somatic VCF by an `INFO/SOMATIC` provenance flag (new `VCFTAG` module) instead of by position, so germline records at somatic coordinates no longer leak into it; ClairS-TO germline records keep their original `FILTER` in `INFO/ORIG_FILTER` (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Added `--smallvar_filter_pass` (default `true`), which restricts each small variant caller's output to `PASS` records before the caller consensus, phasing, VEP and the report; the published per-caller VCFs are unchanged (@robert-a-forsyth). - [#201](https://github.com/IntGenomicsLab/lrsomatic/pull/201) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now pull the fork image from Docker Hub: `oras://docker.io/ljwharbers/clairs-to-sif:0.5.1-verdict-chm13-c0687e8-flat` under Singularity/Apptainer and `docker.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8-flat` otherwise, instead of `ghcr.io/ljwharbers/clairs-to`. The `-cpu` SIF on ghcr failed with `PROTOCOL_ERROR` on slow links: ghcr redirects every blob download to an Azure URL that expires at the next 5-minute mark and resets a stream still open then, and Apptainer resumes neither an `oras://` nor a `docker://` download. Docker Hub's download URLs are valid for 50 minutes and only checked when the request starts. `-flat` is the same software copied into an empty image in a few layers (3.3 GB instead of 7 GB); the software and its outputs are unchanged. `docs/usage.md` describes `pullTimeout`, Docker Hub's anonymous pull limit, pre-pulling, and how to recover the remaining `oras://ghcr.io` SIFs resumably (@ljwharbers). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). @@ -45,7 +44,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3 `src/coverage/processing.py:121` averaged a BAF bin over a slice built from `.index()` lookups; because `.index()` searches by value the slice is empty whenever the position list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError`. The failure was deterministic, so retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now keeps both callers' private calls; the prioritized caller's own private calls were previously dropped. Invalid `combine_method`/`prioritize_caller` values now raise an error (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The caller consensus now splits multi-allelic records before intersecting, so they match across callers, and rejoins them before phasing (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `LRSOMATICREPORT` now renders each sample as soon as its own inputs are ready instead of waiting for the whole batch, including samples without ASCAT's optional raw segments output (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, fixing a deterministic `StatisticsError` crash in BAF binning that aborted the run (@robert-a-forsyth). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). - [#193](https://github.com/IntGenomicsLab/lrsomatic/pull/193) - `--vep_eve https://evemodel.org/api/proteins/bulk/download/` was rejected at launch because the "needs preparing" check keyed on a `.zip` suffix; it now checks whether the value is already a prepared bgzipped file (@AmberVerhasselt). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs a patched modkit 0.6.4 ([ljwharbers/modkit@pacbio-conflict-fix](https://github.com/ljwharbers/modkit/tree/pacbio-conflict-fix)): `ghcr.io/ljwharbers/modkit:0.6.4-pacbiofix-6e0afa2` under Docker and `oras://ghcr.io/ljwharbers/modkit-sif:0.6.4-pacbiofix-6e0afa2` under Singularity/Apptainer. Stock modkit 0.4.3-0.6.4 dropped 32-65 % of reads from recent PacBio HiFi BAMs and returned empty `--cpg` pileups ([nanoporetech/modkit#612](https://github.com/nanoporetech/modkit/issues/612), fix proposed in [nanoporetech/modkit#720](https://github.com/nanoporetech/modkit/pull/720)), and ignored `--phased`/`--modified-bases` for PacBio BAMs with 6mA calls. The image is `linux/amd64` only and Conda is not supported (use `--skip_modkit` there); return to the biocontainer once a release includes the fix (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 58d2116f..c87d5dff 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -121,12 +121,8 @@ process { withName: '.*:BCFTOOLS_NORM' { ext.prefix = { "${meta.id}.${meta.caller}_norm" } - // -m -any splits multi-allelic records into biallelic ones. BCFTOOLS_ISEC matches on exact - // CHROM/POS/REF/ALT, so without this a site one caller reports as A>G,GT and the other as - // A>G never intersects: in consensus mode the variant is dropped, in union mode it appears - // twice. ClairS-TO emits no multi-allelic records at all while DeepVariant emits ~2.5%, so - // every DeepVariant multi-allelic site was systematically excluded from the consensus. - // BCFTOOLS_NORM_REJOIN rejoins the split records after isec, before phasing. + // Split multi-allelics so BCFTOOLS_ISEC, which matches exact REF/ALT, can intersect them per ALT. + // BCFTOOLS_NORM_REJOIN rejoins them before phasing. ext.args = { "-m -any -Oz" } @@ -157,10 +153,7 @@ process { } withName: '.*:BCFTOOLS_ANNOTATE' { ext.prefix = { "${meta.id}.${meta.caller}" } - // Stamps INFO/CALLER and, in 'all' mode, unifies the allele frequency key in the same - // invocation: --rename-annots composes with -a/-c/-h, so this does not need a second - // bcftools annotate pass. meta.rename_to is set only when SMALL_VARIANT_CONSENSUS is - // merging two callers into one VCF. + // Stamp INFO/CALLER; in 'all' mode (meta.rename_to set) also unify the FORMAT allele frequency key. ext.args = { def rename = meta.rename_to == 'VAF' ? "--rename-annots <(printf 'FORMAT/AF\\tFORMAT/VAF\\n') " @@ -176,12 +169,10 @@ process { enabled: false ] } - // GERMLINE VERDICT TRANSFER (tumor-only deep family). These three selectors are matched on the - // alias, so they do not disturb the generic BCFTOOLS_QUERY/ANNOTATE/VIEW entries below. + // GERMLINE VERDICT TRANSFER (tumor-only): DeepSomatic's verdict filters DeepVariant's germline calls. withName: '.*:DS_VERDICT_QUERY' { ext.prefix = { "${meta.id}.ds_verdict" } - // Keep only non-PASS/RefCall rows (GERMLINE/PON in practice). Excludes rather than includes, - // since naming an undeclared FILTER (PON is absent without a PON) makes bcftools fail. + // Exclude PASS/RefCall rather than include GERMLINE/PON: bcftools fails on an undeclared FILTER. ext.args = { "-e 'FILTER=\"PASS\" || FILTER=\"RefCall\"' -f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" } @@ -454,9 +445,7 @@ process { enabled: false ] } - // VCFTAG stamps an INFO provenance flag on each arm before germline and somatic are merged - // for somatic phasing, so PHASING_HAPLOTYPING:BCFTOOLS_VIEW can select the somatic arm by - // origin instead of by position. Intermediates only; the published VCFs come from Longphase. + // Provenance flags stamped before the phasing merge, so the somatic arm is selected by origin. withName: '.*:TAG_SOMATIC' { ext.prefix = { "${meta.id}_somatic_tagged" } publishDir = [ @@ -772,8 +761,8 @@ process { ] } - // PASS-only per-caller copies fed to consensus/phasing/VEP; ClairS-TO is covered by VCFSPLIT. - // --write-index=tbi is required: the module's index output is optional and the join needs it. + // PASS-only per-caller copies for downstream steps; ClairS-TO is covered by VCFSPLIT. + // --write-index=tbi: the module's index output is optional and the join needs it. withName: '.*:(CLAIR3|CLAIRS|DEEPVARIANT|DEEPSOMATIC)_PASS_FILTER' { ext.args = '--apply-filters PASS --output-type z --write-index=tbi' publishDir = [ diff --git a/docs/usage.md b/docs/usage.md index 0c686ce9..427a12a8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -136,21 +136,14 @@ For structural variants, the CHM13 panel of normals is a merged panel combining For tumour-only small variants, ClairS-TO separates germline from somatic calls with a panel of normals and with its Verdict module, which tags each call as germline, somatic or subclonal somatic from tumour purity and allele-specific copy number. `--genome CHM13` supplies five CHM13 PON VCFs (gnomAD, dbSNP, 1000 Genomes, CoLoRSdb and ASAP), which **replace** the GRCh38 databases inside the container. Unless `--skip_ascat` is set, purity and copy number come from the pipeline's own ASCAT run (`CLAIRSTO_VERDICT_TAG`); only with `--skip_ascat` does ClairS-TO estimate them itself, from assembly-specific loci, allele and GC content files. A GRCh38 resource set on a CHM13 run leaves germline variants untagged. When `--germline_var_keep` includes `deepvariant`, the tumour-only germline arm -runs DeepVariant on the **tumour** BAM. DeepVariant is a germline caller with no -somatic discrimination, so on its own those calls mix germline and clonal somatic -variants. When DeepSomatic also runs, the pipeline transfers its verdict onto them: -DeepSomatic evaluates the same sites and labels each `GERMLINE`, `PON`, `RefCall` -or `PASS`, and that label is recorded in `INFO/DS_VERDICT`. Only positively -adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; -`RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed -germline. On a 30x tumour-only sample this keeps about 83% of DeepVariant's -`PASS` calls and removes roughly 1% that DeepSomatic positively calls somatic. - -This restriction to sites DeepSomatic calls `GERMLINE` or `PON` applies only when -`deepsomatic` is also in `--somatic_var_keep`; without it, the DeepVariant germline -calls are used without any verdict filter and may include somatic variants. Note that even after adjudication the tumour-only germline arm -is a tumour-derived proxy, not a call set from normal tissue, and should not be -used for secondary findings without that caveat. +runs DeepVariant on the **tumour** BAM, so on its own those calls mix germline and +clonal somatic variants. When `deepsomatic` is also in `--somatic_var_keep`, the +pipeline records DeepSomatic's verdict at each site in `INFO/DS_VERDICT` and keeps +only sites it calls `GERMLINE` or `PON`; `RefCall` and sites DeepSomatic never +evaluated are dropped. Without `deepsomatic`, the DeepVariant germline calls are +used without a verdict filter and may include somatic variants. Either way the +tumour-only germline arm is a tumour-derived proxy, not a call set from normal +tissue. With `--genome CHM13 --skip_ascat` the pipeline builds a CHM13 resource set from the ASCAT files it already downloads, so no extra setup is needed. LogR correction is GC-only, as ClairS-TO recommends for CHM13: no replication timing file is published for the assembly. Without `--skip_ascat` nothing is built, because the tagging comes from ASCAT's own tables. @@ -410,12 +403,10 @@ These options control how variants from multiple callers are filtered and merged | `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only -for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds -around 13.7 M records of which roughly 50 k are `PASS`, the rest being `RefCall`, -`GERMLINE` or `PON`. Clair3 and ClairS are far less extreme but still keep their -`LowQual` and `NonSomatic` records. With `*_var_combine = 'all'` the union would -otherwise be "every site every caller looked at", which inflates the phased VCFs -by three orders of magnitude and produces a meaningless mutation burden. +for the variants they call, so most of their records are `RefCall`, `GERMLINE` or +`PON`; Clair3 and ClairS also keep their `LowQual` and `NonSomatic` records. Without +a `PASS` filter, `*_var_combine = 'all'` would carry all of these into the phased +VCFs and the mutation burden. `--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's VCF that is handed to the caller consensus, phasing, VEP and the report. In @@ -433,20 +424,17 @@ split is normalised to `PASS`, with its original value kept in `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only whose record represents a variant that both callers found -- it never decides which -variants are kept. +variants are kept. Multi-allelic records are split so they can be matched across +callers, and rejoined before phasing. #### Germline and somatic provenance Germline and somatic small variants are merged into one VCF for somatic phasing, because Longphase needs all variant sites in a single file to produce consistent -phase blocks. The somatic arm is then recovered from the phased result. - -That recovery selects on an `INFO/SOMATIC` flag stamped on each arm before the merge, -not on position. A positional restriction cannot separate the two populations: a -germline record at the same coordinate as a somatic call is indistinguishable from -it, and `FILTER` is no help either, since `VCFSPLIT` normalises the ClairS-TO -germline split to `PASS` so that downstream tools which filter on `PASS` still see -every record. +phase blocks. The somatic arm is then recovered from the phased result by an +`INFO/SOMATIC` flag stamped on each arm before the merge, not by position, since a +germline record at the same coordinate as a somatic call would otherwise be kept. +Tagging leaves `FILTER` unchanged. Three INFO fields carry this provenance: diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index 2b627223..55cb6300 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -38,8 +38,7 @@ process VCFSPLIT { bcftools concat -a -Oz -o germline_tmp.vcf.gz indels_filtered.vcf.gz snv_filtered.vcf.gz tabix -p vcf germline_tmp.vcf.gz - # Normalise FILTER to PASS and keep the original in INFO/ORIG_FILTER (";" stored as ","). - # The header's double quotes arrive via -v q, avoiding fragile escapes in the script block. + # Normalise FILTER to PASS, keeping the original in INFO/ORIG_FILTER (";" stored as ","). bcftools view germline_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } /^#CHROM/ { print "##INFO="; print; next } diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 40fb011e..0f101586 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -142,11 +142,7 @@ workflow PHASING_HAPLOTYPING { // // MODULE: VCFTAG (label: process_single), aliased TAG_SOMATIC / TAG_GERMLINE - // Stamp each arm with an INFO provenance flag before the merge. This is the only point where - // germline-vs-somatic origin is unambiguous for every caller: GERMLINE_CONSENSUS can emit - // records that never passed through VCFSPLIT, so tagging earlier would leave holes. FILTER - // cannot tell the arms apart after the merge, so the flag carries provenance. - // LongPhase preserves custom INFO keys, so the flags survive phasing (verified on v2.0.1). + // Stamp each arm with an INFO provenance flag before the merge; LongPhase keeps it through phasing. // TAG_SOMATIC ( somatic_vcf, 'SOMATIC' ) TAG_GERMLINE( germline_vcf, 'GERMLINE' ) @@ -163,9 +159,7 @@ workflow PHASING_HAPLOTYPING { tagged_germline_vcf .join(tagged_somatic_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> - // Order here is cosmetic: BCFTOOLS_CONCAT sorts its input file list alphabetically - // (modules/nf-core/bcftools/concat/main.nf), so the germline file is passed first - // regardless. With -a the output is coordinate-ordered either way. + // Order is cosmetic: BCFTOOLS_CONCAT sorts its inputs by name. def vcfs = [som_vcf, germ_vcf] def tbis = [som_tbi, germ_tbi] return [ meta, vcfs, tbis] @@ -278,11 +272,7 @@ workflow PHASING_HAPLOTYPING { // // MODULE: BCFTOOLS_VIEW (label: process_medium) - // Reduce the phased somatic+germline VCF to the somatic arm, selecting on the INFO/SOMATIC flag - // stamped before the merge -- by provenance, not position. The previous `-T ` - // targets file matched CHROM/POS only, so germline records co-located with a somatic call were - // retained and became indistinguishable downstream. PS/HP tags on somatic variants survive; - // germline records are dropped here but stay published under variants/phased/ and vep/germline/. + // Keep the somatic arm by its INFO/SOMATIC flag, not by position; PS/HP tags survive. // Input: [meta, phased_combined_vcf, phased_combined_tbi] // Output: .vcf -- [meta, vcf.gz] -- phased somatic-only VCF // .tbi -- [meta, tbi] diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index 4717cb69..6444a492 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -45,23 +45,7 @@ workflow SMALL_VARIANT_CONSENSUS { // ALLELE FREQUENCY KEY -- BCFTOOLS_ANNOTATE below renames the AF FORMAT field to the priority caller's: // FORMAT/AF -> FORMAT/VAF when prioritize_caller is 'deepvariant'/'deepsomatic' // FORMAT/VAF -> FORMAT/AF when prioritize_caller is 'clair' - // This guarantees the merged VCF exposes allele frequency under a single FORMAT key, which is - // what WAKHAN consumes. Every caller currently emits FORMAT/AF and none emits VAF (verified - // against Clair3, ClairS-TO, DeepVariant and DeepSomatic output), so under the default - // prioritize_caller='clair' it is a no-op; it is kept as the guarantee, not the mechanism. - // - // The callers do disagree on the AF *declaration*: ClairS-TO says Number=1, DeepVariant and - // DeepSomatic say Number=A. bcftools concat only warns and keeps the first file's definition. - // Renaming cannot fix that and `annotate -h` cannot override an existing FORMAT definition, - // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record - // reaching here carries one ALT and one AF value, making the two declarations equivalent. - // After BCFTOOLS_NORM_REJOIN, consensus files keep the source caller's own header (lossless round - // trip); 'all' mode takes the Number=A header of isec's 0000.vcf.gz, which concat lists first. - // - // The rename is carried out by BCFTOOLS_ANNOTATE below rather than by a second annotate call: - // meta.rename_to selects the --rename-annots file in conf/modules.config. Only 'all' mode needs - // it, since in 'consensus' mode every surviving record comes from one caller. BCFTOOLS_QUERY - // reads only CHROM/POS/REF/ALT, so it does not care whether the rename has happened yet. + // Only 'all' mode renames: it merges both callers, so the merged VCF needs one AF key for WAKHAN. // // MODULE: BCFTOOLS_QUERY (label: process_single) @@ -80,8 +64,7 @@ workflow SMALL_VARIANT_CONSENSUS { def columns = [] // no extra column specs def header_lines = [] // no extra header lines def rename_chrs = [] // no chromosome renaming - // 'all' mode merges records from both callers into one VCF, so the allele - // frequency key is unified here; 'consensus' mode needs no rename. + // 'all' mode merges both callers, so unify the AF key; 'consensus' needs no rename. def new_meta = combine_method == 'all' ? meta + [rename_to: (prioritize_caller in ['deepvariant', 'deepsomatic'] ? 'VAF' : 'AF')] : meta @@ -109,8 +92,7 @@ workflow SMALL_VARIANT_CONSENSUS { // annotated_vcfs: [meta(+caller), vcf, tbi] -- VCF with CALLER INFO tag // Branch annotated VCFs by caller family for the intersection step - // An unrecognised meta.caller would silently vanish without the `other` arm, taking the whole - // sample out of the results with a successful exit. + // `other` errors on an unrecognised meta.caller instead of silently dropping the sample. annotated_vcfs .branch { meta, _vcfs, _tbi -> deepvariant: meta.caller in [ 'deepvariant', 'deepsomatic' ] @@ -165,8 +147,7 @@ workflow SMALL_VARIANT_CONSENSUS { // deepvariant_ch: [meta (no caller), vcf, tbi] // Join DeepVariant and Clair VCFs per sample into a single tuple for BCFTOOLS_ISEC - // failOnMismatch: a sample present for one caller but not the other would otherwise be dropped - // from every downstream result while the run still reported success. + // failOnMismatch: a sample missing one caller would otherwise be dropped silently. deepvariant_ch .join(clair_ch, failOnMismatch: true, failOnDuplicate: true) .map { meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> @@ -219,10 +200,8 @@ workflow SMALL_VARIANT_CONSENSUS { } else if (combine_method == 'all') { - // Union: all variants from both callers. Shared variants contribute a single record, taken - // from the prioritized caller; both callers' private calls are kept. prioritize_caller only - // selects whose record is used for shared variants, never which calls are kept. - // The three isec sets are disjoint by construction, so BCFTOOLS_CONCAT needs no -d. + // Union: shared calls (prioritized caller's record) plus both callers' private calls. + // The three isec sets are disjoint, so BCFTOOLS_CONCAT needs no -d. if (prioritize_caller in ['deepvariant', 'deepsomatic']) { // shared (DeepVariant record) + DeepVariant-private + Clair-private BCFTOOLS_ISEC.out.deepvar_consensus_vcf diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 5ec14050..ab726ceb 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -12,7 +12,6 @@ include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subwork include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' // Germline verdict transfer: DeepSomatic adjudicates DeepVariant's tumor-derived germline calls. -// Three independent bcftools invocations, so three aliased instances of the upstream modules. include { BCFTOOLS_QUERY as DS_VERDICT_QUERY } from '../../../modules/nf-core/bcftools/query/main' include { BCFTOOLS_ANNOTATE as DS_VERDICT_ANNOTATE } from '../../../modules/nf-core/bcftools/annotate/main' include { BCFTOOLS_VIEW as DS_GERMLINE_SELECT } from '../../../modules/nf-core/bcftools/view/main' @@ -219,25 +218,7 @@ workflow TUMORONLY_SMALLVAR { // Keep only DeepSomatic-adjudicated germline sites; skipped when deepsomatic isn't selected. def deepvariant_germline = deepvariant_vcf if (somatic_var_keep.contains('deepsomatic')) { - // GERMLINE VERDICT TRANSFER (tumor-only, deep family) - // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is - // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its - // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the - // germline VCF therefore carries most of the somatic call set. - // - // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic - // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO - // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 - // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% - // (57,684) PASS -- the last being real somatic calls that must not be published as germline. - // - // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and - // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts - // are transferred to INFO/DS_VERDICT. - // - // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified - // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would - // break INFO parsing. + // GERMLINE VERDICT TRANSFER: DeepVariant on the tumor BAM cannot tell germline from somatic. // // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter From ff88896d455992c705b9520ba384ecec87334df0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 12:57:41 +0200 Subject: [PATCH 27/36] Regenerate pipeline snapshots after the review fixes PASS_FILTER is now reported per alias, BCFTOOLS_NORM_REJOIN is new, and the multi-caller modes phase from rejoined multi-allelics, so union/consensus haplotagged BAMs and WhatsHap stats change. Snapshots regenerated with Nextflow 25.10.4. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/chm13.nf.test.snap | 11 +++++--- tests/clair_only.nf.test.snap | 13 +++++---- tests/consensus.nf.test.snap | 44 +++++++++++++++++++----------- tests/deep_only.nf.test.snap | 13 +++++---- tests/default.nf.test.snap | 13 +++++---- tests/union.nf.test.snap | 50 ++++++++++++++++++++++------------- 6 files changed, 90 insertions(+), 54 deletions(-) diff --git a/tests/chm13.nf.test.snap b/tests/chm13.nf.test.snap index ad12da16..4e196867 100644 --- a/tests/chm13.nf.test.snap +++ b/tests/chm13.nf.test.snap @@ -14,6 +14,9 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, @@ -23,6 +26,9 @@ "CLAIRSTO_CNA_RESOURCES": { "coreutils": 9.5 }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -67,9 +73,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -148,6 +151,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-23T10:46:28.14524359" + "timestamp": "2026-09-25T12:34:22.954806822" } } \ No newline at end of file diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index a7f1925e..557714f7 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -24,12 +24,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -74,9 +80,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -840,8 +843,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T17:07:33.562660027" + "timestamp": "2026-09-25T12:28:35.531071958" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index 889b9399..03293479 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -14,6 +14,9 @@ "BCFTOOLS_NORM": { "bcftools": 1.22 }, + "BCFTOOLS_NORM_REJOIN": { + "bcftools": 1.22 + }, "BCFTOOLS_QUERY": { "bcftools": 1.22 }, @@ -29,12 +32,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -47,6 +56,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -56,6 +68,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -106,9 +121,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -625,10 +637,10 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,a524e54ab0685eeb79e461926d8c7906", - "sample1_normal.bam.bai:md5,ba98fb39fc3cc8b7319376b003aa3dcd", - "sample1_tumor.bam:md5,db8b0da3dd1a6f1c38420a4942881119", - "sample1_tumor.bam.bai:md5,1b75ed3238520172019817b88d254f1f", + "sample1_normal.bam:md5,5d3f0615b0d8a9748e6b6bc7d76ab577", + "sample1_normal.bam.bai:md5,3ef784f2536c7506a50d0ed20c3d78fb", + "sample1_tumor.bam:md5,7317ffe61fe3c4f73961db60329adfa9", + "sample1_tumor.bam.bai:md5,6aaeaebe40c422358bb55296c508db2b", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", @@ -636,18 +648,18 @@ "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", "sample1_whatshap_stats.gtf:md5,30bde8f88b7d4e88b935e88e00997ce7", - "sample1_whatshap_stats.log:md5,51d522df21c6b526909a5cc9de93eaea", - "sample1_whatshap_stats.tsv:md5,8ce67834b3b7471c04151b55585a56bc", + "sample1_whatshap_stats.log:md5,a3cf683c728ce63a5c2be33edd20f9cc", + "sample1_whatshap_stats.tsv:md5,cca55ca6f99eb0857759a4007c10b346", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,0b3633a4d389b4467fb3eac83da94882", + "sample2_normal.bam:md5,ece242f245fc4e2210e13e6de93b8fdc", "sample2_normal.bam.bai:md5,d96d0071ab25ca8dc2327acba4395515", - "sample2_tumor.bam:md5,5c549642cc7192289222d558dc3a07a7", - "sample2_tumor.bam.bai:md5,1554e8cc8ced05271f5d5f7ac7819410", + "sample2_tumor.bam:md5,0c538dc0e27566677313d1f9374fa2b1", + "sample2_tumor.bam.bai:md5,ab2fca59e6729e011c30310eaee2ef1c", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", @@ -655,8 +667,8 @@ "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", "sample2_whatshap_stats.gtf:md5,2e5ace4cac0b42bb6132513062781e47", - "sample2_whatshap_stats.log:md5,9aa99ee14473f65071366ed1729ab064", - "sample2_whatshap_stats.tsv:md5,13cbc783b1357e1bd2c6bb51ae6f1267", + "sample2_whatshap_stats.log:md5,0979359e459a14037a704733f32d3b95", + "sample2_whatshap_stats.tsv:md5,8c974019af5c839c72604e7526ae8a3d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -681,8 +693,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T17:31:36.878403725" + "timestamp": "2026-09-25T12:21:59.090749802" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 6d2a1148..9a25aad9 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -23,6 +23,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -32,6 +35,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -82,9 +88,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -626,8 +629,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T17:50:37.140724409" + "timestamp": "2026-09-25T12:56:04.936282901" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 4ee27de3..62439fb0 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -14,12 +14,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -64,9 +70,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -618,8 +621,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T18:06:32.118064724" + "timestamp": "2026-09-25T11:57:04.096725193" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index de00b2c4..42bea83c 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -14,6 +14,9 @@ "BCFTOOLS_NORM": { "bcftools": 1.22 }, + "BCFTOOLS_NORM_REJOIN": { + "bcftools": 1.22 + }, "BCFTOOLS_QUERY": { "bcftools": 1.22 }, @@ -26,12 +29,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -44,6 +53,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -53,6 +65,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -103,9 +118,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -622,38 +634,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,dcb537c163750d41140d0a4da107f18e", - "sample1_normal.bam.bai:md5,12cf13fae8230cfc1cc8888f80102143", - "sample1_tumor.bam:md5,58cf65517c91958f95a1f4be1c75e2df", - "sample1_tumor.bam.bai:md5,e40049da796ad038e7ebeaa9ac070488", + "sample1_normal.bam:md5,f0809c8be190289f21339696883dea6f", + "sample1_normal.bam.bai:md5,cbff669c086761e893a84ba3ec032b5f", + "sample1_tumor.bam:md5,6267bf3e86a69536e45968b908d1cc6f", + "sample1_tumor.bam.bai:md5,1ec6d68c92182ff9051b8707c4975445", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,9a7823e6b572036a7e95904b0083b8ca", - "sample1_whatshap_stats.log:md5,05c819627f77305a75d776a27dbf247e", - "sample1_whatshap_stats.tsv:md5,5b9d3f6a6e1e1c67c855f8f9b581bb18", + "sample1_whatshap_stats.gtf:md5,f5d331899db63b2ae21e51436bad2ddd", + "sample1_whatshap_stats.log:md5,111538b7e305189fb6de95ea978d06c8", + "sample1_whatshap_stats.tsv:md5,fa0fd5ce2b0919098ffa7d960cf177de", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,74e92562e0cd406a6282bce98ccc93d2", - "sample2_normal.bam.bai:md5,430c3b069a79227396a6b4f1c37a02ef", - "sample2_tumor.bam:md5,0059b8022c674b5daf2fd001ab69a54e", - "sample2_tumor.bam.bai:md5,e236afb5f7a644827e2b38cd36838d98", + "sample2_normal.bam:md5,c18bbb1bc05b0ec830fa1ac3c6ef542f", + "sample2_normal.bam.bai:md5,de77684ef264476b3e0531b61ca23740", + "sample2_tumor.bam:md5,8b8cb5ac7668b8ac2f4097f584481d35", + "sample2_tumor.bam.bai:md5,3b46456b0e7b00688519dfb25c8e3c88", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,8107c5dce892aa11145274fd088788f8", - "sample2_whatshap_stats.log:md5,c2b17319f0f9e6a8ec102be86aa833ba", - "sample2_whatshap_stats.tsv:md5,7028c4a428f5d0fd31472439e287e565", + "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", + "sample2_whatshap_stats.log:md5,9308d0bd8dc9a4a86359f1ec926e6cb1", + "sample2_whatshap_stats.tsv:md5,b81282a609eca306912d4f7b7fdfeebb", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -678,8 +690,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T18:43:33.111991495" + "timestamp": "2026-09-25T12:04:10.544092583" } } \ No newline at end of file From 46e9346d751755a5c715c79522cb310aa5303fe7 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 29 Sep 2026 15:44:57 +0200 Subject: [PATCH 28/36] Phase only alt somatic records, without germline records at their positions LongPhase phases by position, so a germline record at a somatic call's position lent the somatic record its genotype and phase set. Germline records at alt-genotype somatic positions are now left out of somatic phasing (new BCFTOOLS_EXCLUDE_SITES module); 0/0 and ./. somatic records rejoin unphased. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 1 + conf/modules.config | 27 +++++- docs/output.md | 12 +-- docs/usage.md | 7 ++ .../bcftools/excludesites/environment.yml | 9 ++ modules/local/bcftools/excludesites/main.nf | 41 ++++++++++ modules/local/bcftools/excludesites/meta.yml | 77 ++++++++++++++++++ .../bcftools/excludesites/tests/main.nf.test | 55 +++++++++++++ subworkflows/local/phasing_haplotyping.nf | 49 ++++++++++- tests/fixtures/excludesites_input.vcf | 9 ++ tests/fixtures/excludesites_input.vcf.gz | Bin 0 -> 432 bytes tests/fixtures/excludesites_input.vcf.gz.tbi | Bin 0 -> 106 bytes tests/fixtures/excludesites_mask.vcf | 7 ++ tests/fixtures/excludesites_mask.vcf.gz | Bin 0 -> 424 bytes tests/fixtures/excludesites_mask.vcf.gz.tbi | Bin 0 -> 106 bytes 15 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 modules/local/bcftools/excludesites/environment.yml create mode 100644 modules/local/bcftools/excludesites/main.nf create mode 100644 modules/local/bcftools/excludesites/meta.yml create mode 100644 modules/local/bcftools/excludesites/tests/main.nf.test create mode 100644 tests/fixtures/excludesites_input.vcf create mode 100644 tests/fixtures/excludesites_input.vcf.gz create mode 100644 tests/fixtures/excludesites_input.vcf.gz.tbi create mode 100644 tests/fixtures/excludesites_mask.vcf create mode 100644 tests/fixtures/excludesites_mask.vcf.gz create mode 100644 tests/fixtures/excludesites_mask.vcf.gz.tbi diff --git a/CHANGELOG.md b/CHANGELOG.md index 31617a8a..9e7ec59a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `docs/output.md` now lists the ClairS SNV output as `snvs.vcf.gz`, the name the pipeline publishes, instead of `snv.vcf.gz` (@ljwharbers). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now keeps both callers' private calls; the prioritized caller's own private calls were previously dropped. Invalid `combine_method`/`prioritize_caller` values now raise an error (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The caller consensus now splits multi-allelic records before intersecting, so they match across callers, and rejoins them before phasing (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Somatic phasing no longer includes germline records at the position of an alt-genotype somatic call, which gave the somatic record the germline genotype and phase set (new `BCFTOOLS_EXCLUDE_SITES` module); `0/0` and `./.` somatic records are published unphased (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `LRSOMATICREPORT` now renders each sample as soon as its own inputs are ready instead of waiting for the whole batch, including samples without ASCAT's optional raw segments output (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, fixing a deterministic `StatisticsError` crash in BAF binning that aborted the run (@robert-a-forsyth). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index c641777c..d350ffb6 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -458,9 +458,34 @@ process { enabled: false ] } + // Only alt-genotype somatic records are phased; GERMLINE_ANCHORS drops germline records at their positions. + withName: '.*:PHASING_HAPLOTYPING:SOMATIC_ALT' { + ext.prefix = { "${meta.id}_somatic_alt" } + ext.args = { "-i 'GT=\"alt\"'" } + publishDir = [ enabled: false ] + } + withName: '.*:PHASING_HAPLOTYPING:SOMATIC_NONALT' { + ext.prefix = { "${meta.id}_somatic_nonalt" } + ext.args = { "-e 'GT=\"alt\"'" } + publishDir = [ enabled: false ] + } + withName: '.*:PHASING_HAPLOTYPING:GERMLINE_ANCHORS' { + ext.prefix = { "${meta.id}_germline_anchors" } + publishDir = [ enabled: false ] + } withName: '.*:PHASING_HAPLOTYPING:BCFTOOLS_VIEW' { - ext.prefix = { "somatic_smallvariants" } + ext.prefix = { "${meta.id}_somatic_phased_alt" } ext.args = { "-i 'INFO/SOMATIC=1'" } + publishDir = [ enabled: false ] + } + withName: '.*:PHASING_HAPLOTYPING:CONCAT_SOMATIC_UNPHASED' { + ext.prefix = { "${meta.id}_somatic_combined" } + ext.args = { '-Oz -a -W=tbi' } + publishDir = [ enabled: false ] + } + withName: '.*:PHASING_HAPLOTYPING:SORT_SOMATIC_PHASED' { + ext.prefix = { "somatic_smallvariants" } + ext.args = { '-Oz -W=tbi' } publishDir = [ path: { "${params.outdir}/${meta.id}/variants/phased" }, mode: params.publish_dir_mode, diff --git a/docs/output.md b/docs/output.md index 54fb647c..6ddf17e8 100644 --- a/docs/output.md +++ b/docs/output.md @@ -453,12 +453,12 @@ Phased variant calls produced by Longphase. Present in all samples. │ ├── somatic_smallvariants.vcf.gz.tbi ``` -| File | Description | -| ----------------------------------- | ---------------------------------------------------------------- | -| `germline_smallvariants.vcf.gz` | Longphase-phased germline SNV/indel VCF with haplotype (PS) tags | -| `germline_smallvariants.vcf.gz.tbi` | Index for the phased germline VCF | -| `somatic_smallvariants.vcf.gz` | Longphase-phased somatic SNV/indel VCF with haplotype (PS) tags | -| `somatic_smallvariants.vcf.gz.tbi` | Index for the phased somatic VCF | +| File | Description | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `germline_smallvariants.vcf.gz` | Longphase-phased germline SNV/indel VCF with haplotype (PS) tags | +| `germline_smallvariants.vcf.gz.tbi` | Index for the phased germline VCF | +| `somatic_smallvariants.vcf.gz` | Longphase-phased somatic SNV/indel VCF with haplotype (PS) tags; `0/0` and `./.` records are left unphased | +| `somatic_smallvariants.vcf.gz.tbi` | Index for the phased somatic VCF | diff --git a/docs/usage.md b/docs/usage.md index 54192040..ae9cac09 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -438,6 +438,13 @@ phase blocks. The somatic arm is then recovered from the phased result by an germline record at the same coordinate as a somatic call would otherwise be kept. Tagging leaves `FILTER` unchanged. +Longphase phases by position, so a germline record at the position of a somatic +call would lend the somatic record its genotype and phase set. Germline records +at the position of any somatic call with an alternate genotype are therefore +left out of somatic phasing. Somatic records without an alternate genotype +(`0/0` or `./.`, present only with `--smallvar_filter_pass false`) are not +phased: they are added back to `somatic_smallvariants.vcf.gz` unchanged. + Three INFO fields carry this provenance: | Field | Meaning | diff --git a/modules/local/bcftools/excludesites/environment.yml b/modules/local/bcftools/excludesites/environment.yml new file mode 100644 index 00000000..cb55500b --- /dev/null +++ b/modules/local/bcftools/excludesites/environment.yml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + # renovate: datasource=conda depName=bioconda/htslib + - bioconda::bcftools=1.22 + - bioconda::htslib=1.22.1 diff --git a/modules/local/bcftools/excludesites/main.nf b/modules/local/bcftools/excludesites/main.nf new file mode 100644 index 00000000..e0d74e41 --- /dev/null +++ b/modules/local/bcftools/excludesites/main.nf @@ -0,0 +1,41 @@ +process BCFTOOLS_EXCLUDE_SITES { + tag "${meta.id}" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/47/474a5ea8dc03366b04df884d89aeacc4f8e6d1ad92266888e7a8e7958d07cde8/data' + : 'community.wave.seqera.io/library/bcftools_htslib:0a3fa2654b52006f'}" + + input: + tuple val(meta), path(vcf), path(tbi), path(mask), path(mask_tbi) + + output: + tuple val(meta), path("${prefix}.vcf.gz"), emit: vcf + tuple val(meta), path("${prefix}.vcf.gz.tbi"), emit: tbi + tuple val("${task.process}"), val('bcftools'), eval("bcftools --version | sed '1!d; s/^.*bcftools //'"), topic: versions, emit: versions_bcftools + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}_excluded" + """ + # Drop every record at a CHROM:POS present in the mask, whatever its alleles. + bcftools view \\ + -T ^${mask} \\ + -Oz \\ + -W=tbi \\ + ${args} \\ + -o ${prefix}.vcf.gz \\ + ${vcf} + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}_excluded" + """ + echo '' | gzip > ${prefix}.vcf.gz + touch ${prefix}.vcf.gz.tbi + """ +} diff --git a/modules/local/bcftools/excludesites/meta.yml b/modules/local/bcftools/excludesites/meta.yml new file mode 100644 index 00000000..5a7d3d92 --- /dev/null +++ b/modules/local/bcftools/excludesites/meta.yml @@ -0,0 +1,77 @@ +name: bcftools_exclude_sites +description: Remove every record of a VCF at a position (CHROM:POS) that a mask VCF also has, regardless of alleles +keywords: + - filtering + - VCF + - positions +tools: + - view: + description: VCF/BCF conversion, view, subset and filter VCF/BCF files. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + tool_dev_url: https://github.com/samtools/bcftools + doi: "10.1093/bioinformatics/btp352" + licence: ["MIT"] + identifier: biotools:bcftools +input: + - - meta: + type: map + description: Groovy Map containing sample information e.g. [ id:'test' ] + - vcf: + type: file + description: VCF to remove records from + pattern: "*.vcf.gz" + - tbi: + type: file + description: Tabix index of the VCF + pattern: "*.tbi" + - mask: + type: file + description: VCF whose record positions are removed from the input + pattern: "*.vcf.gz" + - mask_tbi: + type: file + description: Tabix index of the mask VCF + pattern: "*.tbi" +output: + vcf: + - - meta: + type: map + description: Groovy Map containing sample information + - ${prefix}.vcf.gz: + type: file + description: Input VCF without the records at mask positions + pattern: "*.vcf.gz" + tbi: + - - meta: + type: map + description: Groovy Map containing sample information + - ${prefix}.vcf.gz.tbi: + type: file + description: Tabix index of the output VCF + pattern: "*.tbi" + versions_bcftools: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version | sed '1!d; s/^.*bcftools //'": + type: string + description: The command used to generate the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version | sed '1!d; s/^.*bcftools //'": + type: string + description: The command used to generate the version of the tool +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/bcftools/excludesites/tests/main.nf.test b/modules/local/bcftools/excludesites/tests/main.nf.test new file mode 100644 index 00000000..be7933cb --- /dev/null +++ b/modules/local/bcftools/excludesites/tests/main.nf.test @@ -0,0 +1,55 @@ +nextflow_process { + + name "Test Process BCFTOOLS_EXCLUDE_SITES" + script "../main.nf" + process "BCFTOOLS_EXCLUDE_SITES" + + tag "modules" + tag "modules_local" + tag "bcftools_exclude_sites" + + test("drops records at mask positions by position only") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file("\${projectDir}/tests/fixtures/excludesites_input.vcf.gz", checkIfExists: true), + file("\${projectDir}/tests/fixtures/excludesites_input.vcf.gz.tbi", checkIfExists: true), + file("\${projectDir}/tests/fixtures/excludesites_mask.vcf.gz", checkIfExists: true), + file("\${projectDir}/tests/fixtures/excludesites_mask.vcf.gz.tbi", checkIfExists: true) + ] + """ + } + } + + then { + assert process.success + + def positions = path(process.out.vcf[0][1]).linesGzip + .findAll { line -> !line.startsWith('#') } + .collect { line -> line.split('\t')[1] } + + // 100 and 200 go despite different alleles; 102 stays although it lies inside the mask's deletion + assert positions == [ '102', '300' ] + } + } + + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [ id:'test' ], [], [], [], [] ] + """ + } + } + + then { + assert process.success + } + } +} diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 0f101586..f268eb8f 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -6,8 +6,13 @@ include { LONGPHASE_MODCALL as LONGPHASE_MODCALL_GERMLINE } from '../../module include { LONGPHASE_MODCALL as LONGPHASE_MODCALL_SOMATIC } from '../../modules/local/longphase/modcall/main.nf' include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main.nf' include { BCFTOOLS_CONCAT } from '../../modules/nf-core/bcftools/concat/main' +include { BCFTOOLS_CONCAT as CONCAT_SOMATIC_UNPHASED } from '../../modules/nf-core/bcftools/concat/main' include { BCFTOOLS_SORT } from '../../modules/nf-core/bcftools/sort/main' +include { BCFTOOLS_SORT as SORT_SOMATIC_PHASED } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_VIEW } from '../../modules/local/bcftools/view/main.nf' +include { BCFTOOLS_VIEW as SOMATIC_ALT } from '../../modules/local/bcftools/view/main.nf' +include { BCFTOOLS_VIEW as SOMATIC_NONALT } from '../../modules/local/bcftools/view/main.nf' +include { BCFTOOLS_EXCLUDE_SITES as GERMLINE_ANCHORS } from '../../modules/local/bcftools/excludesites/main.nf' include { VCFTAG as TAG_SOMATIC } from '../../modules/local/vcftag/main.nf' include { VCFTAG as TAG_GERMLINE } from '../../modules/local/vcftag/main.nf' @@ -155,9 +160,32 @@ workflow PHASING_HAPLOTYPING { .set{ tagged_germline_vcf } // tagged_*_vcf: [meta, vcf, tbi] + // LongPhase phases by position, so a germline record at a somatic call's POS would lend it its phase. + // Only alt-genotype somatic records are phased; 0/0 and ./. records rejoin unphased after the split. + SOMATIC_ALT ( tagged_somatic_vcf ) + SOMATIC_NONALT( tagged_somatic_vcf ) + + SOMATIC_ALT.out.vcf + .join(SOMATIC_ALT.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .set{ somatic_alt_vcf } + SOMATIC_NONALT.out.vcf + .join(SOMATIC_NONALT.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .set{ somatic_nonalt_vcf } + // somatic_alt_vcf / somatic_nonalt_vcf: [meta, vcf, tbi] + + // + // MODULE: GERMLINE_ANCHORS (BCFTOOLS_EXCLUDE_SITES alias) -- germline records not at an alt somatic POS + // Input: [meta, germline_vcf, tbi, somatic_alt_vcf, tbi] + // Output: .vcf / .tbi -- [meta, vcf.gz] / [meta, tbi] + // + GERMLINE_ANCHORS ( + tagged_germline_vcf.join(somatic_alt_vcf, failOnMismatch: true, failOnDuplicate: true) + ) + // Somatic phasing needs germline and somatic sites in one VCF for consistent phase blocks - tagged_germline_vcf - .join(tagged_somatic_vcf) + GERMLINE_ANCHORS.out.vcf + .join(GERMLINE_ANCHORS.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .join(somatic_alt_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> // Order is cosmetic: BCFTOOLS_CONCAT sorts its inputs by name. def vcfs = [som_vcf, germ_vcf] @@ -165,7 +193,7 @@ workflow PHASING_HAPLOTYPING { return [ meta, vcfs, tbis] } .set{germline_somatic_vcfs} - // germline_somatic_vcfs (pre-concat): [meta, [somatic_vcf, germline_vcf], [somatic_tbi, germline_tbi]] + // germline_somatic_vcfs (pre-concat): [meta, [somatic_alt_vcf, germline_anchor_vcf], [tbis...]] // // MODULE: BCFTOOLS_CONCAT (label: process_medium) @@ -279,8 +307,21 @@ workflow PHASING_HAPLOTYPING { // BCFTOOLS_VIEW ( phased_somatic_germline_vcf ) + // Add the unphased 0/0 and ./. somatic records back (only present with --smallvar_filter_pass false) BCFTOOLS_VIEW.out.vcf - .join(BCFTOOLS_VIEW.out.tbi) + .join(BCFTOOLS_VIEW.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .join(somatic_nonalt_vcf, failOnMismatch: true, failOnDuplicate: true) + .map { meta, phased_vcf, phased_tbi, nonalt_vcf, nonalt_tbi -> + return [ meta, [phased_vcf, nonalt_vcf], [phased_tbi, nonalt_tbi] ] + } + .set{ somatic_parts } + // somatic_parts: [meta, [phased_alt_vcf, nonalt_vcf], [tbis...]] + + CONCAT_SOMATIC_UNPHASED ( somatic_parts ) + SORT_SOMATIC_PHASED ( CONCAT_SOMATIC_UNPHASED.out.vcf ) + + SORT_SOMATIC_PHASED.out.vcf + .join(SORT_SOMATIC_PHASED.out.tbi, failOnMismatch: true, failOnDuplicate: true) .set{ phased_somatic_vcf } // phased_somatic_vcf: [meta, vcf.gz, tbi] -- phased somatic-only VCF (germline removed) diff --git a/tests/fixtures/excludesites_input.vcf b/tests/fixtures/excludesites_input.vcf new file mode 100644 index 00000000..615bfb45 --- /dev/null +++ b/tests/fixtures/excludesites_input.vcf @@ -0,0 +1,9 @@ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT testsample +chr1 100 . A T 30 PASS . GT 0/1 +chr1 102 . G C 30 PASS . GT 0/1 +chr1 200 . T C 30 PASS . GT 1/1 +chr1 300 . C G 30 PASS . GT 0/1 +chr1 400 . G A 30 PASS . GT 0/1 diff --git a/tests/fixtures/excludesites_input.vcf.gz b/tests/fixtures/excludesites_input.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..a22f7cb6bc7576d86bdefa740ae3894fa1b6312a GIT binary patch literal 432 zcmb2|=3rp}f&Xj_PR>jWri`T%0`r>;WRA>#y*%Sdp~DhK&eyF?uw{0M=;h}ZmRIY0`pAb| z+~j7g`QpZ7neUpsYY+J@6Pa^NzjoOKr_HaXX`Wk~TqH2nF8Gn%x#h;wCp2;{s(8KB zJZj6 zCnjWz6{)jpHfm%5)u-ak|cPUP6f;o?U-!b#dA<-o|Awp4^Nsv pS0dMpZ?jk~cW9kE)xZ_-)ro;&TL-ITBhV0eG}ELRn8AjF2mtt28KeLJ literal 0 HcmV?d00001 diff --git a/tests/fixtures/excludesites_mask.vcf b/tests/fixtures/excludesites_mask.vcf new file mode 100644 index 00000000..bf2aaef2 --- /dev/null +++ b/tests/fixtures/excludesites_mask.vcf @@ -0,0 +1,7 @@ +##fileformat=VCFv4.2 +##contig= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT testsample +chr1 100 . ACGTA A 30 PASS . GT 0/1 +chr1 200 . T A,G 30 PASS . GT 1/2 +chr1 400 . G A 30 PASS . GT 0/1 diff --git a/tests/fixtures/excludesites_mask.vcf.gz b/tests/fixtures/excludesites_mask.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..ccd5fc68f99b220886080b95ab412f3bd505e054 GIT binary patch literal 424 zcmb2|=3rp}f&Xj_PR>jWCXBfg4(2sENVJ@P9h~zdLSd6*bnbRe!Hp_fVk>N=4#?jr zN#xvp>DqGr2P=K^C!aAlKA919&f>`;O<`Al@AX@KjV_s-zkg))1iPPlcdY}L>{i&^ zcD1Xst55b^;q;&j9x>`chcf@%Ni_=84C9&k=1zoF`>KF+uQ%;KOVWfd8OI6a&8P`+7qs~L zvai!)x;FW&5`JrH)}(EAZ>5){8B5#xo_IfnnM{>B!aozYMdpcJ|KIEWMvW_@hEc|` z_oPhp^wt*vaWj0_C&Xn_{S;C)WR=!uDm!GkN0OG?W43!jucF)@3of1y-+iEM8@B#k^^S0psd|<7rF`&Vjmb(tuHbZmcu|GdOBM1OVD5tH=NV literal 0 HcmV?d00001 diff --git a/tests/fixtures/excludesites_mask.vcf.gz.tbi b/tests/fixtures/excludesites_mask.vcf.gz.tbi new file mode 100644 index 0000000000000000000000000000000000000000..869e7d7819435ff007bbb1d0fc9737f196cf7e13 GIT binary patch literal 106 zcmb2|=3rp}f&Xj_PR>jWz6{)jpHfm%5)u-ak|cPUP6f;o?U-!b#dA<-o|Awp56?1z pu7&JA)xJ%cE~4pa4y-Gx6d4$PuXs_?2sA_<%`|BSX0YKP0szlx85aNm literal 0 HcmV?d00001 From 97801ffb1bf272efa4857b45ee2721e361b14c8d Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 29 Sep 2026 15:45:12 +0200 Subject: [PATCH 29/36] Keep one caller's record per position in the 'all' caller union The multi-allelic rejoin merged different callers' records into one with alleles no genotype used. 'all' now keeps the prioritised caller's records plus the other caller's at positions it lacks, without splitting; only 'consensus' still splits, intersects and rejoins. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- conf/modules.config | 17 +- docs/usage.md | 34 ++-- nextflow_schema.json | 4 +- subworkflows/local/small_variant_consensus.nf | 191 ++++++++---------- 5 files changed, 117 insertions(+), 131 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e7ec59a..1db51053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `CLAIRS` no longer runs with `--haplotagged_tumor_bam_provided_so_skip_intermediate_phasing_and_haplotagging`. Since somatic calling was moved ahead of `PHASING_HAPLOTYPING` (v1.1.0), ClairS has received the untagged minimap2 BAM, so the flag made it skip its own phasing and haplotagging and call every paired sample without haplotype information: the full-alignment model saw no `HP` tags and the haplotype filtering step had nothing to filter on, the same as `--disable_phasing`. ClairS now runs Clair3 on the normal and tumour BAMs and phases the tumour itself. **Paired somatic calls change** (fewer false positives expected), and `CLAIRS` takes longer and uses more work-directory space (@ljwharbers). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `docs/output.md` now lists the ClairS SNV output as `snvs.vcf.gz`, the name the pipeline publishes, instead of `snv.vcf.gz` (@ljwharbers). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now keeps both callers' private calls; the prioritized caller's own private calls were previously dropped. Invalid `combine_method`/`prioritize_caller` values now raise an error (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The caller consensus now splits multi-allelic records before intersecting, so they match across callers, and rejoins them before phasing (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'consensus'` now splits multi-allelic records before intersecting, so they match across callers, and rejoins them before phasing. `'all'` now keeps the prioritised caller's record wherever both callers call a position, so no record mixes two callers' alleles (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Somatic phasing no longer includes germline records at the position of an alt-genotype somatic call, which gave the somatic record the germline genotype and phase set (new `BCFTOOLS_EXCLUDE_SITES` module); `0/0` and `./.` somatic records are published unphased (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `LRSOMATICREPORT` now renders each sample as soon as its own inputs are ready instead of waiting for the whole batch, including samples without ASCAT's optional raw segments output (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, fixing a deterministic `StatisticsError` crash in BAF binning that aborted the run (@robert-a-forsyth). diff --git a/conf/modules.config b/conf/modules.config index d350ffb6..dacaa53d 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -121,16 +121,13 @@ process { withName: '.*:BCFTOOLS_NORM' { ext.prefix = { "${meta.id}.${meta.caller}_norm" } - // Split multi-allelics so BCFTOOLS_ISEC, which matches exact REF/ALT, can intersect them per ALT. - // BCFTOOLS_NORM_REJOIN rejoins them before phasing. - ext.args = { - "-m -any -Oz" - } + // 'consensus' splits multi-allelics so BCFTOOLS_ISEC can match them per ALT; BCFTOOLS_NORM_REJOIN rejoins them. + ext.args = { meta.split ? "-m -any -Oz" : "-Oz" } publishDir = [ enabled: false ] } - // Rejoin split multi-allelics; LongPhase keys variants by position and cannot take two records at one POS. + // Rejoin split multi-allelics ('consensus' only); LongPhase keys variants by position and cannot take two records at one POS. withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_NORM_REJOIN' { ext.prefix = { "${meta.id}_germline_sorted" } ext.args = { '-m +any --output-type z --write-index=tbi' } @@ -142,6 +139,10 @@ process { publishDir = [ enabled: false ] } + withName: '.*_CONSENSUS:BCFTOOLS_EXCLUDE_SITES' { + ext.prefix = { "${meta.id}_nonpriority_only" } + publishDir = [ enabled: false ] + } withName: '.*:BCFTOOLS_ISEC' { ext.prefix = { "${meta.id}_isec" } ext.args ={ @@ -579,14 +580,14 @@ process { ] } withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_SORT' { - ext.prefix = { "${meta.id}_germline_split_sorted" } + ext.prefix = { "${meta.id}_germline_sorted" } ext.args = {'-Oz -W=tbi'} publishDir = [ enabled: false ] } withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_SORT' { - ext.prefix = { "${meta.id}_somatic_split_sorted" } + ext.prefix = { "${meta.id}_somatic_sorted" } ext.args = {'-Oz -W=tbi'} publishDir = [ enabled: false diff --git a/docs/usage.md b/docs/usage.md index ae9cac09..df54bb53 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -394,15 +394,15 @@ Both tools run from `ghcr.io/ljwharbers/sigprofiler`, which adds CHM13 support n These options control how variants from multiple callers are filtered and merged. -| Parameter | Description | -| ------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| `--germline_var_keep` | Comma-separated germline callers to run: `deepvariant`, `clair`. Default = `clair` | -| `--somatic_var_keep` | Comma-separated somatic callers to run: `deepsomatic`, `clair`. Default = `clair` | -| `--germline_var_combine` | How to combine germline caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | -| `--somatic_var_combine` | How to combine somatic caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | -| `--prioritize_caller_germline` | Whose record to use for variants called by both germline callers: `deepvariant` or `clair`. Default = `clair` | -| `--prioritize_caller_somatic` | Whose record to use for variants called by both somatic callers: `deepsomatic` or `clair`. Default = `clair` | -| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | +| Parameter | Description | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| `--germline_var_keep` | Comma-separated germline callers to run: `deepvariant`, `clair`. Default = `clair` | +| `--somatic_var_keep` | Comma-separated somatic callers to run: `deepsomatic`, `clair`. Default = `clair` | +| `--germline_var_combine` | How to combine germline caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | +| `--somatic_var_combine` | How to combine somatic caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | +| `--prioritize_caller_germline` | Whose record to use where both germline callers call a variant: `deepvariant` or `clair`. Default = `clair` | +| `--prioritize_caller_somatic` | Whose record to use where both somatic callers call a variant: `deepsomatic` or `clair`. Default = `clair` | +| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only for the variants they call, so most of their records are `RefCall`, `GERMLINE` or @@ -423,11 +423,17 @@ records are passed on with their original `FILTER`. Only the ClairS-TO germline split is normalised to `PASS`, with its original value kept in `INFO/ORIG_FILTER`. -`consensus` keeps only variants called by both callers; `all` keeps the union, i.e. -every variant called by either. In both modes `--prioritize_caller_*` chooses only -whose record represents a variant that both callers found -- it never decides which -variants are kept. Multi-allelic records are split so they can be matched across -callers, and rejoined before phasing. +`consensus` keeps only alleles called by both callers, using the prioritised +caller's record. Multi-allelic records are split so each allele can be matched +across callers, and rejoined before phasing. A multi-allelic call of which only one +allele is shared (e.g. DeepVariant `1/2`) is therefore kept as that allele alone, +and its `PL` values for the dropped allele are lost. + +`all` keeps the union by position: every record of the prioritised caller, plus +the other caller's records at positions where the prioritised caller has none. +Where both callers call a position, even with different alleles, only the +prioritised caller's record is kept, so each output record is one caller's call. +Records are not split in this mode. #### Germline and somatic provenance diff --git a/nextflow_schema.json b/nextflow_schema.json index 7ccee4ba..0393ebc1 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -91,13 +91,13 @@ }, "prioritize_caller_germline": { "type": "string", - "description": "When both germline callers are used, specifies which caller's format to use for variants called by both. Must be [deepvariant, clair].", + "description": "When both germline callers are used, specifies whose record to keep where both callers call a position. Must be [deepvariant, clair].", "default": "clair", "enum": ["deepvariant", "clair"] }, "prioritize_caller_somatic": { "type": "string", - "description": "When both somatic callers are used, specifies which caller's format to use for variants called by both. Must be [deepsomatic, clair].", + "description": "When both somatic callers are used, specifies whose record to keep where both callers call a position. Must be [deepsomatic, clair].", "default": "clair", "enum": ["deepsomatic", "clair"] }, diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index 6444a492..aaedb58c 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -7,6 +7,7 @@ include { BCFTOOLS_CONCAT } from '../../modul include { BCFTOOLS_SORT } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_SORT as SORT_POST_NORM } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_SORT as BCFTOOLS_SORT_CONSENSUS } from '../../modules/nf-core/bcftools/sort/main' +include { BCFTOOLS_EXCLUDE_SITES } from '../../modules/local/bcftools/excludesites/main' @@ -21,12 +22,22 @@ workflow SMALL_VARIANT_CONSENSUS { main: + if (!(combine_method in ['consensus', 'all'])) { + error("combine_method must be 'consensus' or 'all', got '${combine_method}'") + } + if (!(prioritize_caller in ['deepvariant', 'deepsomatic', 'clair'])) { + error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") + } + // - // MODULE: BCFTOOLS_NORM (label: process_medium) -- left-align and split multi-allelics for isec; rejoined before phasing - // Input: [meta, vcf, tbi] -- per-caller VCF + // MODULE: BCFTOOLS_NORM (label: process_medium) -- left-align; in 'consensus' mode also split multi-allelics for isec + // Input: [meta(+split), vcf, tbi] -- per-caller VCF // Output: .vcf -- [meta, vcf] -- left-aligned, normalised VCF (unsorted) // - BCFTOOLS_NORM(mixed_vcfs, fasta) + BCFTOOLS_NORM( + mixed_vcfs.map { meta, vcf, tbi -> [meta + [split: combine_method == 'consensus'], vcf, tbi] }, + fasta + ) // // MODULE: SORT_POST_NORM (BCFTOOLS_SORT alias, label: process_medium) -- re-sort and index after normalisation @@ -85,7 +96,7 @@ workflow SMALL_VARIANT_CONSENSUS { BCFTOOLS_ANNOTATE.out.vcf .join(BCFTOOLS_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true) .map { meta, vcf, tbi -> - def clean_meta = meta.findAll { k, _v -> k != 'rename_to' } + def clean_meta = meta.findAll { k, _v -> !(k in ['rename_to', 'split']) } return [clean_meta, vcf, tbi] } .set{annotated_vcfs} @@ -158,117 +169,85 @@ workflow SMALL_VARIANT_CONSENSUS { .set{mixed_vcfs} // mixed_vcfs (re-paired): [meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi]] - // Add empty optional fields required by BCFTOOLS_ISEC - mixed_vcfs - .map{ meta, vcfs, tbis -> - def file = [] // no regions file - def target = [] // no target sites - def regions = [] // no region string - return [meta, vcfs, tbis, file, target, regions] - } - .set{isec_input} - // isec_input: [meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi], [], [], []] - - // - // MODULE: BCFTOOLS_ISEC (label: process_medium) -- shared and private sets of the two callers - // Input: [meta, [vcf1, vcf2], [tbi1, tbi2], [], [], []] - // Output: .deepvar_consensus_vcf / .clair_consensus_vcf -- [meta, vcf] -- shared calls, DeepVariant or Clair record - // .deepvar_private_vcf / .clair_private_vcf -- [meta, vcf] -- caller-private calls (+ .tbi for each) - // - BCFTOOLS_ISEC(isec_input) - if (combine_method == 'consensus') { - // Take only the intersection: variants called by BOTH callers - // Use the record from the prioritized caller - if (prioritize_caller in ['deepvariant', 'deepsomatic']) { - BCFTOOLS_ISEC.out.deepvar_consensus_vcf - .set{isec_consensus_vcf} - } - else if (prioritize_caller == 'clair') { - BCFTOOLS_ISEC.out.clair_consensus_vcf - .set{isec_consensus_vcf} - } - else { - error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") - } + // Add empty optional fields required by BCFTOOLS_ISEC + mixed_vcfs + .map{ meta, vcfs, tbis -> + def file = [] // no regions file + def target = [] // no target sites + def regions = [] // no region string + return [meta, vcfs, tbis, file, target, regions] + } + .set{isec_input} + // isec_input: [meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi], [], [], []] + + // + // MODULE: BCFTOOLS_ISEC (label: process_medium) -- shared and private sets of the two callers + // Input: [meta, [vcf1, vcf2], [tbi1, tbi2], [], [], []] + // Output: .deepvar_consensus_vcf / .clair_consensus_vcf -- [meta, vcf] -- shared calls, DeepVariant or Clair record + // + BCFTOOLS_ISEC(isec_input) + + // Take only the intersection: variants called by BOTH callers, from the prioritized caller's record + def isec_consensus_vcf = prioritize_caller in ['deepvariant', 'deepsomatic'] + ? BCFTOOLS_ISEC.out.deepvar_consensus_vcf + : BCFTOOLS_ISEC.out.clair_consensus_vcf // ISEC always writes 0002.vcf.gz, so germline and somatic would collide by basename in BCFTOOLS_CONCAT; // BCFTOOLS_SORT_CONSENSUS renames it per sample (conf/modules.config) BCFTOOLS_SORT_CONSENSUS(isec_consensus_vcf) - BCFTOOLS_SORT_CONSENSUS.out.vcf.set{vcf} - BCFTOOLS_SORT_CONSENSUS.out.tbi.set{tbi} - // vcf/tbi: [meta, vcf/tbi] -- consensus-only calls from the priority caller, renamed - } - else if (combine_method == 'all') { - // Union: shared calls (prioritized caller's record) plus both callers' private calls. - // The three isec sets are disjoint, so BCFTOOLS_CONCAT needs no -d. - if (prioritize_caller in ['deepvariant', 'deepsomatic']) { - // shared (DeepVariant record) + DeepVariant-private + Clair-private - BCFTOOLS_ISEC.out.deepvar_consensus_vcf - .join(BCFTOOLS_ISEC.out.deepvar_consensus_tbi) - .join(BCFTOOLS_ISEC.out.deepvar_private_vcf) - .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) - .join(BCFTOOLS_ISEC.out.clair_private_vcf) - .join(BCFTOOLS_ISEC.out.clair_private_tbi) - .map{ meta, shared_vcf, shared_tbi, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> - return[meta, [shared_vcf, deepvar_vcf, clair_vcf], [shared_tbi, deepvar_tbi, clair_tbi]] - } - .set{concat_input} - } - else if (prioritize_caller == 'clair') { - // shared (Clair record) + DeepVariant-private + Clair-private - BCFTOOLS_ISEC.out.clair_consensus_vcf - .join(BCFTOOLS_ISEC.out.clair_consensus_tbi) - .join(BCFTOOLS_ISEC.out.deepvar_private_vcf) - .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) - .join(BCFTOOLS_ISEC.out.clair_private_vcf) - .join(BCFTOOLS_ISEC.out.clair_private_tbi) - .map{ meta, shared_vcf, shared_tbi, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> - return[meta, [shared_vcf, deepvar_vcf, clair_vcf], [shared_tbi, deepvar_tbi, clair_tbi]] - } - .set{concat_input} - } - else { - error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") - } - // concat_input: [meta, [shared_vcf, deepvar_private_vcf, clair_private_vcf], [tbis...]] - BCFTOOLS_CONCAT(concat_input) - BCFTOOLS_CONCAT.out.vcf - .set{concat_out} - // concat_out: [meta, vcf] -- unsorted union of both callers' calls - BCFTOOLS_SORT(concat_out) - BCFTOOLS_SORT.out.vcf - .set{vcf} - BCFTOOLS_SORT.out.tbi - .set{tbi} - // vcf/tbi: [meta, vcf/tbi] -- sorted union VCF + // + // MODULE: BCFTOOLS_NORM_REJOIN (BCFTOOLS_NORM alias) -- rejoin split sites (-m +any) so LongPhase and Wakhan see one record per position + // Input: [meta, vcf, tbi] -- sorted consensus VCF (one caller's records only) + // Output: .vcf -- [meta, vcf.gz] + // .tbi -- [meta, tbi] + // + BCFTOOLS_NORM_REJOIN( + BCFTOOLS_SORT_CONSENSUS.out.vcf.join(BCFTOOLS_SORT_CONSENSUS.out.tbi, failOnMismatch: true, failOnDuplicate: true), + fasta + ) + BCFTOOLS_NORM_REJOIN.out.vcf.set{ vcf } + BCFTOOLS_NORM_REJOIN.out.tbi.set{ tbi } + // vcf/tbi: [meta, vcf/tbi] -- consensus calls from the priority caller, multi-allelics rejoined } - else { - error("combine_method must be 'consensus' or 'all', got '${combine_method}'") - } - - // - // MODULE: BCFTOOLS_NORM_REJOIN (BCFTOOLS_NORM alias) -- rejoin split sites (-m +any) so LongPhase and Wakhan see one record per position - // Input: [meta, vcf, tbi] -- sorted consensus/union VCF - // Output: .vcf -- [meta, vcf.gz] - // .tbi -- [meta, tbi] - // - BCFTOOLS_NORM_REJOIN( - vcf.join(tbi, failOnMismatch: true, failOnDuplicate: true), - fasta - ) + // Union by locus: the priority caller's records, plus the other caller's at positions it has no record for. + // Records are unsplit here, so every output record is one caller's call. + mixed_vcfs + .multiMap { meta, vcfs, tbis -> + def prio = prioritize_caller in ['deepvariant', 'deepsomatic'] ? 0 : 1 + priority: [meta, vcfs[prio], tbis[prio]] + exclude: [meta, vcfs[1 - prio], tbis[1 - prio], vcfs[prio], tbis[prio]] + } + .set{ by_priority } + // by_priority.priority: [meta, prio_vcf, prio_tbi] + // by_priority.exclude: [meta, other_vcf, other_tbi, prio_vcf, prio_tbi] + + // + // MODULE: BCFTOOLS_EXCLUDE_SITES (label: process_single) -- the other caller's records at positions the priority caller lacks + // Input: [meta, other_vcf, other_tbi, prio_vcf, prio_tbi] + // Output: .vcf / .tbi -- [meta, vcf.gz] / [meta, tbi] + // + BCFTOOLS_EXCLUDE_SITES(by_priority.exclude) + + by_priority.priority + .join(BCFTOOLS_EXCLUDE_SITES.out.vcf, failOnMismatch: true, failOnDuplicate: true) + .join(BCFTOOLS_EXCLUDE_SITES.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .map { meta, prio_vcf, prio_tbi, other_vcf, other_tbi -> + return [meta, [prio_vcf, other_vcf], [prio_tbi, other_tbi]] + } + .set{concat_input} + // concat_input: [meta, [prio_vcf, other_only_vcf], [tbis...]] - BCFTOOLS_NORM_REJOIN.out.vcf - .join(BCFTOOLS_NORM_REJOIN.out.tbi, failOnMismatch: true, failOnDuplicate: true) - .multiMap { meta, rejoined_vcf, rejoined_tbi -> - vcf: [meta, rejoined_vcf] - tbi: [meta, rejoined_tbi] - } - .set { rejoined } + BCFTOOLS_CONCAT(concat_input) + BCFTOOLS_SORT(BCFTOOLS_CONCAT.out.vcf) + BCFTOOLS_SORT.out.vcf.set{ vcf } + BCFTOOLS_SORT.out.tbi.set{ tbi } + // vcf/tbi: [meta, vcf/tbi] -- sorted union VCF, one record per position + } emit: - vcf = rejoined.vcf // [meta, vcf] -- final consensus/combined VCF, multi-allelics rejoined - tbi = rejoined.tbi // [meta, tbi] + vcf // [meta, vcf] -- final consensus/combined VCF, one record per position + tbi // [meta, tbi] } From cadbdeefef968926aadc416e1abef98ea2c44177 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 29 Sep 2026 15:45:12 +0200 Subject: [PATCH 30/36] Assert record contents of the phased VCFs in the pipeline tests Replaces size() > 0, which a header-only VCF passes, with record checks: PASS-only and SOMATIC-tagged in default mode, non-PASS records present and no phased non-alt somatic record with the PASS filter off. Co-Authored-By: Claude Opus 5.5 (1M context) --- nf-test.config | 3 +++ tests/chm13.nf.test | 3 +++ tests/clair_only.nf.test | 13 ++--------- tests/consensus.nf.test | 13 ++--------- tests/deep_only.nf.test | 13 ++--------- tests/default.nf.test | 3 +++ tests/lib/PhasedVcf.groovy | 44 ++++++++++++++++++++++++++++++++++++++ tests/union.nf.test | 35 ++++++++++++++++++------------ 8 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 tests/lib/PhasedVcf.groovy diff --git a/nf-test.config b/nf-test.config index d1220b10..f37973a1 100644 --- a/nf-test.config +++ b/nf-test.config @@ -8,6 +8,9 @@ config { // location of an optional nextflow.config file specific for executing tests configFile "tests/nextflow.config" + // Groovy helpers shared by the pipeline tests + libDir "tests/lib" + // ignore tests coming from the nf-core/modules repo ignore = [ 'modules/nf-core/**/tests/*', diff --git a/tests/chm13.nf.test b/tests/chm13.nf.test index 4e73e43e..e8020b88 100644 --- a/tests/chm13.nf.test +++ b/tests/chm13.nf.test @@ -19,6 +19,9 @@ nextflow_pipeline { then { assertAll( { assert workflow.success}, + + // ── Phased VCFs: records present, all PASS, somatic arm tagged ── + { PhasedVcf.assertPassOnly("$outputDir", ['sample1', 'sample2', 'sample3'], ['sample3']) }, { //files exist assert file("$outputDir/sample1/variants/clair3/merge_output.vcf.gz").exists() assert file("$outputDir/sample1/variants/clairs/indel.vcf.gz").exists() diff --git a/tests/clair_only.nf.test b/tests/clair_only.nf.test index 13f6d77c..d8f2c35d 100644 --- a/tests/clair_only.nf.test +++ b/tests/clair_only.nf.test @@ -50,17 +50,8 @@ nextflow_pipeline { } }, - // ── Phased VCFs exist and have data ────────────────────────── - { - ['sample1', 'sample2', 'sample3', 'sample4', 'sample5'].each { s -> - def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") - def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") - assert germline.exists() - assert somatic.exists() - assert germline.size() > 0 - assert somatic.size() > 0 - } - }, + // ── Phased VCFs: records present, all PASS, somatic arm tagged ── + { PhasedVcf.assertPassOnly("$outputDir", ['sample1', 'sample2', 'sample3', 'sample4', 'sample5'], ['sample3', 'sample4', 'sample5']) }, // ── BAM files exist ────────────────────────────────────────── { diff --git a/tests/consensus.nf.test b/tests/consensus.nf.test index 9689f212..cc1fd98f 100644 --- a/tests/consensus.nf.test +++ b/tests/consensus.nf.test @@ -51,17 +51,8 @@ nextflow_pipeline { assert file("$launchDir/output/sample3/variants/clairsto/somatic.vcf.gz").exists() }, - // ── Phased consensus VCFs exist and have data ──────────────── - { - ['sample1', 'sample2', 'sample3'].each { s -> - def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") - def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") - assert germline.exists() - assert somatic.exists() - assert germline.size() > 0 - assert somatic.size() > 0 - } - }, + // ── Phased VCFs: records present, all PASS, somatic arm tagged ── + { PhasedVcf.assertPassOnly("$outputDir", ['sample1', 'sample2', 'sample3'], ['sample3']) }, // ── BAM files ──────────────────────────────────────────────── { diff --git a/tests/deep_only.nf.test b/tests/deep_only.nf.test index 5da64375..b5ad3347 100644 --- a/tests/deep_only.nf.test +++ b/tests/deep_only.nf.test @@ -41,17 +41,8 @@ nextflow_pipeline { assert !file("$launchDir/output/sample3/variants/clairsto/somatic.vcf.gz").exists() }, - // ── Phased VCFs exist and have data ────────────────────────── - { - ['sample1', 'sample2', 'sample3'].each { s -> - def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") - def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") - assert germline.exists() - assert somatic.exists() - assert germline.size() > 0 - assert somatic.size() > 0 - } - }, + // ── Phased VCFs: records present, all PASS, somatic arm tagged ── + { PhasedVcf.assertPassOnly("$outputDir", ['sample1', 'sample2', 'sample3'], ['sample3']) }, // ── BAM files ──────────────────────────────────────────────── { diff --git a/tests/default.nf.test b/tests/default.nf.test index 7c473eba..4e1d7d2c 100644 --- a/tests/default.nf.test +++ b/tests/default.nf.test @@ -22,6 +22,9 @@ nextflow_pipeline { def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') assertAll( { assert workflow.success}, + + // ── Phased VCFs: records present, all PASS, somatic arm tagged ── + { PhasedVcf.assertPassOnly("$outputDir", ['sample1', 'sample2', 'sample3'], ['sample3']) }, { //files exist assert file("$launchDir/output/sample1/variants/clair3/merge_output.vcf.gz").exists() assert file("$launchDir/output/sample1/variants/clair3/merge_output.vcf.gz.tbi").exists() diff --git a/tests/lib/PhasedVcf.groovy b/tests/lib/PhasedVcf.groovy new file mode 100644 index 00000000..12161928 --- /dev/null +++ b/tests/lib/PhasedVcf.groovy @@ -0,0 +1,44 @@ +import java.util.zip.GZIPInputStream + +// Record-level checks on the published phased VCFs, shared by the pipeline tests. +class PhasedVcf { + + // Non-header records of a bgzipped VCF, each split into its tab-separated fields + static List> records(String path) { + new GZIPInputStream(new FileInputStream(path)).withReader { reader -> + reader.readLines().findAll { !it.startsWith('#') }.collect { it.split('\t') as List } + } + } + + static boolean hasFlag(List rec, String flag) { + rec[7].split(';').contains(flag) + } + + // Value of a FORMAT field in the first sample column, or null if absent + static String format(List rec, String key) { + def i = rec[8].split(':').toList().indexOf(key) + def values = rec[9].split(':') + i >= 0 && i < values.size() ? values[i] : null + } + + static boolean isAlt(List rec) { + format(rec, 'GT').split(/[\/|]/).any { it != '0' && it != '.' } + } + + // Default-mode checks: both arms all PASS, somatic records all carry SOMATIC, germline not empty + static void assertPassOnly(String outdir, List samples, List tumourOnly) { + samples.each { s -> + def germ = records("${outdir}/${s}/variants/phased/germline_smallvariants.vcf.gz") + def som = records("${outdir}/${s}/variants/phased/somatic_smallvariants.vcf.gz") + assert !germ.isEmpty() : "${s}: phased germline VCF has no records" + assert germ.every { it[6] == 'PASS' } : "${s}: non-PASS germline record with the PASS filter on" + assert som.every { it[6] == 'PASS' } : "${s}: non-PASS somatic record with the PASS filter on" + assert som.every { hasFlag(it, 'SOMATIC') } : "${s}: somatic record without INFO/SOMATIC" + } + // Paired test samples have no PASS somatic calls, so the checks above only bite on tumour-only ones + tumourOnly.each { s -> + assert !records("${outdir}/${s}/variants/phased/somatic_smallvariants.vcf.gz").isEmpty() : + "${s}: tumour-only phased somatic VCF has no records" + } + } +} diff --git a/tests/union.nf.test b/tests/union.nf.test index 8acb4956..155ffa60 100644 --- a/tests/union.nf.test +++ b/tests/union.nf.test @@ -51,16 +51,19 @@ nextflow_pipeline { assert file("$launchDir/output/sample3/variants/clairsto/somatic.vcf.gz").exists() }, - // ── Phased union VCFs exist and have data ──────────────────── + // ── Phased VCFs: records present, all PASS, somatic arm tagged ── + { PhasedVcf.assertPassOnly("$outputDir", ['sample1', 'sample2', 'sample3'], ['sample3']) }, + // ── 'all' mode keeps one caller's record per position ──────── { ['sample1', 'sample2', 'sample3'].each { s -> - def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") - def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") - assert germline.exists() - assert somatic.exists() - assert germline.size() > 0 - assert somatic.size() > 0 + def germ = PhasedVcf.records("$outputDir/${s}/variants/phased/germline_smallvariants.vcf.gz") + def pos = germ.collect { "${it[0]}:${it[1]}" } + assert pos.size() == pos.toUnique().size() : "${s}: two germline records at one position" } + // DeepVariant calls T>A,G 1/2 here and Clair3 T>A 1/1; the prioritised Clair3 record is kept whole + def site = PhasedVcf.records("$outputDir/sample2/variants/phased/germline_smallvariants.vcf.gz") + .find { it[0] == 'chr19' && it[1] == '24897193' } + assert site && site[4] == 'A' && site[7].split(';').contains('CALLER=clair3') }, // ── BAM files ──────────────────────────────────────────────── @@ -141,15 +144,19 @@ nextflow_pipeline { assert !versions.text.contains('_PASS_FILTER') }, - // ── Phased VCFs still exist and have data ──────────────────── + // ── Phased VCFs keep non-PASS records; somatic 0/0 records are not phased ── { ['sample1', 'sample2', 'sample3'].each { s -> - def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") - def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") - assert germline.exists() - assert somatic.exists() - assert germline.size() > 0 - assert somatic.size() > 0 + def germ = PhasedVcf.records("$outputDir/${s}/variants/phased/germline_smallvariants.vcf.gz") + def som = PhasedVcf.records("$outputDir/${s}/variants/phased/somatic_smallvariants.vcf.gz") + assert !germ.isEmpty() : "${s}: phased germline VCF has no records" + assert !som.isEmpty() : "${s}: phased somatic VCF has no records" + assert germ.any { it[6] != 'PASS' } : "${s}: no non-PASS germline record with the filter off" + assert som.any { it[6] != 'PASS' } : "${s}: no non-PASS somatic record with the filter off" + assert som.every { PhasedVcf.hasFlag(it, 'SOMATIC') } : "${s}: somatic record without INFO/SOMATIC" + // LongPhase would copy a colliding germline record's phase onto a non-alt somatic record + assert som.findAll { !PhasedVcf.isAlt(it) }.every { PhasedVcf.format(it, 'PS') in [null, '.'] } : + "${s}: non-alt somatic record carries a phase set" } } ) From 24404b870b6b2afb32d4ee80fdea7f53b8886066 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 29 Sep 2026 15:45:12 +0200 Subject: [PATCH 31/36] Warn when tumour-only DeepVariant calls run without DeepSomatic's verdict Co-Authored-By: Claude Opus 5.5 (1M context) --- workflows/lrsomatic.nf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index 03517069..04898524 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -190,6 +190,15 @@ workflow LRSOMATIC { if (params.clairsto_cna_resources && !params.skip_ascat) { log.warn("--clairsto_cna_resources is ignored without --skip_ascat: Verdict's germline tagging then comes from ASCAT's purity and copy number.") } + // Tumour-only DeepVariant germline calls are adjudicated only by DeepSomatic's verdict; warn once if it is not run + if (params.germline_var_keep.contains('deepvariant') && !params.somatic_var_keep.contains('deepsomatic')) { + ch_samplesheet + .filter { meta, _bams -> !meta.paired_data } + .first() + .subscribe { meta, _bams -> + log.warn("Tumour-only samples (e.g. ${meta.id}) use DeepVariant germline calls without DeepSomatic's verdict, so they may include somatic variants. Add 'deepsomatic' to --somatic_var_keep to filter them.") + } + } // CHM13 has no ascat_loci_rt attribute, so the built set is GC-only by construction build_clairsto_cna = clairsto_cna_dir == null && params.genome == 'CHM13' && params.skip_ascat From 19827245cf0612efa39340a99075685c5bc11ec8 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 29 Sep 2026 15:45:12 +0200 Subject: [PATCH 32/36] Regenerate pipeline snapshots after the review fixes Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/chm13.nf.test.snap | 17 ++++++++++++++++- tests/clair_only.nf.test.snap | 17 ++++++++++++++++- tests/consensus.nf.test.snap | 17 ++++++++++++++++- tests/deep_only.nf.test.snap | 17 ++++++++++++++++- tests/default.nf.test.snap | 17 ++++++++++++++++- tests/union.nf.test.snap | 34 +++++++++++++++++++++++----------- 6 files changed, 103 insertions(+), 16 deletions(-) diff --git a/tests/chm13.nf.test.snap b/tests/chm13.nf.test.snap index 4e196867..c6dfc0a5 100644 --- a/tests/chm13.nf.test.snap +++ b/tests/chm13.nf.test.snap @@ -29,12 +29,18 @@ "CLAIRS_PASS_FILTER": { "bcftools": "1.23.1" }, + "CONCAT_SOMATIC_UNPHASED": { + "bcftools": 1.22 + }, "CRAMINO_POST": { "cramino": "1.3.0" }, "CRAMINO_PRE": { "cramino": "1.3.0" }, + "GERMLINE_ANCHORS": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -97,11 +103,20 @@ "SEVERUS": { "severus": 1.6 }, + "SOMATIC_ALT": { + "bcftools": 1.22 + }, + "SOMATIC_NONALT": { + "bcftools": 1.22 + }, "SOMATIC_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, "tabix": 1.21 }, + "SORT_SOMATIC_PHASED": { + "bcftools": 1.22 + }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -151,6 +166,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T12:34:22.954806822" + "timestamp": "2026-09-29T14:27:45.948524511" } } \ No newline at end of file diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index 557714f7..08fdf6aa 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -36,12 +36,18 @@ "CLAIRS_PASS_FILTER": { "bcftools": "1.23.1" }, + "CONCAT_SOMATIC_UNPHASED": { + "bcftools": 1.22 + }, "CRAMINO_POST": { "cramino": "1.3.0" }, "CRAMINO_PRE": { "cramino": "1.3.0" }, + "GERMLINE_ANCHORS": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -110,11 +116,20 @@ "SEVERUS": { "severus": 1.6 }, + "SOMATIC_ALT": { + "bcftools": 1.22 + }, + "SOMATIC_NONALT": { + "bcftools": 1.22 + }, "SOMATIC_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, "tabix": 1.21 }, + "SORT_SOMATIC_PHASED": { + "bcftools": 1.22 + }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -845,6 +860,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T12:28:35.531071958" + "timestamp": "2026-09-29T13:57:41.300856488" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index 03293479..7260fad2 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -44,6 +44,9 @@ "CLAIRS_PASS_FILTER": { "bcftools": "1.23.1" }, + "CONCAT_SOMATIC_UNPHASED": { + "bcftools": 1.22 + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -83,6 +86,9 @@ "DS_VERDICT_QUERY": { "bcftools": 1.22 }, + "GERMLINE_ANCHORS": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -145,6 +151,12 @@ "SEVERUS": { "severus": 1.6 }, + "SOMATIC_ALT": { + "bcftools": 1.22 + }, + "SOMATIC_NONALT": { + "bcftools": 1.22 + }, "SOMATIC_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -153,6 +165,9 @@ "SORT_POST_NORM": { "bcftools": 1.22 }, + "SORT_SOMATIC_PHASED": { + "bcftools": 1.22 + }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -695,6 +710,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T12:21:59.090749802" + "timestamp": "2026-09-29T13:31:57.487084446" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 9a25aad9..95f8fdb1 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -11,6 +11,9 @@ "BCFTOOLS_VIEW": { "bcftools": 1.22 }, + "CONCAT_SOMATIC_UNPHASED": { + "bcftools": 1.22 + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -50,6 +53,9 @@ "DS_VERDICT_QUERY": { "bcftools": 1.22 }, + "GERMLINE_ANCHORS": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -112,11 +118,20 @@ "SEVERUS": { "severus": 1.6 }, + "SOMATIC_ALT": { + "bcftools": 1.22 + }, + "SOMATIC_NONALT": { + "bcftools": 1.22 + }, "SOMATIC_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, "tabix": 1.21 }, + "SORT_SOMATIC_PHASED": { + "bcftools": 1.22 + }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -631,6 +646,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T12:56:04.936282901" + "timestamp": "2026-09-29T15:13:24.013345908" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 62439fb0..b36e5f38 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -26,12 +26,18 @@ "CLAIRS_PASS_FILTER": { "bcftools": "1.23.1" }, + "CONCAT_SOMATIC_UNPHASED": { + "bcftools": 1.22 + }, "CRAMINO_POST": { "cramino": "1.3.0" }, "CRAMINO_PRE": { "cramino": "1.3.0" }, + "GERMLINE_ANCHORS": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -94,11 +100,20 @@ "SEVERUS": { "severus": 1.6 }, + "SOMATIC_ALT": { + "bcftools": 1.22 + }, + "SOMATIC_NONALT": { + "bcftools": 1.22 + }, "SOMATIC_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, "tabix": 1.21 }, + "SORT_SOMATIC_PHASED": { + "bcftools": 1.22 + }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -623,6 +638,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T11:57:04.096725193" + "timestamp": "2026-09-29T13:03:39.7913195" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 42bea83c..9979ecac 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -8,15 +8,12 @@ "BCFTOOLS_CONCAT": { "bcftools": 1.22 }, - "BCFTOOLS_ISEC": { + "BCFTOOLS_EXCLUDE_SITES": { "bcftools": 1.22 }, "BCFTOOLS_NORM": { "bcftools": 1.22 }, - "BCFTOOLS_NORM_REJOIN": { - "bcftools": 1.22 - }, "BCFTOOLS_QUERY": { "bcftools": 1.22 }, @@ -41,6 +38,9 @@ "CLAIRS_PASS_FILTER": { "bcftools": "1.23.1" }, + "CONCAT_SOMATIC_UNPHASED": { + "bcftools": 1.22 + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -80,6 +80,9 @@ "DS_VERDICT_QUERY": { "bcftools": 1.22 }, + "GERMLINE_ANCHORS": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -142,6 +145,12 @@ "SEVERUS": { "severus": 1.6 }, + "SOMATIC_ALT": { + "bcftools": 1.22 + }, + "SOMATIC_NONALT": { + "bcftools": 1.22 + }, "SOMATIC_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -150,6 +159,9 @@ "SORT_POST_NORM": { "bcftools": 1.22 }, + "SORT_SOMATIC_PHASED": { + "bcftools": 1.22 + }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -634,8 +646,8 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,f0809c8be190289f21339696883dea6f", - "sample1_normal.bam.bai:md5,cbff669c086761e893a84ba3ec032b5f", + "sample1_normal.bam:md5,969a2009965e03c84ca3ec46c2fbbd38", + "sample1_normal.bam.bai:md5,62da15ac99896f0e90a694bf52ba19d3", "sample1_tumor.bam:md5,6267bf3e86a69536e45968b908d1cc6f", "sample1_tumor.bam.bai:md5,1ec6d68c92182ff9051b8707c4975445", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", @@ -645,8 +657,8 @@ "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", "sample1_whatshap_stats.gtf:md5,f5d331899db63b2ae21e51436bad2ddd", - "sample1_whatshap_stats.log:md5,111538b7e305189fb6de95ea978d06c8", - "sample1_whatshap_stats.tsv:md5,fa0fd5ce2b0919098ffa7d960cf177de", + "sample1_whatshap_stats.log:md5,0f455b7a59d41ee276844a1ac894971c", + "sample1_whatshap_stats.tsv:md5,41c0b3aeed7b8bc7770c7bce5d40cfcf", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", @@ -664,8 +676,8 @@ "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", - "sample2_whatshap_stats.log:md5,9308d0bd8dc9a4a86359f1ec926e6cb1", - "sample2_whatshap_stats.tsv:md5,b81282a609eca306912d4f7b7fdfeebb", + "sample2_whatshap_stats.log:md5,e7293b22e0b3ac0798b31d897d781336", + "sample2_whatshap_stats.tsv:md5,1e9adab54274feead2597e09f558222c", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -692,6 +704,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T12:04:10.544092583" + "timestamp": "2026-09-29T12:18:55.043468734" } } \ No newline at end of file From d17aa7bccd2f0b51c279ffb743925472afff2e98 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 30 Sep 2026 16:28:45 +0200 Subject: [PATCH 33/36] Find the downloaded AA data repo and surface AmpliconClassifier's log UNTAR strips the tarball's single GRCh38/ dir, but AC reads $AA_DATA_REPO//file_list.txt, so every downloaded-repo run failed. Link the staged dir under the ref name when it is the ref dir itself, and tee the log so a traceback reaches .command.out. Co-Authored-By: Claude Opus 5.5 (1M context) --- modules/local/ampliconclassifier/main.nf | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/local/ampliconclassifier/main.nf b/modules/local/ampliconclassifier/main.nf index 6060babe..820386c1 100644 --- a/modules/local/ampliconclassifier/main.nf +++ b/modules/local/ampliconclassifier/main.nf @@ -32,14 +32,21 @@ process AMPLICONCLASSIFIER { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - export AA_DATA_REPO=\$(readlink -f aa_data_repo) + # A downloaded repo arrives as the dir itself (UNTAR strips it); AC wants its parent + if [ -e aa_data_repo/file_list.txt ]; then + mkdir repo + ln -s "\$(readlink -f aa_data_repo)" repo/${ac_ref} + export AA_DATA_REPO=\$PWD/repo + else + export AA_DATA_REPO=\$(readlink -f aa_data_repo) + fi amplicon_classifier.py \\ --ref ${ac_ref} \\ --AA_results ${reconstruction} \\ -o ${prefix} \\ ${args} \\ - > ${prefix}_classifier.log 2>&1 + 2>&1 | tee ${prefix}_classifier.log """ stub: From 55473b007a676eecd6325004c485b86e7e6ba959 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 30 Sep 2026 16:28:45 +0200 Subject: [PATCH 34/36] Pin the AA data repo download by MD5 and skip AC when no repo exists AADATAREPO_DOWNLOAD replaces the WGET alias and checks --aa_data_repo_md5, defaulting on GRCh38 to the host-published MD5. CHM13 without --aa_data_repo now warns and skips the classifier instead of aborting at launch. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/igenomes.config | 2 + .../local/aadatarepo/download/environment.yml | 8 +++ modules/local/aadatarepo/download/main.nf | 52 +++++++++++++++++++ modules/local/aadatarepo/download/meta.yml | 48 +++++++++++++++++ .../aadatarepo/download/tests/main.nf.test | 38 ++++++++++++++ .../download/tests/main.nf.test.snap | 35 +++++++++++++ subworkflows/local/prepare_aa_data_repo.nf | 27 +++++++--- workflows/lrsomatic.nf | 4 +- 8 files changed, 205 insertions(+), 9 deletions(-) create mode 100644 modules/local/aadatarepo/download/environment.yml create mode 100644 modules/local/aadatarepo/download/main.nf create mode 100644 modules/local/aadatarepo/download/meta.yml create mode 100644 modules/local/aadatarepo/download/tests/main.nf.test create mode 100644 modules/local/aadatarepo/download/tests/main.nf.test.snap diff --git a/conf/igenomes.config b/conf/igenomes.config index c95d60e7..32ab6ddc 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -29,6 +29,8 @@ params.genomes = [ // Plain build, not GRCh38_indexed: the extra BWA index is for AmpliconArchitect's // alignment step, which AmpliconClassifier never runs. aa_data_repo_url : "https://refs.ampliconrepository.org/data/module_support_files/AmpliconArchitect/GRCh38.tar.gz", + // Published by the host as GRCh38_md5sum.txt; the tarball is updated in place + aa_data_repo_md5 : "2bcd1fdaed027466ed296bf78b2907eb", vep_alphamissense : "https://storage.googleapis.com/dm_alphamissense/AlphaMissense_hg38.tsv.gz", vep_alphamissense_tbi : "https://g-608c0c.273595.03c0.data.globus.org/VEP_plugins/AlphaMissense_hg38.tsv.gz.tbi", // A dated release rather than the rolling vcf_GRCh38/clinvar.vcf.gz, whose VCF and diff --git a/modules/local/aadatarepo/download/environment.yml b/modules/local/aadatarepo/download/environment.yml new file mode 100644 index 00000000..daa07275 --- /dev/null +++ b/modules/local/aadatarepo/download/environment.yml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + # renovate: datasource=conda depName=conda-forge/wget + - conda-forge::wget=1.21.4 diff --git a/modules/local/aadatarepo/download/main.nf b/modules/local/aadatarepo/download/main.nf new file mode 100644 index 00000000..f9056e34 --- /dev/null +++ b/modules/local/aadatarepo/download/main.nf @@ -0,0 +1,52 @@ +process AADATAREPO_DOWNLOAD { + tag "${url.toString().tokenize('/').last()}" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/3b/3b54fa9135194c72a18d00db6b399c03248103f87e43ca75e4b50d61179994b3/data' + : 'community.wave.seqera.io/library/wget:1.21.4--8b0fcde81c17be5e'}" + + input: + tuple val(meta), val(url), val(md5) + + output: + tuple val(meta), path("${archive_name}"), emit: archive + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + archive_name = url.toString().tokenize('/').last() + def check = md5 ? "echo '${md5} ${archive_name}' | md5sum -c -" : '' + """ + wget \\ + --no-verbose \\ + --tries=10 \\ + --waitretry=10 \\ + ${args} \\ + -O ${archive_name} \\ + ${url} + + # The tarball is updated in place, so a pinned MD5 fails the task on a re-published repo + ${check} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wget: \$(wget --version | head -1 | cut -d ' ' -f 3) + END_VERSIONS + """ + + stub: + archive_name = url.toString().tokenize('/').last() + """ + echo "" | gzip > ${archive_name} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wget: \$(wget --version | head -1 | cut -d ' ' -f 3) + END_VERSIONS + """ +} diff --git a/modules/local/aadatarepo/download/meta.yml b/modules/local/aadatarepo/download/meta.yml new file mode 100644 index 00000000..71fff961 --- /dev/null +++ b/modules/local/aadatarepo/download/meta.yml @@ -0,0 +1,48 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "aadatarepo_download" +description: Download the AmpliconArchitect data repository tarball that AmpliconClassifier reads, checked against a pinned MD5 +keywords: + - ecdna + - ampliconclassifier + - download +tools: + - "wget": + description: "GNU Wget is a free software package for retrieving files using HTTP, HTTPS, FTP and FTPS" + homepage: "https://www.gnu.org/software/wget/" + documentation: "https://www.gnu.org/software/wget/manual/wget.html" + licence: ["GPL-3.0-or-later"] + identifier: "" + +input: + - - meta: + type: map + description: Groovy Map containing an id for the data repository + - url: + type: string + description: URL of the data repository tarball; its basename becomes the output name + - md5: + type: string + description: | + Expected MD5 of the tarball. When set, a download that does not match fails + the task. May be null, in which case the repository is not verified. + +output: + archive: + - - meta: + type: map + description: Groovy Map containing an id for the data repository + - "${archive_name}": + type: file + description: The data repository tarball + pattern: "*.tar.gz" + versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" diff --git a/modules/local/aadatarepo/download/tests/main.nf.test b/modules/local/aadatarepo/download/tests/main.nf.test new file mode 100644 index 00000000..579696e0 --- /dev/null +++ b/modules/local/aadatarepo/download/tests/main.nf.test @@ -0,0 +1,38 @@ +nextflow_process { + + name "Test Process AADATAREPO_DOWNLOAD" + script "../main.nf" + process "AADATAREPO_DOWNLOAD" + + tag "small" + tag "modules" + tag "modules_local" + tag "aadatarepo_download" + + // Stub only: a real run downloads the 1.1 GB repository + test("GRCh38 data repo - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id: 'aa_data_repo' ], + 'https://refs.ampliconrepository.org/data/module_support_files/AmpliconArchitect/GRCh38.tar.gz', + '2bcd1fdaed027466ed296bf78b2907eb' + ] + """ + } + } + + then { + assert process.success + assertAll( + { assert file(process.out.archive[0][1]).name == 'GRCh38.tar.gz' }, + { assert snapshot(process.out).match() } + ) + } + + } +} diff --git a/modules/local/aadatarepo/download/tests/main.nf.test.snap b/modules/local/aadatarepo/download/tests/main.nf.test.snap new file mode 100644 index 00000000..083aa38f --- /dev/null +++ b/modules/local/aadatarepo/download/tests/main.nf.test.snap @@ -0,0 +1,35 @@ +{ + "GRCh38 data repo - stub": { + "content": [ + { + "0": [ + [ + { + "id": "aa_data_repo" + }, + "GRCh38.tar.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + "versions.yml:md5,3d07074650f444dad718bd48add94911" + ], + "archive": [ + [ + { + "id": "aa_data_repo" + }, + "GRCh38.tar.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,3d07074650f444dad718bd48add94911" + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-30T16:21:36.209439147" + } +} \ No newline at end of file diff --git a/subworkflows/local/prepare_aa_data_repo.nf b/subworkflows/local/prepare_aa_data_repo.nf index 822457fc..69386299 100644 --- a/subworkflows/local/prepare_aa_data_repo.nf +++ b/subworkflows/local/prepare_aa_data_repo.nf @@ -2,7 +2,7 @@ // Stage the AmpliconArchitect data repository AmpliconClassifier reads at runtime // -include { WGET as WGET_AA_DATA_REPO } from '../../modules/nf-core/wget/main' +include { AADATAREPO_DOWNLOAD } from '../../modules/local/aadatarepo/download/main' include { UNTAR as UNTAR_AA_DATA_REPO } from '../../modules/nf-core/untar/main' workflow PREPARE_AA_DATA_REPO { @@ -10,36 +10,47 @@ workflow PREPARE_AA_DATA_REPO { take: data_repo // path or null -- params.aa_data_repo repo_url // URL or null -- genome attribute aa_data_repo_url + repo_md5 // MD5 or null -- --aa_data_repo_md5, else the genome attribute main: ch_versions = channel.empty() + // The MD5 is checked by the download task, so a local repo would silently skip it + if (data_repo && params.aa_data_repo_md5) { + error("--aa_data_repo_md5: only checks a repository the pipeline downloads. Drop it when --aa_data_repo is set.") + } + if (data_repo) { ch_data_repo = channel.value([ [ id: 'aa_data_repo' ], file(data_repo, checkIfExists: true) ]) } else if (repo_url) { // - // MODULES: WGET_AA_DATA_REPO -> UNTAR_AA_DATA_REPO (labels: process_single) + // MODULES: AADATAREPO_DOWNLOAD -> UNTAR_AA_DATA_REPO (labels: process_single) // ~1.1 GB tarball; the plain build, not GRCh38_indexed, whose extra BWA index AC never reads // - WGET_AA_DATA_REPO ( - channel.value([ [ id: 'aa_data_repo' ], repo_url ]) + if (!repo_md5) { + log.warn("AmpliconClassifier: the data repository '${repo_url}' is downloaded without --aa_data_repo_md5, so the repository is not verified.") + } + AADATAREPO_DOWNLOAD ( + channel.value([ [ id: 'aa_data_repo' ], repo_url, repo_md5 ]) ) UNTAR_AA_DATA_REPO ( - WGET_AA_DATA_REPO.out.outfile + AADATAREPO_DOWNLOAD.out.archive ) // .first(): UNTAR emits a queue channel, and every sample's classifier task // needs the same repo -- without this only the first sample would get it. ch_data_repo = UNTAR_AA_DATA_REPO.out.untar.first() - ch_versions = ch_versions.mix(WGET_AA_DATA_REPO.out.versions, UNTAR_AA_DATA_REPO.out.versions) + ch_versions = ch_versions.mix(AADATAREPO_DOWNLOAD.out.versions, UNTAR_AA_DATA_REPO.out.versions) } else { - error("AmpliconClassifier needs an AmpliconArchitect data repository, which is not published for ${params.genome}. Set --aa_data_repo , or use --skip_ampliconclassifier.") + // An empty repo runs no classifier tasks; CoRAL reconstruction is unaffected + log.warn("No AmpliconArchitect data repository is published for ${params.genome}: skipping AmpliconClassifier. Set --aa_data_repo to classify.") + ch_data_repo = channel.empty() } emit: - data_repo = ch_data_repo // [[id:'aa_data_repo'], dir] + data_repo = ch_data_repo // [[id:'aa_data_repo'], dir], or empty when no repo is available versions = ch_versions } diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index 3b9fc394..b5c6b7e9 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -1314,9 +1314,11 @@ workflow LRSOMATIC { // The data repo is only fetched when the classifier will actually use it ch_aa_data_repo = channel.empty() if (!params.skip_ampliconclassifier) { + // Overriding the repo drops the default MD5, which belongs to the published tarball PREPARE_AA_DATA_REPO ( params.aa_data_repo, - params.aa_data_repo_url + params.aa_data_repo_url, + params.aa_data_repo_md5 ?: (params.aa_data_repo ? null : getGenomeAttribute('aa_data_repo_md5')) ) ch_aa_data_repo = PREPARE_AA_DATA_REPO.out.data_repo ch_versions = ch_versions.mix(PREPARE_AA_DATA_REPO.out.versions) From a4d9ee3a55daec9e7e874a0f3cafffd0845f18ef Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 30 Sep 2026 16:28:45 +0200 Subject: [PATCH 35/36] Warn on dropped CoRAL samples, plot the classified cycles, right-size CoRAL errorStrategy 'ignore' now has the warning its comment promised, for reconstruct and cycle_all. CORAL_PLOT takes the classifier's channel, so --coral_run_cycle plots what AC classifies. Reconstruct moves to process_medium and cycle to process_low. Adds ECDNA stub tests for the empty-seed branch and a run with no data repo. Co-Authored-By: Claude Opus 5.5 (1M context) --- modules/local/coral/cycle/main.nf | 3 +- modules/local/coral/reconstruct/main.nf | 2 +- modules/local/coral/seed/main.nf | 6 +- subworkflows/local/ecdna.nf | 28 +++++--- subworkflows/local/tests/ecdna.nf.test | 86 +++++++++++++++++++++++-- 5 files changed, 106 insertions(+), 19 deletions(-) diff --git a/modules/local/coral/cycle/main.nf b/modules/local/coral/cycle/main.nf index 2ca506cc..207e9c31 100644 --- a/modules/local/coral/cycle/main.nf +++ b/modules/local/coral/cycle/main.nf @@ -1,7 +1,8 @@ process CORAL_CYCLE { tag "$meta.id" - label 'process_high' + label 'process_low' + // As for CORAL_RECONSTRUCT: the subworkflow warns on the missing output errorStrategy { task.exitStatus in 130..145 ? 'retry' : 'ignore' } container "docker.io/robertaforsyth/coral:3.0.0-chm13-847f3d4" diff --git a/modules/local/coral/reconstruct/main.nf b/modules/local/coral/reconstruct/main.nf index fab0a1b7..76094ba8 100644 --- a/modules/local/coral/reconstruct/main.nf +++ b/modules/local/coral/reconstruct/main.nf @@ -1,6 +1,6 @@ process CORAL_RECONSTRUCT { tag "$meta.id" - label 'process_high' + label 'process_medium' // A single unsolvable amplicon should not fail a whole cohort; the subworkflow // warns on the missing output rather than letting the report slot go silent. diff --git a/modules/local/coral/seed/main.nf b/modules/local/coral/seed/main.nf index dc489a9d..2ee14460 100644 --- a/modules/local/coral/seed/main.nf +++ b/modules/local/coral/seed/main.nf @@ -34,9 +34,9 @@ process CORAL_SEED { stub: def prefix = task.ext.prefix ?: "${meta.id}" - // A non-empty seed, so the stub exercises the reconstruct path rather than the - // empty-seed branch. Use --coral_gain to force the empty case in a test. + // A non-empty seed by default; meta.stub_empty_seed, set only by the ECDNA test, reaches the empty-seed branch + def seed_cmd = meta.stub_empty_seed ? "touch ${prefix}_CNV_SEEDS.bed" : "printf 'chr1\\t100000\\t400000\\t8\\n' > ${prefix}_CNV_SEEDS.bed" """ - printf 'chr1\\t100000\\t400000\\t8\\n' > ${prefix}_CNV_SEEDS.bed + ${seed_cmd} """ } diff --git a/subworkflows/local/ecdna.nf b/subworkflows/local/ecdna.nf index 66c94197..2c39d5af 100644 --- a/subworkflows/local/ecdna.nf +++ b/subworkflows/local/ecdna.nf @@ -16,7 +16,7 @@ workflow ECDNA { tumor_bam // [meta, bam, bai] -- tumour BAMs only ascat_cnvs // [meta, cnvs_txt] -- ASCAT.out.cnvs fai // [[:], fai] -- value channel, reused by every sample - data_repo // [[:], dir] -- value channel; empty with --skip_ampliconclassifier + data_repo // [[:], dir] -- value channel; empty with --skip_ampliconclassifier or no repo coral_ref // val 'hg38' | 't2t' ac_ref // val 'GRCh38' | 'CHM13' @@ -38,8 +38,7 @@ workflow ECDNA { // Input: [meta, cn_seg_bed, bam, bai] // Output: .seeds -- [meta, bed] -- amplified intervals above --gain // - // multiMap, not two reads of one channel: a channel feeding both a process and an - // operator is not allowed, and reconstruct needs the same cn_seg/bam that seed used. + // multiMap: seed, reconstruct and plot each take the same cn_seg/bam in their own shape ASCAT_TO_CORAL_BED.out.bed .join(tumor_bam, failOnMismatch: true, failOnDuplicate: true) .multiMap { meta, cn_seg, bam, bai -> @@ -62,7 +61,7 @@ workflow ECDNA { // CORAL_SEED.out.seeds .branch { _meta, bed -> - seeded: bed.toFile().length() > 0 + seeded: bed.size() > 0 unseeded: true } .set { branched_seeds } @@ -71,7 +70,7 @@ workflow ECDNA { .subscribe { meta, _bed -> log.info("No amplified intervals found for ${meta.id}: skipping ecDNA reconstruction.") } // - // MODULE: CORAL_RECONSTRUCT (label: process_high) + // MODULE: CORAL_RECONSTRUCT (label: process_medium) // Input: [meta, seeds, cn_seg_bed, bam, bai] // Output: .reconstruction -- [meta, dir] -- graph/cycles/summary, named as AC expects // @@ -85,8 +84,15 @@ workflow ECDNA { coral_reconstruct_input ) + // errorStrategy 'ignore' drops a failed sample silently, so name it here + branched_seeds.seeded + .map { meta, _seeds -> [meta] } + .join(CORAL_RECONSTRUCT.out.reconstruction, remainder: true) + .filter { _meta, dir -> dir == null } + .subscribe { meta, _dir -> log.warn("CoRAL reconstruct failed for ${meta.id}: no ecDNA results for this sample.") } + // - // MODULE: CORAL_CYCLE (label: process_high) -- opt-in cycle re-extraction + // MODULE: CORAL_CYCLE (label: process_low) -- opt-in cycle re-extraction // Input: [meta, reconstruction_dir] // Output: .reconstruction -- [meta, dir] -- re-extracted cycles beside the copied graphs // @@ -94,6 +100,12 @@ workflow ECDNA { CORAL_CYCLE ( CORAL_RECONSTRUCT.out.reconstruction ) + + CORAL_RECONSTRUCT.out.reconstruction + .map { meta, _dir -> [meta] } + .join(CORAL_CYCLE.out.reconstruction, remainder: true) + .filter { _meta, dir -> dir == null } + .subscribe { meta, _dir -> log.warn("CoRAL cycle_all failed for ${meta.id}: it will not be classified.") } ch_for_classifier = CORAL_CYCLE.out.reconstruction } else { @@ -104,11 +116,11 @@ workflow ECDNA { // // MODULE: CORAL_PLOT (label: process_medium) // Runs beside the classifier rather than in front of it: plotting is cosmetic - // and must never gate classification. + // and must never gate classification. Plots the same cycles the classifier reads. // ch_plots = channel.empty() if (params.coral_plot) { - CORAL_RECONSTRUCT.out.reconstruction + ch_for_classifier .join(coral_inputs.plot, failOnDuplicate: true) .set { coral_plot_input } // coral_plot_input: [meta, reconstruction_dir, bam, bai] diff --git a/subworkflows/local/tests/ecdna.nf.test b/subworkflows/local/tests/ecdna.nf.test index 99df521e..f0553c07 100644 --- a/subworkflows/local/tests/ecdna.nf.test +++ b/subworkflows/local/tests/ecdna.nf.test @@ -11,7 +11,7 @@ nextflow_workflow { // Stub-only: CoRAL and AmpliconClassifier need real BAMs, a solver and a ~1 GB data // repo. What these prove is the wiring -- that the joins pair, that the empty-seed - // branch drops a sample before reconstruct, and that the params gate the optional steps. + // branch drops a sample before reconstruct, and that the params and repo gate the optional steps. test("-stub - reconstructs and classifies a seeded sample") { options "-stub" @@ -51,14 +51,14 @@ nextflow_workflow { } } - test("-stub - coral_run_cycle classifies the re-extracted cycles") { + test("-stub - coral_run_cycle classifies and plots the re-extracted cycles") { options "-stub" when { params { coral_run_cycle = true - coral_plot = false + coral_plot = true skip_ampliconclassifier = false } workflow { @@ -79,9 +79,9 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert names.contains('ECDNA:CORAL_CYCLE') }, - // Plotting is off, and must not gate the classifier - { assert !names.contains('ECDNA:CORAL_PLOT') }, - { assert names.contains('ECDNA:AMPLICONCLASSIFIER') } + { assert names.contains('ECDNA:CORAL_PLOT') }, + { assert names.contains('ECDNA:AMPLICONCLASSIFIER') }, + { assert workflow.out.plots.size() == 1 } ) } } @@ -118,4 +118,78 @@ nextflow_workflow { ) } } + + test("-stub - an empty seed skips reconstruction and classification") { + + options "-stub" + + when { + params { + coral_run_cycle = false + coral_plot = true + skip_ampliconclassifier = false + } + workflow { + """ + // stub_empty_seed makes the CORAL_SEED stub write an empty BED; the joins match on the whole meta + input[0] = channel.of([ [ id:'sample1', stub_empty_seed:true ], file('sample1.bam'), file('sample1.bam.bai') ]) + input[1] = channel.of([ [ id:'sample1', stub_empty_seed:true ], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) ]) + input[2] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/chr.fai", checkIfExists: true) ]) + input[3] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures") ]) + input[4] = 't2t' + input[5] = 'CHM13' + """ + } + } + + then { + def names = workflow.trace.tasks().collect { it.name.split(' ')[0] } + + assertAll( + { assert workflow.success }, + { assert names.contains('ECDNA:CORAL_SEED') }, + { assert !names.contains('ECDNA:CORAL_RECONSTRUCT') }, + { assert !names.contains('ECDNA:CORAL_PLOT') }, + { assert !names.contains('ECDNA:AMPLICONCLASSIFIER') }, + { assert workflow.out.reconstruction.size() == 0 }, + { assert workflow.out.classification.size() == 0 } + ) + } + } + + test("-stub - no data repo reconstructs without classifying") { + + options "-stub" + + when { + params { + coral_run_cycle = false + coral_plot = false + skip_ampliconclassifier = false + } + workflow { + """ + // PREPARE_AA_DATA_REPO emits nothing on CHM13 without --aa_data_repo + input[0] = channel.of([ [ id:'sample1' ], file('sample1.bam'), file('sample1.bam.bai') ]) + input[1] = channel.of([ [ id:'sample1' ], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/sample1.cnvs.txt", checkIfExists: true) ]) + input[2] = channel.value([ [:], file("\${projectDir}/modules/local/ascattocoralbed/tests/fixtures/chr.fai", checkIfExists: true) ]) + input[3] = channel.empty() + input[4] = 't2t' + input[5] = 'CHM13' + """ + } + } + + then { + def names = workflow.trace.tasks().collect { it.name.split(' ')[0] } + + assertAll( + { assert workflow.success }, + { assert names.contains('ECDNA:CORAL_RECONSTRUCT') }, + { assert !names.contains('ECDNA:AMPLICONCLASSIFIER') }, + { assert workflow.out.reconstruction.size() == 1 }, + { assert workflow.out.classification.size() == 0 } + ) + } + } } From de94f9bf3ec02113182196f5607313eb439b3cd4 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 30 Sep 2026 16:28:45 +0200 Subject: [PATCH 36/36] Config and docs for the review fixes Tie CoRAL solver threads to task.cpus and drop coral_solver_threads; mount the Gurobi licence with --volume outside Singularity/Apptainer; stop the AA repo download and unpack being published; add aa_data_repo_md5 to the schema; fix CHANGELOG PR links and test claims; prettier. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 ++--- conf/modules.config | 22 +++++++++++------ docs/output.md | 31 ++++++++++++----------- docs/usage.md | 22 ++++++++++++----- nextflow.config | 2 +- nextflow_schema.json | 59 +++++++++++++------------------------------- 6 files changed, 68 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86b1204f..85e82f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Added` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Added ecDNA and focal amplification analysis: CoRAL reconstructs amplicon structures from each tumour BAM, seeded by ASCAT's copy-number calls, and AmpliconClassifier labels each amplicon as ecDNA, BFB or linear. Runs by default on tumour samples with ASCAT calls; turn it off with `--skip_coral`, or keep reconstruction without classification with `--skip_ampliconclassifier`. CoRAL's optimisation defaults to the open-source SCIP solver rather than Gurobi: across the 1102 solver logs from the previous standalone cohort the largest model was 620 rows by 438 columns and the slowest solve took 0.49 s, so a licensed solver buys nothing at this scale, though `--coral_solver gurobi_direct` with `--gurobi_license` remains available. AmpliconClassifier's AmpliconArchitect data repository is downloaded automatically for GRCh38 and must be supplied with `--aa_data_repo` for CHM13, which has no published repository (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Added `ascat_to_coral_bed.py` and the `ASCAT_TO_CORAL_BED` module, converting ASCAT's `cnvs.txt` into the headerless BED CoRAL seeds from. Contigs are respelled to match the reference, since ASCAT writes `1` where the BAM may say `chr1` and CoRAL builds its chromosome sizes from the BAM header; zero-length and inverted segments are dropped, which CoRAL's own parser would otherwise reject (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Added stub nf-tests for the `ECDNA` subworkflow and `ASCAT_TO_CORAL_BED` (tag `small`), covering the empty-seed branch, the opt-in cycle re-extraction and the `--skip_ampliconclassifier` path (@robert-a-forsyth). +- [#205](https://github.com/IntGenomicsLab/lrsomatic/pull/205) - Added ecDNA and focal amplification analysis: CoRAL reconstructs amplicon structures from each tumour BAM, seeded by ASCAT's copy-number calls, and AmpliconClassifier labels each amplicon as ecDNA, BFB or linear. Runs by default on tumour samples with ASCAT calls; turn it off with `--skip_coral`, or keep reconstruction without classification with `--skip_ampliconclassifier`. CoRAL's optimisation defaults to the open-source SCIP solver rather than Gurobi: across the 1102 solver logs from the previous standalone cohort the largest model was 620 rows by 438 columns and the slowest solve took 0.49 s, so a licensed solver buys nothing at this scale, though `--coral_solver gurobi_direct` with `--gurobi_license` remains available. AmpliconClassifier's AmpliconArchitect data repository is downloaded automatically for GRCh38, checked against a pinned `--aa_data_repo_md5`, and can be supplied with `--aa_data_repo` (the reference directory or its parent); CHM13 has no published repository, so without `--aa_data_repo` the classifier is skipped with a warning (@robert-a-forsyth). +- [#205](https://github.com/IntGenomicsLab/lrsomatic/pull/205) - Added `ascat_to_coral_bed.py` and the `ASCAT_TO_CORAL_BED` module, converting ASCAT's `cnvs.txt` into the headerless BED CoRAL seeds from. Contigs are respelled to match the reference, since ASCAT writes `1` where the BAM may say `chr1` and CoRAL builds its chromosome sizes from the BAM header; zero-length and inverted segments are dropped, which CoRAL's own parser would otherwise reject (@robert-a-forsyth). +- [#205](https://github.com/IntGenomicsLab/lrsomatic/pull/205) - Added stub nf-tests for the `ECDNA` subworkflow and `ASCAT_TO_CORAL_BED` (tag `small`), covering the seeded path, the empty-seed branch, the opt-in cycle re-extraction with plotting, the `--skip_ampliconclassifier` path and a run with no data repository, plus a stub test for `AADATAREPO_DOWNLOAD` (@robert-a-forsyth). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - Added CHM13 support for ClairS-TO's Verdict module, which tags tumour-only calls as germline, somatic or subclonal somatic; its resources were GRCh38-only, so on CHM13 germline variants leaked into `somatic.vcf.gz`. With `--genome CHM13 --skip_ascat` the pipeline builds a CHM13 resource set from the ASCAT files it already downloads and passes it as `--cna_resource_dir`; a prepared directory can be given with `--clairsto_cna_resources` (validated at launch). Without `--skip_ascat` tagging comes from ASCAT's own tables instead (next entry) (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - Added `CLAIRSTO_VERDICT_TAG`: when ASCAT is in the run, Verdict's germline tagging is computed from ASCAT's purity, ploidy and segments instead of Verdict's own estimate, so `CLAIRSTO` runs with `--disable_verdict` and ASCAT runs before small variant calling. Output names are unchanged. The tables the tags were computed from are published as `_Tumor_Purity_Ploidy.txt` and `_Tumor_CNA.txt`, also on `--skip_ascat` runs (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - Added a stub nf-test for `TUMORONLY_SMALLVAR` covering both germline tagging paths (tag `small`) (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 5a6732f2..198687f0 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -703,10 +703,11 @@ process { withName: '.*:CORAL_.*' { // Gurobi licences are user-supplied and mounted; SCIP, the default, needs nothing. + // Apptainer/Singularity mount with --bind, Docker/Podman with --volume. containerOptions = { - params.coral_solver == 'gurobi_direct' && params.gurobi_license - ? "--bind ${params.gurobi_license}:/opt/gurobi/gurobi.lic:ro --env GRB_LICENSE_FILE=/opt/gurobi/gurobi.lic" - : null + if (params.coral_solver != 'gurobi_direct' || !params.gurobi_license) { return null } + def mount = workflow.containerEngine in ['singularity', 'apptainer'] ? '--bind' : '--volume' + "${mount} ${params.gurobi_license}:/opt/gurobi/gurobi.lic:ro --env GRB_LICENSE_FILE=/opt/gurobi/gurobi.lic" } publishDir = [ path: { "${params.outdir}/${meta.id}/ecdna/coral" }, @@ -732,7 +733,7 @@ process { ext.args = { [ "--solver ${params.coral_solver}", - "--solver-threads ${params.coral_solver_threads}", + "--solver-threads ${task.cpus}", "--solver-time-limit ${params.coral_solver_time_limit}", "--global-time-limit ${params.coral_global_time_limit}", "--min-bp-support ${params.coral_min_bp_support}", @@ -747,7 +748,7 @@ process { ext.args = { [ "--solver ${params.coral_solver}", - "--threads ${params.coral_solver_threads}", + "--threads ${task.cpus}", "--solver-time-limit ${params.coral_solver_time_limit}", "--global-time-limit ${params.coral_global_time_limit}", "--alpha ${params.coral_cycle_decomp_alpha}", @@ -823,7 +824,7 @@ process { ] } - withName : '.*:UNTAR' { + withName : '.*:UNTAR(_AA_DATA_REPO)?' { publishDir = [ enabled: false ] @@ -843,10 +844,17 @@ process { } // The wget container carries no CA bundle, as for WGET above; the pinned MD5 still checks the download - withName : '.*:VEPPLUGIN_CLINVAR' { + withName : '.*:(VEPPLUGIN_CLINVAR|AADATAREPO_DOWNLOAD)' { ext.args = { "--no-check-certificate" } } + // Unpacked into the work dir only: pass --aa_data_repo to reuse an extracted copy + withName : '.*:AADATAREPO_DOWNLOAD' { + publishDir = [ + enabled: false + ] + } + // Published so a later run can skip both the download and the reshaping by pointing // --vep_revel / --vep_eve / --vep_clinvar (and their _tbi) at these files withName : '.*:VEPPLUGIN_(REVEL|EVE|CLINVAR)' { diff --git a/docs/output.md b/docs/output.md index decfb60d..f0bd0515 100644 --- a/docs/output.md +++ b/docs/output.md @@ -368,7 +368,8 @@ amplicon (ecDNA, BFB, linear, and so on). Runs on every tumour sample with ASCAT A sample with no segment above `--coral_gain` produces an empty seed BED and is skipped with a log message rather than failing. CoRAL defaults to the open-source SCIP solver; `--coral_solver gurobi_direct` is faster but needs `--gurobi_license`. AmpliconClassifier needs an AmpliconArchitect -data repository, downloaded automatically for GRCh38 and supplied with `--aa_data_repo` for CHM13. +data repository, downloaded automatically for GRCh38 and supplied with `--aa_data_repo` for CHM13; +a CHM13 run without one skips classification with a warning. ``` ├── ecdna @@ -391,20 +392,20 @@ data repository, downloaded automatically for GRCh38 and supplied with `--aa_dat │ └── sample_classification_bed_files/ ``` -| File | Description | -| ----------------------------------------------- | ------------------------------------------------------------------------------ | -| `sample_coral_cn.bed` | ASCAT's copy number as the BED CoRAL seeds from | -| `sample_CNV_SEEDS.bed` | Amplified intervals above `--coral_gain`; empty means no amplicons | -| `sample_amplicon_graph.txt` | Breakpoint graph per amplicon, in AmpliconArchitect format | -| `sample_amplicon_cycles.txt` | Decomposed cycles and paths per amplicon | -| `sample_summary.txt` | Per-run amplicon summary; written even when no amplicon is found | -| `sample_reconstruct.log` | CoRAL reconstruction log, including solver output | -| `sample_amplicon_{graph,cycles}.png` | Per-amplicon copy-number and cycle plots | -| `sample_amplicon_classification_profiles.tsv` | The headline call per amplicon: ecDNA+, BFB+, decomposition class | -| `sample_gene_list.tsv` | Genes intersecting each classified amplicon | -| `sample_ecDNA_counts.tsv` | Number of distinct ecDNA species detected | -| `sample_result_table.tsv` | Combined per-sample table, the format AmpliconRepository ingests | -| `sample_classification_bed_files/` | Per-feature BED intervals for each classified amplicon | +| File | Description | +| --------------------------------------------- | ------------------------------------------------------------------ | +| `sample_coral_cn.bed` | ASCAT's copy number as the BED CoRAL seeds from | +| `sample_CNV_SEEDS.bed` | Amplified intervals above `--coral_gain`; empty means no amplicons | +| `sample_amplicon_graph.txt` | Breakpoint graph per amplicon, in AmpliconArchitect format | +| `sample_amplicon_cycles.txt` | Decomposed cycles and paths per amplicon | +| `sample_summary.txt` | Per-run amplicon summary; written even when no amplicon is found | +| `sample_reconstruct.log` | CoRAL reconstruction log, including solver output | +| `sample_amplicon_{graph,cycles}.png` | Per-amplicon copy-number and cycle plots | +| `sample_amplicon_classification_profiles.tsv` | The headline call per amplicon: ecDNA+, BFB+, decomposition class | +| `sample_gene_list.tsv` | Genes intersecting each classified amplicon | +| `sample_ecDNA_counts.tsv` | Number of distinct ecDNA species detected | +| `sample_result_table.tsv` | Combined per-sample table, the format AmpliconRepository ingests | +| `sample_classification_bed_files/` | Per-feature BED intervals for each classified amplicon | #### `savana` diff --git a/docs/usage.md b/docs/usage.md index 30dc8a55..5089434b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -705,9 +705,10 @@ licence. Gurobi is available with: --coral_solver gurobi_direct --gurobi_license /path/to/gurobi.lic ``` -The licence is bind-mounted into the CoRAL tasks; it is never baked into the image. Under Gurobi the +The licence is mounted into the CoRAL tasks (`--bind` under Singularity/Apptainer, `--volume` under +Docker/Podman), so give an absolute path; it is never baked into the image. Under Gurobi the reconstruction steps are serialised (`maxForks = 1`), because a Web License Service licence caps -concurrent solver sessions. +concurrent solver sessions, and the solver uses the task's CPUs. SCIP runs single-threaded. Gurobi is faster, but on the models this pipeline produces that has not mattered: across 1102 solver logs from the earlier standalone cohort, the largest model was 620 rows by 438 columns and the @@ -717,15 +718,24 @@ you have measured a reason not to. ### AmpliconClassifier reference data AmpliconClassifier reads an AmpliconArchitect data repository at runtime. For `--genome GRCh38` it is -downloaded automatically (about 1.1 GB) and needs no configuration. For `--genome CHM13` there is no -published repository, so one must be supplied: +downloaded automatically (about 1.1 GB) and checked against the MD5 the host publishes +(`--aa_data_repo_md5`), so a re-published repository fails the run rather than changing results +silently. The download and its ~4 GB unpack repeat on every run; to avoid both, extract +`GRCh38.tar.gz` once and pass it: + +```bash +--aa_data_repo /path/to/GRCh38 +``` + +For `--genome CHM13` there is no published repository. Without `--aa_data_repo` the classifier is +skipped with a warning and reconstruction still runs: ```bash --genome CHM13 --aa_data_repo /path/to/AA_DATA_REPO ``` -The directory must contain a `CHM13/` subdirectory. Alternatively run reconstruction alone with -`--skip_ampliconclassifier`, which needs no reference data. +`--aa_data_repo` may be the reference directory itself (`GRCh38/`, `CHM13/`) or the directory that +contains it. `--skip_ampliconclassifier` drops classification explicitly and needs no reference data. ### Tuning diff --git a/nextflow.config b/nextflow.config index a1864918..437856a1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -83,7 +83,6 @@ params { coral_min_seed_size = 100000 coral_max_seg_gap = 300000 coral_solver = 'scip' - coral_solver_threads = -1 coral_solver_time_limit = 7200 coral_global_time_limit = 21600 coral_min_bp_support = 1.75 @@ -92,6 +91,7 @@ params { coral_plot = true gurobi_license = null aa_data_repo = null + aa_data_repo_md5 = null // Skip options skip_qc = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 866c0ebc..d04c3942 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,10 +10,7 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": [ - "input", - "outdir" - ], + "required": ["input", "outdir"], "properties": { "input": { "type": "string", @@ -84,37 +81,25 @@ "type": "string", "description": "When two germline callers are used, specifies how to combine them. 'consensus' keeps only variants called by both callers; 'all' keeps all variants from both callers.", "default": "all", - "enum": [ - "consensus", - "all" - ] + "enum": ["consensus", "all"] }, "somatic_var_combine": { "type": "string", "description": "When two somatic callers are used, specifies how to combine them. 'consensus' keeps only variants called by both callers; 'all' keeps all variants from both callers.", "default": "all", - "enum": [ - "consensus", - "all" - ] + "enum": ["consensus", "all"] }, "prioritize_caller_germline": { "type": "string", "description": "When both germline callers are used, specifies whose record to keep where both callers call a position. Must be [deepvariant, clair].", "default": "clair", - "enum": [ - "deepvariant", - "clair" - ] + "enum": ["deepvariant", "clair"] }, "prioritize_caller_somatic": { "type": "string", "description": "When both somatic callers are used, specifies whose record to keep where both callers call a position. Must be [deepsomatic, clair].", "default": "clair", - "enum": [ - "deepsomatic", - "clair" - ] + "enum": ["deepsomatic", "clair"] }, "smallvar_filter_pass": { "type": "boolean", @@ -170,10 +155,7 @@ "description": "Name of iGenomes reference.", "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details.", - "enum": [ - "GRCh38", - "CHM13" - ] + "enum": ["GRCh38", "CHM13"] }, "igenomes_ignore": { "type": "boolean", @@ -700,14 +682,7 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": [ - "symlink", - "rellink", - "link", - "copy", - "copyNoFollow", - "move" - ], + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true }, "email_on_fail": { @@ -814,17 +789,9 @@ "coral_solver": { "type": "string", "default": "scip", - "enum": [ - "scip", - "gurobi_direct" - ], + "enum": ["scip", "gurobi_direct"], "description": "Optimiser for CoRAL's non-convex MIQCP cycle decomposition. SCIP is open-source and needs no licence; gurobi_direct is faster but requires --gurobi_license." }, - "coral_solver_threads": { - "type": "integer", - "default": -1, - "description": "Solver threads; -1 lets the solver decide (CoRAL's --solver-threads)." - }, "coral_solver_time_limit": { "type": "integer", "default": 7200, @@ -862,7 +829,15 @@ "aa_data_repo": { "type": "string", "format": "directory-path", - "description": "AmpliconArchitect data repository for AmpliconClassifier. Downloaded automatically for GRCh38; required for CHM13, which has no published repo." + "description": "AmpliconArchitect data repository for AmpliconClassifier: the directory holding `GRCh38/` or `CHM13/`, or that reference directory itself. Downloaded automatically for GRCh38; CHM13 has no published repo, so without one the classifier is skipped with a warning.", + "help_text": "Downloading the GRCh38 repo fetches 1.1 GB and unpacks about 4 GB on every run. Extract `GRCh38.tar.gz` once and pass the directory here to skip both." + }, + "aa_data_repo_md5": { + "type": "string", + "description": "Expected MD5 of the downloaded AmpliconArchitect data repository tarball.", + "fa_icon": "fas fa-fingerprint", + "pattern": "^[0-9a-fA-F]{32}$", + "help_text": "The GRCh38 default carries the MD5 the host publishes, and a tarball that does not match fails the run, so the repository cannot change silently. A local --aa_data_repo takes no MD5, and setting --aa_data_repo drops the default." } } }