From 59c2216e21d717aec67adba7ab390b40f2ae9172 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 18 Sep 2026 11:38:19 +0200 Subject: [PATCH 01/25] make passing variants the only form pushed through to downstream analysis in Deepvariant by default (toggleable) --- CHANGELOG.md | 1 + conf/modules.config | 22 +++++++++ docs/usage.md | 18 ++++++++ nextflow.config | 2 + nextflow_schema.json | 14 ++++++ .../local/paired/paired_smallvar_germline.nf | 19 ++++++++ .../local/paired/paired_smallvar_somatic.nf | 19 ++++++++ .../local/tumor_only/tumoronly_smallvar.nf | 38 +++++++++++++++ tests/consensus.nf.test.snap | 42 +++++++++-------- tests/deep_only.nf.test.snap | 26 +++++++---- tests/union.nf.test.snap | 46 +++++++++++-------- 11 files changed, 199 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41af5f5d..42204985 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - DeepVariant and DeepSomatic output is now restricted to `PASS` records before it is used downstream, controlled by the new `--deepvariant_filter_pass` and `--deepsomatic_filter_pass` parameters (both `true` by default). Both callers emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. ClairS/ClairS-TO calls are already `PASS`-only downstream (`VCFSPLIT`), so with the default `*_var_combine = 'all'` the union was "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP, signature fitting and the report see far fewer variants. Set `--deepsomatic_filter_pass false` or `--deepvariant_filter_pass false` for the previous behaviour. The raw per-caller VCFs published under `variants/deepsomatic` and `variants/deepvariant` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). - [#186](https://github.com/IntGenomicsLab/lrsomatic/pull/186) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.0, which adds tickbox dropdown filters on the categorical columns of both variant tables and turns the report's gene panel selector into checkboxes (@ljwharbers). - [#176](https://github.com/IntGenomicsLab/lrsomatic/pull/176) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.2: facet dropdown counts follow the active filters, opening a facet menu no longer resets the table's horizontal scroll, a flatter clinical theme, and inline code comments trimmed to one line (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 9a46a239..b360097b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -650,6 +650,28 @@ process { ] } + // DeepVariant and DeepSomatic emit every evaluated site, not just calls. These + // filtered copies are what the consensus, phasing, VEP, signature and report steps + // consume; the raw per-caller VCFs under variants/deepvariant and variants/deepsomatic + // are published from the POSTPROCESSVARIANTS processes and are left untouched. + // --write-index is required because the module's index output is optional, and the + // downstream join() would otherwise drop every sample. + withName : '.*:DEEPVARIANT_PASS_FILTER' { + ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } + ext.prefix = { "${meta.id}_germline_pass" } + publishDir = [ + enabled: false + ] + } + + withName : '.*:DEEPSOMATIC_PASS_FILTER' { + ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } + ext.prefix = { "${meta.id}_somatic_pass" } + publishDir = [ + enabled: false + ] + } + withName : '.*:SIGPROFILER_MATRIXGENERATOR' { ext.args = { params.sigprofiler_matrix_args ?: '' } publishDir = [ diff --git a/docs/usage.md b/docs/usage.md index 33dcf18e..8b29b096 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -361,6 +361,24 @@ These options control how variants from multiple callers are filtered and merged | `--somatic_var_combine` | Strategy for combining somatic variant caller outputs (e.g. union, intersection). Default = `null` | | `--prioritize_caller_germline` | Comma-separated caller priority order used when combining germline calls. Default = `null` | | `--prioritize_caller_somatic` | Comma-separated caller priority order used when combining somatic calls. Default = `null` | +| `--deepvariant_filter_pass` | Keep only PASS records from DeepVariant for downstream use. Default = `true` | +| `--deepsomatic_filter_pass` | Keep only PASS records from DeepSomatic for downstream use. Default = `true` | + +DeepVariant and DeepSomatic emit a record for every site they evaluate, not only +for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds +around 13.7 M records of which roughly 50 k are `PASS`, the rest being `RefCall`, +`GERMLINE` or `PON`. ClairS/ClairS-TO output is already restricted to `PASS` +before it is used downstream, so with `*_var_combine = 'all'` the union would +otherwise be "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at", +which inflates the phased VCFs by three orders of magnitude and produces a +meaningless mutation burden. + +`--deepvariant_filter_pass` and `--deepsomatic_filter_pass` (both `true` by +default) restrict the copy handed to the caller consensus, phasing, VEP, signature +fitting and the report. Set either to `false` to restore the previous unfiltered +behaviour. The raw per-caller VCFs published under +`//variants/deepvariant` and `//variants/deepsomatic` +are never filtered, so no calls are lost from the results directory. #### PON Options diff --git a/nextflow.config b/nextflow.config index ada7c82a..f64eef4a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -20,6 +20,8 @@ params { somatic_var_combine = 'all' prioritize_caller_germline = 'clair' prioritize_caller_somatic = 'clair' + deepvariant_filter_pass = true + deepsomatic_filter_pass = true generate_gvcf = false // Longphase options diff --git a/nextflow_schema.json b/nextflow_schema.json index d5e93d85..e1e17365 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -101,6 +101,20 @@ "default": "clair", "enum": ["deepsomatic", "clair"] }, + "deepvariant_filter_pass": { + "type": "boolean", + "default": true, + "description": "Keep only PASS records from DeepVariant for downstream use.", + "help_text": "DeepVariant emits every evaluated site, so most records are RefCall rather than calls. Those records otherwise flow into the caller consensus, phasing, VEP and the report. Set to false to consume the unfiltered DeepVariant output. The raw VCF published to `//variants/deepvariant` is unaffected either way.", + "fa_icon": "fas fa-filter" + }, + "deepsomatic_filter_pass": { + "type": "boolean", + "default": true, + "description": "Keep only PASS records from DeepSomatic for downstream use.", + "help_text": "DeepSomatic emits every evaluated site, so most records are RefCall, GERMLINE or PON rather than somatic calls. Those records otherwise flow into the caller consensus, phasing, VEP, signature fitting and the report. Set to false to consume the unfiltered DeepSomatic output. The raw VCF published to `//variants/deepsomatic` is unaffected either way.", + "fa_icon": "fas fa-filter" + }, "generate_gvcf": { "type": "boolean" } diff --git a/subworkflows/local/paired/paired_smallvar_germline.nf b/subworkflows/local/paired/paired_smallvar_germline.nf index 3b473006..0ae9a23c 100644 --- a/subworkflows/local/paired/paired_smallvar_germline.nf +++ b/subworkflows/local/paired/paired_smallvar_germline.nf @@ -1,5 +1,6 @@ // IMPORT MODULES include { CLAIR3 } from '../../../modules/local/clair3/main.nf' +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' @@ -121,6 +122,24 @@ workflow PAIRED_SMALLVAR_GERMLINE { DEEPVARIANT.out.vcf .join(DEEPVARIANT.out.vcf_index) + .set{deepvariant_raw_ch} + + // DeepVariant emits every evaluated site, not just calls, so most records are + // RefCall. Clair3 is already PASS-filtered downstream, so without this the + // union is "PASS Clair3 + every site DeepVariant evaluated". + // The raw VCF published under variants/deepvariant/ is unaffected. + if (params.deepvariant_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) + DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index) + .set{deepvariant_pass_ch} + } + else { + deepvariant_raw_ch + .set{deepvariant_pass_ch} + } + + deepvariant_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/paired/paired_smallvar_somatic.nf b/subworkflows/local/paired/paired_smallvar_somatic.nf index cf5a749d..d6ac19c4 100644 --- a/subworkflows/local/paired/paired_smallvar_somatic.nf +++ b/subworkflows/local/paired/paired_smallvar_somatic.nf @@ -2,6 +2,7 @@ include { CLAIRS } from '../../../modules/local/clairs/main.nf' include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat' include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' @@ -111,6 +112,24 @@ workflow PAIRED_SMALLVAR_SOMATIC { DEEPSOMATIC.out.vcf .join(DEEPSOMATIC.out.vcf_index) + .set{deepsomatic_raw_ch} + + // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. + // ClairS is already PASS-filtered downstream, so without this the union is + // "PASS ClairS + every site DeepSomatic evaluated". + // The raw VCF published under variants/deepsomatic/ is unaffected. + if (params.deepsomatic_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) + DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index) + .set{deepsomatic_pass_ch} + } + else { + deepsomatic_raw_ch + .set{deepsomatic_pass_ch} + } + + deepsomatic_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 10f18b59..796bc0ab 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -1,6 +1,8 @@ // IMPORT MODULES include { CLAIRSTO } from '../../../modules/local/clairsto/main.nf' include { VCFSPLIT } from '../../../modules/local/vcfsplit/main.nf' +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' @@ -126,6 +128,24 @@ workflow TUMORONLY_SMALLVAR { DEEPVARIANT.out.vcf .join(DEEPVARIANT.out.vcf_index) + .set{deepvariant_raw_ch} + + // DeepVariant emits every evaluated site, not just calls, so most records are + // RefCall. ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without + // this the union is "PASS ClairS-TO + every site DeepVariant evaluated". + // The raw VCF published under variants/deepvariant/ is unaffected. + if (params.deepvariant_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) + DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index) + .set{deepvariant_pass_ch} + } + else { + deepvariant_raw_ch + .set{deepvariant_pass_ch} + } + + deepvariant_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] @@ -194,6 +214,24 @@ workflow TUMORONLY_SMALLVAR { ) DEEPSOMATIC.out.vcf .join(DEEPSOMATIC.out.vcf_index) + .set{deepsomatic_raw_ch} + + // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. + // ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without this the + // union is "PASS ClairS-TO + every site DeepSomatic evaluated". + // The raw VCF published under variants/deepsomatic/ is unaffected. + if (params.deepsomatic_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) + DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index) + .set{deepsomatic_pass_ch} + } + else { + deepsomatic_raw_ch + .set{deepsomatic_pass_ch} + } + + deepsomatic_pass_ch .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index 5123680d..d0a4150e 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -47,6 +47,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -56,6 +59,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -575,38 +581,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,93dd8ac8b67eb4eb4bf27e09c8f5f99b", - "sample1_normal.bam.bai:md5,75402ef1cc35229cc131155d9ec973e0", - "sample1_tumor.bam:md5,69cba03cad51bcc1d1ee8c48da042527", - "sample1_tumor.bam.bai:md5,5d633ed05021ad81ce24b1f18cbf38b4", + "sample1_normal.bam:md5,28ad28688852dfe5c2527c88fe52d2c7", + "sample1_normal.bam.bai:md5,3f1a1e308eb5f7b6a3805e76ff7034a3", + "sample1_tumor.bam:md5,e25dec2664b709de329909c5f1be74f2", + "sample1_tumor.bam.bai:md5,04d8b757427e0ff00b19aa338c681a54", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,9f09f9ad1a788384cb8e46a933f77b3b", - "sample1_whatshap_stats.log:md5,20135b4e9965a31d3f9bb0df7d2cec90", - "sample1_whatshap_stats.tsv:md5,264d2d76a9b8d34ea4933aee325ce36e", + "sample1_whatshap_stats.gtf:md5,19483ff8a25ef082cde1d2c7d5faac6f", + "sample1_whatshap_stats.log:md5,ede92982bd2c122b30c5625ff1627c46", + "sample1_whatshap_stats.tsv:md5,4c347cec00fefec30623219a865e3eb3", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,f2ba30d007c521d479c6158e1e22367a", - "sample2_normal.bam.bai:md5,c3096f52115ec1e24c46fedc41f1f3d3", - "sample2_tumor.bam:md5,1c0287d24fa5b25b86e48024f2f55031", - "sample2_tumor.bam.bai:md5,62849cea5a005e3d8dbe8f9edcefaf60", + "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", + "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", + "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,f15fb43f0af73d02fc73b66fdc12d5d8", - "sample2_whatshap_stats.log:md5,ca87088fc2f11665eca3fb9c80489085", - "sample2_whatshap_stats.tsv:md5,ca53f81e39bf5d46aa4f604216add1f6", + "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", + "sample2_whatshap_stats.log:md5,f9628717e0c4944888687c13098c62eb", + "sample2_whatshap_stats.tsv:md5,4fd295d10a8c461fd2c91851027503a4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -629,10 +635,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:55:50.117252839", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T11:10:43.315178155" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 6a48eeaa..4c20adca 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -23,6 +23,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -32,6 +35,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -531,8 +537,8 @@ "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", "sample1_whatshap_stats.gtf:md5,e1d0e87353a5f9aed8a9ac4bf7973427", - "sample1_whatshap_stats.log:md5,bd6b83a062e22cd3201523dc4c2c13e7", - "sample1_whatshap_stats.tsv:md5,7a1508751cb1daa841a577ae25f55586", + "sample1_whatshap_stats.log:md5,43811a1aa4726c2ef62646ff22d1fad0", + "sample1_whatshap_stats.tsv:md5,a821c5d0e3451f82327645fee89e68eb", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", @@ -550,8 +556,8 @@ "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", "sample2_whatshap_stats.gtf:md5,af33281699a1d0da83fbe7eaff198d03", - "sample2_whatshap_stats.log:md5,bbd9ab2ce07a009d9348a1d78bc6fc70", - "sample2_whatshap_stats.tsv:md5,c65436f930c23ddbfd568532d07dce70", + "sample2_whatshap_stats.log:md5,0cd536df69e244a5271e9c5440ea0f3f", + "sample2_whatshap_stats.tsv:md5,c7cc47024ef622a72e7f4f0d5fa119eb", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -564,8 +570,8 @@ "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,4f7648e763004ab764143cb4f8b6499e", - "sample3_whatshap_stats.tsv:md5,4cb58bb3b663aaba23da004d69adab3e", + "sample3_whatshap_stats.log:md5,061ecb622d83678229f5d0a7b0e95d75", + "sample3_whatshap_stats.tsv:md5,7c7453a5eed17e9ccc405844f3bd511a", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -574,10 +580,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:51:30.278291697", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T11:19:11.135286604" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 08d81773..2e110695 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -44,6 +44,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -53,6 +56,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -575,38 +581,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,cbfc940a38c74cbe8435c18b9da5dd32", - "sample1_normal.bam.bai:md5,c1498328929d45b2898fa2265b0d617c", - "sample1_tumor.bam:md5,b78866edf991393806d37505d16f7e3d", - "sample1_tumor.bam.bai:md5,f613de14ab19fc3a85403661a4f6188c", + "sample1_normal.bam:md5,cd28bdb7edefa810b86a12029783e1b6", + "sample1_normal.bam.bai:md5,6c19fcce6bd9050fb03a00e66ba1b866", + "sample1_tumor.bam:md5,dcb89119e272efbe0168d1737f714a8a", + "sample1_tumor.bam.bai:md5,9577c39e6f59056421a8192b0ba43acc", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,9ae556e13516dd47d4108acf2104bddb", - "sample1_whatshap_stats.log:md5,eaddcf6a1666d4a3c1ad3316dac24139", - "sample1_whatshap_stats.tsv:md5,c2773e011c2781160fd9a7741b10546b", + "sample1_whatshap_stats.gtf:md5,43773d064cd353d1dbbae45adb3be334", + "sample1_whatshap_stats.log:md5,13f4530236189960576b5bc45e63bc1a", + "sample1_whatshap_stats.tsv:md5,bf14b5a1fd9d95797f36a48e7e16953d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,f2ba30d007c521d479c6158e1e22367a", - "sample2_normal.bam.bai:md5,c3096f52115ec1e24c46fedc41f1f3d3", - "sample2_tumor.bam:md5,1c0287d24fa5b25b86e48024f2f55031", - "sample2_tumor.bam.bai:md5,62849cea5a005e3d8dbe8f9edcefaf60", + "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", + "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", + "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,f15fb43f0af73d02fc73b66fdc12d5d8", - "sample2_whatshap_stats.log:md5,a6767b3490cafdcbaf3b7114644028de", - "sample2_whatshap_stats.tsv:md5,570796e5e291229e8872733425e0b133", + "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", + "sample2_whatshap_stats.log:md5,af84d397bf163be8de77fc3c7401493b", + "sample2_whatshap_stats.tsv:md5,fb38bde18fbfc61e52947b68d4260829", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -619,8 +625,8 @@ "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,679dcfa209888a9e69a07e4c4e4b049e", - "sample3_whatshap_stats.tsv:md5,035d5aa0425ba3fc32d65268b793b424", + "sample3_whatshap_stats.log:md5,496a531c3c19585603fa7fded0c958ba", + "sample3_whatshap_stats.tsv:md5,fba638c8da0d7a9fdacb11734e219b16", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -629,10 +635,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:55:27.445799687", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T11:35:45.671299133" } } \ No newline at end of file From c596e1b92f66b73286378cf87000ed672de9e2bf Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 18 Sep 2026 18:04:28 +0200 Subject: [PATCH 02/25] fixes to the filtering --- CHANGELOG.md | 2 +- conf/modules.config | 38 ++++++++---- docs/usage.md | 26 ++++----- nextflow.config | 3 +- nextflow_schema.json | 13 +---- .../local/paired/paired_smallvar_germline.nf | 39 ++++++------- .../local/paired/paired_smallvar_somatic.nf | 39 ++++++------- .../local/tumor_only/tumoronly_smallvar.nf | 58 ++++++------------- subworkflows/local/vcf_pass_filter.nf | 40 +++++++++++++ tests/clair_only.nf.test.snap | 43 +++++++------- tests/consensus.nf.test.snap | 35 +++++------ tests/deep_only.nf.test.snap | 11 ++-- tests/default.nf.test.snap | 39 +++++++------ tests/union.nf.test | 42 ++++++++++++++ tests/union.nf.test.snap | 35 +++++------ 15 files changed, 259 insertions(+), 204 deletions(-) create mode 100644 subworkflows/local/vcf_pass_filter.nf diff --git a/CHANGELOG.md b/CHANGELOG.md index 42204985..e316511a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - DeepVariant and DeepSomatic output is now restricted to `PASS` records before it is used downstream, controlled by the new `--deepvariant_filter_pass` and `--deepsomatic_filter_pass` parameters (both `true` by default). Both callers emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. ClairS/ClairS-TO calls are already `PASS`-only downstream (`VCFSPLIT`), so with the default `*_var_combine = 'all'` the union was "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP, signature fitting and the report see far fewer variants. Set `--deepsomatic_filter_pass false` or `--deepvariant_filter_pass false` for the previous behaviour. The raw per-caller VCFs published under `variants/deepsomatic` and `variants/deepvariant` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. Clair3 and ClairS are far less extreme but still kept their `LowQual` and `NonSomatic` records, so with the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`, and signature fitting is unchanged because `SIGNATURES_BCFTOOLS_VIEW` already applied its own `PASS` filter. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP and the report see far fewer variants. Set `--smallvar_filter_pass false` for the previous behaviour. The per-caller VCFs published under `variants/deepvariant`, `variants/deepsomatic`, `variants/clair3` and `variants/clairs` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). - [#186](https://github.com/IntGenomicsLab/lrsomatic/pull/186) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.0, which adds tickbox dropdown filters on the categorical columns of both variant tables and turns the report's gene panel selector into checkboxes (@ljwharbers). - [#176](https://github.com/IntGenomicsLab/lrsomatic/pull/176) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.2: facet dropdown counts follow the active filters, opening a facet menu no longer resets the table's horizontal scroll, a flatter clinical theme, and inline code comments trimmed to one line (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index b360097b..078a1db1 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -650,23 +650,41 @@ process { ] } - // DeepVariant and DeepSomatic emit every evaluated site, not just calls. These - // filtered copies are what the consensus, phasing, VEP, signature and report steps - // consume; the raw per-caller VCFs under variants/deepvariant and variants/deepsomatic - // are published from the POSTPROCESSVARIANTS processes and are left untouched. + // VCF_PASS_FILTER copies, one alias per small variant caller. DeepVariant and + // DeepSomatic emit every evaluated site rather than only their calls, and Clair3/ClairS + // keep their LowQual and NonSomatic records; these filtered copies are what the + // consensus, phasing, VEP and report steps consume. The per-caller VCFs under + // variants// are published from the caller processes and are left untouched. + // ClairS-TO has no entry here: VCFSPLIT already restricts it to PASS. // --write-index is required because the module's index output is optional, and the // downstream join() would otherwise drop every sample. - withName : '.*:DEEPVARIANT_PASS_FILTER' { - ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } - ext.prefix = { "${meta.id}_germline_pass" } + withName: '.*:DEEPVARIANT_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_deepvariant_pass" } publishDir = [ enabled: false ] } - withName : '.*:DEEPSOMATIC_PASS_FILTER' { - ext.args = { "--apply-filters PASS --output-type z --write-index=tbi" } - ext.prefix = { "${meta.id}_somatic_pass" } + withName: '.*:DEEPSOMATIC_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_deepsomatic_pass" } + publishDir = [ + enabled: false + ] + } + + withName: '.*:CLAIR3_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_clair3_pass" } + publishDir = [ + enabled: false + ] + } + + withName: '.*:CLAIRS_PASS_FILTER:PASS_FILTER' { + ext.args = '--apply-filters PASS --output-type z --write-index=tbi' + ext.prefix = { "${meta.id}_clairs_pass" } publishDir = [ enabled: false ] diff --git a/docs/usage.md b/docs/usage.md index 8b29b096..488fd69f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -361,24 +361,22 @@ These options control how variants from multiple callers are filtered and merged | `--somatic_var_combine` | Strategy for combining somatic variant caller outputs (e.g. union, intersection). Default = `null` | | `--prioritize_caller_germline` | Comma-separated caller priority order used when combining germline calls. Default = `null` | | `--prioritize_caller_somatic` | Comma-separated caller priority order used when combining somatic calls. Default = `null` | -| `--deepvariant_filter_pass` | Keep only PASS records from DeepVariant for downstream use. Default = `true` | -| `--deepsomatic_filter_pass` | Keep only PASS records from DeepSomatic for downstream use. Default = `true` | +| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds around 13.7 M records of which roughly 50 k are `PASS`, the rest being `RefCall`, -`GERMLINE` or `PON`. ClairS/ClairS-TO output is already restricted to `PASS` -before it is used downstream, so with `*_var_combine = 'all'` the union would -otherwise be "PASS Clair calls plus every site DeepVariant/DeepSomatic looked at", -which inflates the phased VCFs by three orders of magnitude and produces a -meaningless mutation burden. - -`--deepvariant_filter_pass` and `--deepsomatic_filter_pass` (both `true` by -default) restrict the copy handed to the caller consensus, phasing, VEP, signature -fitting and the report. Set either to `false` to restore the previous unfiltered -behaviour. The raw per-caller VCFs published under -`//variants/deepvariant` and `//variants/deepsomatic` -are never filtered, so no calls are lost from the results directory. +`GERMLINE` or `PON`. Clair3 and ClairS are far less extreme but still keep their +`LowQual` and `NonSomatic` records. With `*_var_combine = 'all'` the union would +otherwise be "every site every caller looked at", which inflates the phased VCFs +by three orders of magnitude and produces a meaningless mutation burden. + +`--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's +VCF that is handed to the caller consensus, phasing, VEP and the report. Set it to +`false` to restore the previous unfiltered behaviour. In tumor-only mode ClairS-TO +is unaffected by the setting: `VCFSPLIT` already restricts it to `PASS`. The +per-caller VCFs published under `//variants/` are never +filtered, so no calls are lost from the results directory. #### PON Options diff --git a/nextflow.config b/nextflow.config index f64eef4a..3e213473 100644 --- a/nextflow.config +++ b/nextflow.config @@ -20,8 +20,7 @@ params { somatic_var_combine = 'all' prioritize_caller_germline = 'clair' prioritize_caller_somatic = 'clair' - deepvariant_filter_pass = true - deepsomatic_filter_pass = true + smallvar_filter_pass = true generate_gvcf = false // Longphase options diff --git a/nextflow_schema.json b/nextflow_schema.json index e1e17365..4dc48241 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -101,18 +101,11 @@ "default": "clair", "enum": ["deepsomatic", "clair"] }, - "deepvariant_filter_pass": { + "smallvar_filter_pass": { "type": "boolean", "default": true, - "description": "Keep only PASS records from DeepVariant for downstream use.", - "help_text": "DeepVariant emits every evaluated site, so most records are RefCall rather than calls. Those records otherwise flow into the caller consensus, phasing, VEP and the report. Set to false to consume the unfiltered DeepVariant output. The raw VCF published to `//variants/deepvariant` is unaffected either way.", - "fa_icon": "fas fa-filter" - }, - "deepsomatic_filter_pass": { - "type": "boolean", - "default": true, - "description": "Keep only PASS records from DeepSomatic for downstream use.", - "help_text": "DeepSomatic emits every evaluated site, so most records are RefCall, GERMLINE or PON rather than somatic calls. Those records otherwise flow into the caller consensus, phasing, VEP, signature fitting and the report. Set to false to consume the unfiltered DeepSomatic output. The raw VCF published to `//variants/deepsomatic` is unaffected either way.", + "description": "Keep only PASS records from each small variant caller for downstream use.", + "help_text": "DeepVariant and DeepSomatic emit a record for every site they evaluate, so most records are RefCall (or GERMLINE/PON) rather than calls, and Clair3/ClairS keep their LowQual and NonSomatic calls. Those records otherwise flow into the caller consensus, phasing, VEP and the report. Set to false to consume each caller's unfiltered output. The per-caller VCFs published under `//variants/` are unaffected either way.", "fa_icon": "fas fa-filter" }, "generate_gvcf": { diff --git a/subworkflows/local/paired/paired_smallvar_germline.nf b/subworkflows/local/paired/paired_smallvar_germline.nf index 0ae9a23c..798e5b3f 100644 --- a/subworkflows/local/paired/paired_smallvar_germline.nf +++ b/subworkflows/local/paired/paired_smallvar_germline.nf @@ -1,10 +1,11 @@ // IMPORT MODULES include { CLAIR3 } from '../../../modules/local/clair3/main.nf' -include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' +include { VCF_PASS_FILTER as CLAIR3_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_GERMLINE { @@ -74,8 +75,14 @@ workflow PAIRED_SMALLVAR_GERMLINE { fai ) - CLAIR3.out.vcf - .join(CLAIR3.out.tbi) + // Clair3's merge_output.vcf.gz keeps LowQual and RefCall records; restrict them the + // same way DeepVariant's output is restricted so both callers enter the union on + // equal terms. The VCF published under variants/clair3/ is unaffected. + CLAIR3_PASS_FILTER ( + CLAIR3.out.vcf.join(CLAIR3.out.tbi) + ) + + CLAIR3_PASS_FILTER.out.vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clair3'] return [new_meta, vcf, tbi] @@ -120,26 +127,14 @@ workflow PAIRED_SMALLVAR_GERMLINE { [[:],[]] // GFF annotation (not used) ) - DEEPVARIANT.out.vcf - .join(DEEPVARIANT.out.vcf_index) - .set{deepvariant_raw_ch} - - // DeepVariant emits every evaluated site, not just calls, so most records are - // RefCall. Clair3 is already PASS-filtered downstream, so without this the - // union is "PASS Clair3 + every site DeepVariant evaluated". - // The raw VCF published under variants/deepvariant/ is unaffected. - if (params.deepvariant_filter_pass) { - DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) - DEEPVARIANT_PASS_FILTER.out.vcf - .join(DEEPVARIANT_PASS_FILTER.out.index) - .set{deepvariant_pass_ch} - } - else { - deepvariant_raw_ch - .set{deepvariant_pass_ch} - } + // DeepVariant emits a record for every site it evaluates, not just its calls, so + // most records are RefCall. Without this the union is "every site DeepVariant + // looked at". The VCF published under variants/deepvariant/ is unaffected. + DEEPVARIANT_PASS_FILTER ( + DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + ) - deepvariant_pass_ch + DEEPVARIANT_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/paired/paired_smallvar_somatic.nf b/subworkflows/local/paired/paired_smallvar_somatic.nf index d6ac19c4..953ed01d 100644 --- a/subworkflows/local/paired/paired_smallvar_somatic.nf +++ b/subworkflows/local/paired/paired_smallvar_somatic.nf @@ -2,11 +2,12 @@ include { CLAIRS } from '../../../modules/local/clairs/main.nf' include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat' include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort' -include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' +include { VCF_PASS_FILTER as CLAIRS_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_SOMATIC { @@ -72,8 +73,14 @@ workflow PAIRED_SMALLVAR_SOMATIC { BCFTOOLS_CONCAT.out.vcf ) - BCFTOOLS_SORT.out.vcf - .join(BCFTOOLS_SORT.out.tbi) + // The concatenated ClairS VCF still carries its LowQual and NonSomatic calls; + // restrict them the same way DeepSomatic's output is restricted so both callers + // enter the union on equal terms. variants/clairs/ is unaffected. + CLAIRS_PASS_FILTER ( + BCFTOOLS_SORT.out.vcf.join(BCFTOOLS_SORT.out.tbi) + ) + + CLAIRS_PASS_FILTER.out.vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clairs'] return [new_meta, vcf, tbi] @@ -110,26 +117,14 @@ workflow PAIRED_SMALLVAR_SOMATIC { ds_pon_channel ) - DEEPSOMATIC.out.vcf - .join(DEEPSOMATIC.out.vcf_index) - .set{deepsomatic_raw_ch} - - // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. - // ClairS is already PASS-filtered downstream, so without this the union is - // "PASS ClairS + every site DeepSomatic evaluated". - // The raw VCF published under variants/deepsomatic/ is unaffected. - if (params.deepsomatic_filter_pass) { - DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) - DEEPSOMATIC_PASS_FILTER.out.vcf - .join(DEEPSOMATIC_PASS_FILTER.out.index) - .set{deepsomatic_pass_ch} - } - else { - deepsomatic_raw_ch - .set{deepsomatic_pass_ch} - } + // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), + // not just its calls. Without this the union is "every site DeepSomatic looked + // at". The VCF published under variants/deepsomatic/ is unaffected. + DEEPSOMATIC_PASS_FILTER ( + DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + ) - deepsomatic_pass_ch + DEEPSOMATIC_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 796bc0ab..299bc0c7 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -1,14 +1,14 @@ // IMPORT MODULES include { CLAIRSTO } from '../../../modules/local/clairsto/main.nf' include { VCFSPLIT } from '../../../modules/local/vcfsplit/main.nf' -include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' -include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' +include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow TUMORONLY_SMALLVAR { @@ -126,26 +126,15 @@ workflow TUMORONLY_SMALLVAR { [[:],[]] // GFF annotation (not used) ) - DEEPVARIANT.out.vcf - .join(DEEPVARIANT.out.vcf_index) - .set{deepvariant_raw_ch} - - // DeepVariant emits every evaluated site, not just calls, so most records are - // RefCall. ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without - // this the union is "PASS ClairS-TO + every site DeepVariant evaluated". - // The raw VCF published under variants/deepvariant/ is unaffected. - if (params.deepvariant_filter_pass) { - DEEPVARIANT_PASS_FILTER ( deepvariant_raw_ch, [], [], [] ) - DEEPVARIANT_PASS_FILTER.out.vcf - .join(DEEPVARIANT_PASS_FILTER.out.index) - .set{deepvariant_pass_ch} - } - else { - deepvariant_raw_ch - .set{deepvariant_pass_ch} - } + // DeepVariant emits a record for every site it evaluates, not just its calls, so + // most records are RefCall. ClairS-TO needs no equivalent step here because + // VCFSPLIT already restricts it to PASS. The VCF published under + // variants/deepvariant/ is unaffected. + DEEPVARIANT_PASS_FILTER ( + DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + ) - deepvariant_pass_ch + DEEPVARIANT_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] @@ -212,26 +201,15 @@ workflow TUMORONLY_SMALLVAR { [[:],[]], // GZI (empty if FASTA is uncompressed) ds_pon_channel ) - DEEPSOMATIC.out.vcf - .join(DEEPSOMATIC.out.vcf_index) - .set{deepsomatic_raw_ch} - - // DeepSomatic emits every evaluated site (RefCall/GERMLINE/PON), not just calls. - // ClairS-TO is already PASS-filtered downstream by VCFSPLIT, so without this the - // union is "PASS ClairS-TO + every site DeepSomatic evaluated". - // The raw VCF published under variants/deepsomatic/ is unaffected. - if (params.deepsomatic_filter_pass) { - DEEPSOMATIC_PASS_FILTER ( deepsomatic_raw_ch, [], [], [] ) - DEEPSOMATIC_PASS_FILTER.out.vcf - .join(DEEPSOMATIC_PASS_FILTER.out.index) - .set{deepsomatic_pass_ch} - } - else { - deepsomatic_raw_ch - .set{deepsomatic_pass_ch} - } + // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), + // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT + // already restricts it to PASS. The VCF published under variants/deepsomatic/ is + // unaffected. + DEEPSOMATIC_PASS_FILTER ( + DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + ) - deepsomatic_pass_ch + DEEPSOMATIC_PASS_FILTER.out.vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/vcf_pass_filter.nf b/subworkflows/local/vcf_pass_filter.nf new file mode 100644 index 00000000..51f0833a --- /dev/null +++ b/subworkflows/local/vcf_pass_filter.nf @@ -0,0 +1,40 @@ +// IMPORT MODULES +// Aliased so the software-versions report keeps this separate from the local +// BCFTOOLS_VIEW used by PHASING_HAPLOTYPING, which is a different bcftools build. +include { BCFTOOLS_VIEW as PASS_FILTER } from '../../modules/nf-core/bcftools/view/main' + +// +// SUBWORKFLOW: VCF_PASS_FILTER +// Restrict a per-caller VCF to its PASS records before it is handed to the caller +// consensus, phasing, VEP and the report. DeepVariant and DeepSomatic emit a record for +// every site they evaluate (RefCall/GERMLINE/PON), and Clair3/ClairS keep their LowQual +// and NonSomatic calls, so without this the union is "every site every caller looked at". +// The per-caller VCFs published under variants// are produced elsewhere and are +// never filtered, so no calls are lost from the results directory. +// +// Include once per caller under an alias so conf/modules.config can give each one its own +// prefix, e.g. include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER }. +// +workflow VCF_PASS_FILTER { + + take: + vcfs // [meta, vcf, tbi] + + main: + if (params.smallvar_filter_pass) { + // --write-index is set in conf/modules.config because the module's index output is + // optional, and the join() below would otherwise drop every sample. + PASS_FILTER ( vcfs, [], [], [] ) + + PASS_FILTER.out.vcf + .join(PASS_FILTER.out.index) + .set{ filtered } + } + else { + vcfs + .set{ filtered } + } + + emit: + vcf = filtered // [meta, vcf, tbi] +} diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index e3db716b..1a13f798 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -3,11 +3,11 @@ "content": [ "88c8d3cf9cb49fdbc53372b2275d5f3e" ], - "timestamp": "2026-09-02T10:24:37.511974063", "meta": { "nf-test": "0.9.4", "nextflow": "26.04.3" - } + }, + "timestamp": "2026-09-02T10:24:37.511974063" }, "-profile test, clair only, extended samplesheet": { "content": [ @@ -74,6 +74,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -717,38 +720,38 @@ "sample5/vep/somatic/sample5_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,772e41f7cd86c03a22afbe5ec0592a6b", - "sample1_normal.bam.bai:md5,1b501f6a11efe5d2e6f47b7f1523220b", - "sample1_tumor.bam:md5,c8315c80dc92dfb5d874aef3f5dd46fb", - "sample1_tumor.bam.bai:md5,bc35f807be4b93fc795a14d701469367", + "sample1_normal.bam:md5,9a73c3f90bc4d9a140bcf8f652b0f269", + "sample1_normal.bam.bai:md5,37a866c569f24ed2b38f093f9475b9d6", + "sample1_tumor.bam:md5,dc15bf0e9ff1d401491347c15c408b6d", + "sample1_tumor.bam.bai:md5,fa57db4206692079d9b2084a866bf411", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,eff050a68e36e778b06e0ec19435c569", - "sample1_whatshap_stats.log:md5,76b73731f74fe32ef2d11f6bb0a0f71a", - "sample1_whatshap_stats.tsv:md5,f566ae25b3c5a8f7e94b3d6c1b0417f8", + "sample1_whatshap_stats.gtf:md5,36bda647c08358df0eac0be321c24b20", + "sample1_whatshap_stats.log:md5,938f792fd22bb658a2c7cb1b7653035f", + "sample1_whatshap_stats.tsv:md5,cf8917be389dfef1344eeb6b7e99c7c5", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,3157bd11ba095a884c7951aafcfcfb1c", - "sample2_normal.bam.bai:md5,edebda44c4383173caea728acde4ac43", - "sample2_tumor.bam:md5,47b2c5f86e0493ba94ff72cea77eeae3", - "sample2_tumor.bam.bai:md5,abf2c290c815f54c2b3f8179f717d9bd", + "sample2_normal.bam:md5,c18bbb1bc05b0ec830fa1ac3c6ef542f", + "sample2_normal.bam.bai:md5,de77684ef264476b3e0531b61ca23740", + "sample2_tumor.bam:md5,8b8cb5ac7668b8ac2f4097f584481d35", + "sample2_tumor.bam.bai:md5,3b46456b0e7b00688519dfb25c8e3c88", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,4d8f4393e3aebe4e945c0b8236cf3b3e", - "sample2_whatshap_stats.log:md5,10bba7bae6dd99b989ece5e5dac7a8f9", - "sample2_whatshap_stats.tsv:md5,bb46226e486af9026ab76e014624e903", + "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", + "sample2_whatshap_stats.log:md5,75a69a8e651979e25467d6e3c84cdf90", + "sample2_whatshap_stats.tsv:md5,92d7234e355833b1ec6f54951c38c09d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -797,10 +800,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:54:42.01587726", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T16:39:52.362456747" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index d0a4150e..c7b025c3 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -47,9 +47,6 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, - "DEEPSOMATIC_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -59,9 +56,6 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, - "DEEPVARIANT_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -103,6 +97,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -581,27 +578,27 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,28ad28688852dfe5c2527c88fe52d2c7", - "sample1_normal.bam.bai:md5,3f1a1e308eb5f7b6a3805e76ff7034a3", - "sample1_tumor.bam:md5,e25dec2664b709de329909c5f1be74f2", - "sample1_tumor.bam.bai:md5,04d8b757427e0ff00b19aa338c681a54", + "sample1_normal.bam:md5,35824329c737d89e70234e0b8e2de3fa", + "sample1_normal.bam.bai:md5,2eccbe13acc6273ebf6332ec5a5da815", + "sample1_tumor.bam:md5,969daba408c88235a28c6ed38d513965", + "sample1_tumor.bam.bai:md5,4face49fc270117ab49d72d1413a7efb", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,19483ff8a25ef082cde1d2c7d5faac6f", - "sample1_whatshap_stats.log:md5,ede92982bd2c122b30c5625ff1627c46", - "sample1_whatshap_stats.tsv:md5,4c347cec00fefec30623219a865e3eb3", + "sample1_whatshap_stats.gtf:md5,b85799ad0a512d6c30ff05f8b5dca54a", + "sample1_whatshap_stats.log:md5,5889131d24524deb76e5d585e3a2dd3f", + "sample1_whatshap_stats.tsv:md5,648f4ba86c78fc80d388c95193e3c4f4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", - "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", + "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", @@ -610,9 +607,9 @@ "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", - "sample2_whatshap_stats.log:md5,f9628717e0c4944888687c13098c62eb", - "sample2_whatshap_stats.tsv:md5,4fd295d10a8c461fd2c91851027503a4", + "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", + "sample2_whatshap_stats.log:md5,e7336c45aa5738a67f032898accb305e", + "sample2_whatshap_stats.tsv:md5,e478af5b3adb92a590a485f444a0dabe", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -639,6 +636,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T11:10:43.315178155" + "timestamp": "2026-09-18T17:28:32.306705325" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 4c20adca..8ccb3085 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -23,9 +23,6 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, - "DEEPSOMATIC_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -35,9 +32,6 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, - "DEEPVARIANT_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -79,6 +73,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -584,6 +581,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T11:19:11.135286604" + "timestamp": "2026-09-18T17:36:01.066378821" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 47b61a3f..c0b8e75a 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -64,6 +64,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -521,38 +524,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,772e41f7cd86c03a22afbe5ec0592a6b", - "sample1_normal.bam.bai:md5,1b501f6a11efe5d2e6f47b7f1523220b", - "sample1_tumor.bam:md5,c8315c80dc92dfb5d874aef3f5dd46fb", - "sample1_tumor.bam.bai:md5,bc35f807be4b93fc795a14d701469367", + "sample1_normal.bam:md5,9a73c3f90bc4d9a140bcf8f652b0f269", + "sample1_normal.bam.bai:md5,37a866c569f24ed2b38f093f9475b9d6", + "sample1_tumor.bam:md5,dc15bf0e9ff1d401491347c15c408b6d", + "sample1_tumor.bam.bai:md5,fa57db4206692079d9b2084a866bf411", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,eff050a68e36e778b06e0ec19435c569", - "sample1_whatshap_stats.log:md5,76b73731f74fe32ef2d11f6bb0a0f71a", - "sample1_whatshap_stats.tsv:md5,f566ae25b3c5a8f7e94b3d6c1b0417f8", + "sample1_whatshap_stats.gtf:md5,36bda647c08358df0eac0be321c24b20", + "sample1_whatshap_stats.log:md5,938f792fd22bb658a2c7cb1b7653035f", + "sample1_whatshap_stats.tsv:md5,cf8917be389dfef1344eeb6b7e99c7c5", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,3157bd11ba095a884c7951aafcfcfb1c", - "sample2_normal.bam.bai:md5,edebda44c4383173caea728acde4ac43", - "sample2_tumor.bam:md5,47b2c5f86e0493ba94ff72cea77eeae3", - "sample2_tumor.bam.bai:md5,abf2c290c815f54c2b3f8179f717d9bd", + "sample2_normal.bam:md5,c18bbb1bc05b0ec830fa1ac3c6ef542f", + "sample2_normal.bam.bai:md5,de77684ef264476b3e0531b61ca23740", + "sample2_tumor.bam:md5,8b8cb5ac7668b8ac2f4097f584481d35", + "sample2_tumor.bam.bai:md5,3b46456b0e7b00688519dfb25c8e3c88", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,4d8f4393e3aebe4e945c0b8236cf3b3e", - "sample2_whatshap_stats.log:md5,10bba7bae6dd99b989ece5e5dac7a8f9", - "sample2_whatshap_stats.tsv:md5,bb46226e486af9026ab76e014624e903", + "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", + "sample2_whatshap_stats.log:md5,75a69a8e651979e25467d6e3c84cdf90", + "sample2_whatshap_stats.tsv:md5,92d7234e355833b1ec6f54951c38c09d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -575,10 +578,10 @@ "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50" ] ], - "timestamp": "2026-09-09T13:50:22.195327407", "meta": { - "nf-test": "0.9.4", - "nextflow": "26.04.3" - } + "nf-test": "0.9.3", + "nextflow": "26.04.1" + }, + "timestamp": "2026-09-18T17:44:15.090049384" } } \ No newline at end of file diff --git a/tests/union.nf.test b/tests/union.nf.test index 2fe8925b..aa14800b 100644 --- a/tests/union.nf.test +++ b/tests/union.nf.test @@ -115,4 +115,46 @@ nextflow_pipeline { ) } } + + test("-profile test, union combine mode, smallvar_filter_pass=false") { + + when { + params { + outdir = "$outputDir" + germline_var_combine = 'all' + somatic_var_combine = 'all' + germline_var_keep = 'clair, deepvariant' + somatic_var_keep = 'clair, deepsomatic' + smallvar_filter_pass = false + } + } + + then { + assertAll( + { assert workflow.success }, + + // ── No PASS filter process may run ─────────────────────────── + // VCF_PASS_FILTER passes its input straight through when the param is + // false, so PASS_FILTER never executes and never reports a version. + { + def versions = file("$outputDir/pipeline_info/lrsomatic_software_mqc_versions.yml") + assert versions.exists() + assert !versions.text.contains('PASS_FILTER') + }, + + // ── Phased VCFs still exist and have data ──────────────────── + { + ['sample1', 'sample2', 'sample3'].each { s -> + def germline = file("$launchDir/output/${s}/variants/phased/germline_smallvariants.vcf.gz") + def somatic = file("$launchDir/output/${s}/variants/phased/somatic_smallvariants.vcf.gz") + assert germline.exists() + assert somatic.exists() + assert germline.size() > 0 + assert somatic.size() > 0 + } + } + ) + } + } + } diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 2e110695..0663c675 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -44,9 +44,6 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, - "DEEPSOMATIC_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -56,9 +53,6 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, - "DEEPVARIANT_PASS_FILTER": { - "bcftools": "1.23.1" - }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -100,6 +94,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -581,27 +578,27 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,cd28bdb7edefa810b86a12029783e1b6", - "sample1_normal.bam.bai:md5,6c19fcce6bd9050fb03a00e66ba1b866", - "sample1_tumor.bam:md5,dcb89119e272efbe0168d1737f714a8a", - "sample1_tumor.bam.bai:md5,9577c39e6f59056421a8192b0ba43acc", + "sample1_normal.bam:md5,271da2f58d613f1f101a12ef06eca766", + "sample1_normal.bam.bai:md5,c51c824e0ac5451f9c10eee78f40cb98", + "sample1_tumor.bam:md5,f5f68131d379181bbc335c9530e9cc7e", + "sample1_tumor.bam.bai:md5,75e62a6480344075038ce6ad2499bfbb", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,43773d064cd353d1dbbae45adb3be334", - "sample1_whatshap_stats.log:md5,13f4530236189960576b5bc45e63bc1a", - "sample1_whatshap_stats.tsv:md5,bf14b5a1fd9d95797f36a48e7e16953d", + "sample1_whatshap_stats.gtf:md5,2b8ff0a9e055c9b4432bc3da9cf98a4d", + "sample1_whatshap_stats.log:md5,70a5270fec94f8b569a4a7bda49751db", + "sample1_whatshap_stats.tsv:md5,1eab3a16f1a51016c3b74f104de152a6", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,69202a1786f0683c6d319fe55951fed5", - "sample2_normal.bam.bai:md5,8a5169239448f6e8fae3133aa6e1b45f", + "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", + "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", @@ -610,9 +607,9 @@ "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,d719a3c7e642c25f3be96253a0b8df85", - "sample2_whatshap_stats.log:md5,af84d397bf163be8de77fc3c7401493b", - "sample2_whatshap_stats.tsv:md5,fb38bde18fbfc61e52947b68d4260829", + "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", + "sample2_whatshap_stats.log:md5,be94280217c27d05773afa1465152c8a", + "sample2_whatshap_stats.tsv:md5,0c97b6033a149898e88aaf9c632b2de4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -639,6 +636,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T11:35:45.671299133" + "timestamp": "2026-09-18T17:54:05.409593728" } } \ No newline at end of file From 9a1e8e663de353a6b783df9b18c404daf04d0de6 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Mon, 21 Sep 2026 11:56:57 +0200 Subject: [PATCH 03/25] testing --- CHANGELOG.md | 2 + conf/modules.config | 19 ++++- docs/usage.md | 54 ++++++++++---- modules/local/bcftools/view/main.nf | 4 +- modules/local/vcfsplit/main.nf | 28 ++++++-- modules/local/vcftag/environment.yml | 7 ++ modules/local/vcftag/main.nf | 53 ++++++++++++++ modules/local/vcftag/tests/main.nf.test | 68 ++++++++++++++++++ subworkflows/local/phasing_haplotyping.nf | 52 +++++++++----- subworkflows/local/small_variant_consensus.nf | 58 +++++++++------ .../local/tumor_only/tumoronly_smallvar.nf | 4 +- subworkflows/local/vcf_pass_filter.nf | 7 +- tests/fixtures/vcftag_input.vcf | 11 +++ tests/fixtures/vcftag_input.vcf.gz | Bin 0 -> 505 bytes tests/fixtures/vcftag_input.vcf.gz.tbi | Bin 0 -> 112 bytes 15 files changed, 303 insertions(+), 64 deletions(-) create mode 100644 modules/local/vcftag/environment.yml create mode 100644 modules/local/vcftag/main.nf create mode 100644 modules/local/vcftag/tests/main.nf.test create mode 100644 tests/fixtures/vcftag_input.vcf create mode 100644 tests/fixtures/vcftag_input.vcf.gz create mode 100644 tests/fixtures/vcftag_input.vcf.gz.tbi diff --git a/CHANGELOG.md b/CHANGELOG.md index e316511a..7a3a8f3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so every germline record sitting at a somatic coordinate was retained; on B2037620 it removed none of the 13,708,100 germline records, and `variants/phased/somatic_smallvariants.vcf.gz` ended up holding 4,129,270 germline variants that the report could not distinguish from somatic ones. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) now stamps `INFO/SOMATIC` and `INFO/GERMLINE` on the two arms immediately before they are merged for phasing — the only point at which origin is unambiguous for every caller, since `GERMLINE_CONSENSUS` can emit records that never passed through `VCFSPLIT` — and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. Longphase preserves custom INFO keys, verified on v2.0.1. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits; it still normalises the germline split's `FILTER` to `PASS` so that downstream tools which filter on `PASS` continue to see every record, but that value is no longer destroyed. Germline calls remain published in full under `variants/phased/germline_smallvariants.vcf.gz`, `vep/germline/` and `variants//` (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620), and with `'deepsomatic'` it would have dropped every DeepSomatic-private call (26,560 `PASS` records on the same sample). Both private sets are now kept alongside the shared record, matching `nextflow_schema.json` ("keeps all variants from both callers"); `prioritize_caller_*` selects only whose record represents a shared variant. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. Clair3 and ClairS are far less extreme but still kept their `LowQual` and `NonSomatic` records, so with the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`, and signature fitting is unchanged because `SIGNATURES_BCFTOOLS_VIEW` already applied its own `PASS` filter. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP and the report see far fewer variants. Set `--smallvar_filter_pass false` for the previous behaviour. The per-caller VCFs published under `variants/deepvariant`, `variants/deepsomatic`, `variants/clair3` and `variants/clairs` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). - [#186](https://github.com/IntGenomicsLab/lrsomatic/pull/186) - Re-synced the vendored [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) to v1.3.0, which adds tickbox dropdown filters on the categorical columns of both variant tables and turns the report's gene panel selector into checkboxes (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 078a1db1..b41bfdb0 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -401,6 +401,21 @@ process { enabled: false ] } + // VCFTAG stamps an INFO provenance flag on each arm before germline and somatic are merged + // for somatic phasing, so PHASING_HAPLOTYPING:BCFTOOLS_VIEW can select the somatic arm by + // origin instead of by position. Intermediates only; the published VCFs come from Longphase. + withName: '.*:TAG_SOMATIC' { + ext.prefix = { "${meta.id}_somatic_tagged" } + publishDir = [ + enabled: false + ] + } + withName: '.*:TAG_GERMLINE' { + ext.prefix = { "${meta.id}_germline_tagged" } + publishDir = [ + enabled: false + ] + } withName: '.*:PHASING_HAPLOTYPING:BCFTOOLS_VIEW' { ext.prefix = { "somatic_smallvariants" } publishDir = [ @@ -655,7 +670,9 @@ process { // keep their LowQual and NonSomatic records; these filtered copies are what the // consensus, phasing, VEP and report steps consume. The per-caller VCFs under // variants// are published from the caller processes and are left untouched. - // ClairS-TO has no entry here: VCFSPLIT already restricts it to PASS. + // ClairS-TO has no entry here: VCFSPLIT already restricts its SOMATIC split to PASS. + // Note this is not true of the germline split, whose FILTER is rewritten to PASS rather + // than filtered; germline/somatic origin is carried in INFO by VCFTAG instead. // --write-index is required because the module's index output is optional, and the // downstream join() would otherwise drop every sample. withName: '.*:DEEPVARIANT_PASS_FILTER:PASS_FILTER' { diff --git a/docs/usage.md b/docs/usage.md index 488fd69f..3ddb2a4d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -353,15 +353,15 @@ The tools run from a purpose-built image (`ghcr.io/ljwharbers/sigprofiler`) beca These options control how variants from multiple callers are filtered and merged. -| Parameter | Description | -| ------------------------------ | --------------------------------------------------------------------------------------------------- | -| `--germline_var_keep` | Expression or threshold for retaining germline variants after calling. Default = `null` | -| `--somatic_var_keep` | Expression or threshold for retaining somatic variants after calling. Default = `null` | -| `--germline_var_combine` | Strategy for combining germline variant caller outputs (e.g. union, intersection). Default = `null` | -| `--somatic_var_combine` | Strategy for combining somatic variant caller outputs (e.g. union, intersection). Default = `null` | -| `--prioritize_caller_germline` | Comma-separated caller priority order used when combining germline calls. Default = `null` | -| `--prioritize_caller_somatic` | Comma-separated caller priority order used when combining somatic calls. Default = `null` | -| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | +| Parameter | Description | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------- | +| `--germline_var_keep` | Expression or threshold for retaining germline variants after calling. Default = `null` | +| `--somatic_var_keep` | Expression or threshold for retaining somatic variants after calling. Default = `null` | +| `--germline_var_combine` | How to combine germline caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | +| `--somatic_var_combine` | How to combine somatic caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | +| `--prioritize_caller_germline` | Whose record to use for variants called by both germline callers: `deepvariant` or `clair`. Default = `clair` | +| `--prioritize_caller_somatic` | Whose record to use for variants called by both somatic callers: `deepsomatic` or `clair`. Default = `clair` | +| `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds @@ -374,9 +374,39 @@ by three orders of magnitude and produces a meaningless mutation burden. `--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's VCF that is handed to the caller consensus, phasing, VEP and the report. Set it to `false` to restore the previous unfiltered behaviour. In tumor-only mode ClairS-TO -is unaffected by the setting: `VCFSPLIT` already restricts it to `PASS`. The -per-caller VCFs published under `//variants/` are never -filtered, so no calls are lost from the results directory. +is unaffected by the setting: `VCFSPLIT` already restricts its somatic split to +`PASS`. The per-caller VCFs published under `//variants/` +are never filtered, so no calls are lost from the results directory. + +`consensus` keeps only variants called by both callers; `all` keeps the union, i.e. +every variant called by either. In both modes `--prioritize_caller_*` chooses only +whose record represents a variant that both callers found -- it never decides which +variants are kept. + +#### Germline and somatic provenance + +Germline and somatic small variants are merged into one VCF for somatic phasing, +because Longphase needs all variant sites in a single file to produce consistent +phase blocks. The somatic arm is then recovered from the phased result. + +That recovery selects on an `INFO/SOMATIC` flag stamped on each arm before the merge, +not on position. A positional restriction cannot separate the two populations: a +germline record at the same coordinate as a somatic call is indistinguishable from +it, and `FILTER` is no help either, since `VCFSPLIT` normalises the ClairS-TO +germline split to `PASS` so that downstream tools which filter on `PASS` still see +every record. + +Three INFO fields carry this provenance: + +| Field | Meaning | +| ----- | ------- | +| `SOMATIC` | Record came from the somatic call set | +| `GERMLINE` | Record came from the germline call set | +| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS` | + +Germline calls dropped from `variants/phased/somatic_smallvariants.vcf.gz` are not +lost: they remain in `variants/phased/germline_smallvariants.vcf.gz`, in +`vep/germline/`, and in the unfiltered per-caller VCFs under `variants//`. #### PON Options diff --git a/modules/local/bcftools/view/main.nf b/modules/local/bcftools/view/main.nf index 652da9ac..ae81084e 100644 --- a/modules/local/bcftools/view/main.nf +++ b/modules/local/bcftools/view/main.nf @@ -8,7 +8,7 @@ process BCFTOOLS_VIEW { : 'community.wave.seqera.io/library/bcftools_htslib:0a3fa2654b52006f'}" input: - tuple val(meta), path(vcf), path(tbi), path(targets), path(targets_tbi) + tuple val(meta), path(vcf), path(tbi) output: tuple val(meta), path("*.vcf.gz"), emit: vcf @@ -23,7 +23,7 @@ process BCFTOOLS_VIEW { def prefix = task.ext.prefix ?: "${meta.id}" """ bcftools view \\ - -T ${targets} \\ + -i 'INFO/SOMATIC=1' \\ -Oz \\ -W=tbi \\ ${args} \\ diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index f6156d34..b075dcca 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -28,7 +28,16 @@ process VCFSPLIT { bcftools view -i 'FILTER="PASS"' $snv_vcf | bgzip -c > snv_pass.vcf.gz tabix -p vcf indels_pass.vcf.gz tabix -p vcf snv_pass.vcf.gz - bcftools concat -a -Oz -o somatic.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz + bcftools concat -a -Oz -o somatic_tmp.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz + tabix -p vcf somatic_tmp.vcf.gz + + # Record the caller's original FILTER in INFO. These records are already PASS, but stamping + # both splits keeps them symmetric and self-describing alongside the germline arm below. + bcftools view somatic_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + /^##/ { print; next } + /^#CHROM/ { print "##INFO="; print; next } + { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; print } + ' | bgzip -c > somatic.vcf.gz tabix -p vcf somatic.vcf.gz bcftools view -i 'FILTER~"NonSomatic" || INFO/Verdict_Germline=1' $indel_vcf | bgzip -c > indels_filtered.vcf.gz @@ -38,13 +47,22 @@ process VCFSPLIT { bcftools concat -a -Oz -o germline_tmp.vcf.gz indels_filtered.vcf.gz snv_filtered.vcf.gz tabix -p vcf germline_tmp.vcf.gz - bcftools view germline_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} /^#/ {print} !/^#/ { \$7="PASS"; print }' | \ - bgzip -c > germline.vcf.gz + # FILTER is normalised to PASS so that downstream tools which filter on PASS -- implicitly or + # otherwise -- see every germline record. The caller's original FILTER (typically NonSomatic) + # would otherwise be destroyed here, which is what previously made germline records + # indistinguishable from somatic ones once the two sets were merged for phasing; it is kept in + # INFO/ORIG_FILTER instead. Germline/somatic provenance itself is stamped later, by + # PHASING_HAPLOTYPING:TAG_GERMLINE / TAG_SOMATIC, which covers callers that bypass VCFSPLIT. + bcftools view germline_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + /^##/ { print; next } + /^#CHROM/ { print "##INFO="; print; next } + { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; \$7 = "PASS"; print } + ' | bgzip -c > germline.vcf.gz tabix -p vcf germline.vcf.gz # Cleanup intermediate files - rm indels_pass.vcf.gz snv_pass.vcf.gz - rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi + rm indels_pass.vcf.gz snv_pass.vcf.gz somatic_tmp.vcf.gz + rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi somatic_tmp.vcf.gz.tbi """ stub: diff --git a/modules/local/vcftag/environment.yml b/modules/local/vcftag/environment.yml new file mode 100644 index 00000000..b276efd9 --- /dev/null +++ b/modules/local/vcftag/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - bioconda::bcftools=1.20 diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf new file mode 100644 index 00000000..851dff6b --- /dev/null +++ b/modules/local/vcftag/main.nf @@ -0,0 +1,53 @@ +process VCFTAG { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bcftools:1.20--h8b25389_0': + 'biocontainers/bcftools:1.20--h8b25389_0' }" + + input: + tuple val(meta), path(vcf), path(tbi) + val flag + + output: + tuple val(meta), path("${prefix}.vcf.gz") , emit: vcf + tuple val(meta), path("${prefix}.vcf.gz.tbi") , emit: tbi + + tuple val("${task.process}"), val('bcftools'), eval("bcftools --version |& sed '1!d ; s/bcftools //'"), topic: versions, emit: versions_bcftools + + when: + task.ext.when == null || task.ext.when + + script: + prefix = task.ext.prefix ?: "${meta.id}_${flag.toLowerCase()}" + """ + # Stamp a constant INFO flag recording which arm this record came from, and normalise FILTER + # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and + # filter on FILTER, so a FILTER-based label does not survive to where it is needed. + # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. + bcftools view ${vcf} | awk -v flag="${flag}" 'BEGIN{FS=OFS="\\t"} + /^##/ { print; next } + /^#CHROM/ { + print "##INFO=" + print + next + } + { + \$7 = "PASS" + \$8 = (\$8 == "." || \$8 == "") ? flag : \$8 ";" flag + print + } + ' | bgzip -c > ${prefix}.vcf.gz + + tabix -p vcf ${prefix}.vcf.gz + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}_${flag.toLowerCase()}" + """ + echo "" | gzip > ${prefix}.vcf.gz + touch ${prefix}.vcf.gz.tbi + """ +} diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test new file mode 100644 index 00000000..18b79c41 --- /dev/null +++ b/modules/local/vcftag/tests/main.nf.test @@ -0,0 +1,68 @@ +nextflow_process { + + name "Test Process VCFTAG" + script "../main.nf" + process "VCFTAG" + + tag "modules" + tag "modules_local" + tag "vcftag" + + // Runs for real (no -stub): the tagging is an awk program embedded in the Nextflow script + // block, so the escaping only holds if it is actually executed. + test("stamps the flag, declares its header and normalises FILTER") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file("\${projectDir}/tests/fixtures/vcftag_input.vcf.gz", checkIfExists: true), + file("\${projectDir}/tests/fixtures/vcftag_input.vcf.gz.tbi", checkIfExists: true) + ] + input[1] = 'SOMATIC' + """ + } + } + + then { + assert process.success + + def lines = path(process.out.vcf[0][1]).vcf.variantsAsStrings + def header = path(process.out.vcf[0][1]).vcf.header.getMetaDataInSortedOrder().join('\n') + + assertAll( + // every input record survives -- tagging must not filter + { assert lines.size() == 4 }, + // the flag is declared, so bcftools can query it downstream + { assert header.contains('ID=SOMATIC') }, + // every record carries the flag, including the one whose INFO was '.' + { assert lines.every { it.contains('SOMATIC') } }, + // FILTER is normalised to PASS on all records, including NonSomatic and RefCall + { assert lines.every { it.split('\t')[6] == 'PASS' } }, + // a pre-existing INFO field is preserved rather than overwritten + { assert lines.any { it.contains('EXISTING') && it.contains('CALLER=clairs-to') } }, + // the record whose INFO was '.' gets the bare flag, not '.;SOMATIC' + { assert lines.any { it.split('\t')[7] == 'SOMATIC' } } + ) + } + } + + test("stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [ id:'test' ], [], [] ] + input[1] = 'GERMLINE' + """ + } + } + + then { + assert process.success + } + } +} diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 84bde7d0..f68deb57 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -8,6 +8,8 @@ include { SAMTOOLS_INDEX } from '../../module include { BCFTOOLS_CONCAT } from '../../modules/nf-core/bcftools/concat/main' include { BCFTOOLS_SORT } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_VIEW } from '../../modules/local/bcftools/view/main.nf' +include { VCFTAG as TAG_SOMATIC } from '../../modules/local/vcftag/main.nf' +include { VCFTAG as TAG_GERMLINE } from '../../modules/local/vcftag/main.nf' workflow PHASING_HAPLOTYPING { @@ -139,10 +141,30 @@ workflow PHASING_HAPLOTYPING { } + // + // MODULE: VCFTAG (label: process_single), aliased TAG_SOMATIC / TAG_GERMLINE + // Stamp each arm with an INFO provenance flag before they are merged. This is the only + // point where germline-vs-somatic origin is unambiguous for every caller: GERMLINE_CONSENSUS + // can emit records that never passed through VCFSPLIT, so tagging further upstream would + // leave holes. After the merge the two populations are otherwise indistinguishable -- both + // carry FILTER=PASS, and VCFSPLIT rewrites the ClairS-TO germline FILTER to PASS as well. + // LongPhase preserves custom INFO keys, so the flags survive phasing (verified on v2.0.1). + // + TAG_SOMATIC ( somatic_vcf, 'SOMATIC' ) + TAG_GERMLINE( germline_vcf, 'GERMLINE' ) + + TAG_SOMATIC.out.vcf + .join(TAG_SOMATIC.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .set{ tagged_somatic_vcf } + TAG_GERMLINE.out.vcf + .join(TAG_GERMLINE.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .set{ tagged_germline_vcf } + // tagged_*_vcf: [meta, vcf, tbi] + // Merge germline and somatic VCFs into a single file for somatic phasing // Longphase requires all variant sites in one VCF to produce a consistent phase block - germline_vcf - .join(somatic_vcf) + tagged_germline_vcf + .join(tagged_somatic_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> def vcfs = [som_vcf, germ_vcf] // somatic first (higher priority in phasing) def tbis = [som_tbi, germ_tbi] @@ -178,7 +200,7 @@ workflow PHASING_HAPLOTYPING { if (!params.skip_modcall) { // With modcall: include base-modification VCF as additional phasing evidence normal_bams_w_tumoronly_ch - .join(germline_vcf) + .join(tagged_germline_vcf) .join(LONGPHASE_MODCALL_GERMLINE.out.mod_vcf) .map { meta, bam, bai, vcf, _tbi, mods-> def svs = [] // SVs for phasing are not used here @@ -200,7 +222,7 @@ workflow PHASING_HAPLOTYPING { else { // Without modcall: empty lists for SVs and mods normal_bams_w_tumoronly_ch - .join(germline_vcf) + .join(tagged_germline_vcf) .map { meta, bam, bai, vcf, _tbi -> def svs = [] def mods = [] @@ -258,23 +280,17 @@ workflow PHASING_HAPLOTYPING { // // MODULE: BCFTOOLS_VIEW (label: process_medium) - // Filter the phased somatic+germline VCF to somatic-only positions. - // Uses the original somatic VCF as a targets (-T) file so only positions - // called as somatic are retained. Phase tags (PS/HP) on somatic variants - // are preserved; germline records are dropped. - // Input: [meta, phased_combined_vcf, phased_combined_tbi, somatic_vcf, somatic_tbi] + // Reduce the phased somatic+germline VCF to the somatic arm, selecting on the INFO/SOMATIC + // flag stamped before the merge. This selects by provenance rather than by position: the + // previous `-T ` targets file matched CHROM/POS only, so every germline record + // co-located with a somatic call was retained and became indistinguishable from a somatic + // one downstream. Phase tags (PS/HP) on somatic variants are preserved; germline records + // are dropped here but remain published in full under variants/phased/ and vep/germline/. + // Input: [meta, phased_combined_vcf, phased_combined_tbi] // Output: .vcf -- [meta, vcf.gz] -- phased somatic-only VCF // .tbi -- [meta, tbi] // - phased_somatic_germline_vcf - .join(somatic_vcf) - .map { meta, phased_vcf, phased_tbi, som_vcf, som_tbi -> - return [ meta, phased_vcf, phased_tbi, som_vcf, som_tbi ] - } - .set { bcftools_view_input_ch } - // bcftools_view_input_ch: [meta, phased_combined_vcf, tbi, somatic_vcf, somatic_tbi] - - BCFTOOLS_VIEW ( bcftools_view_input_ch ) + BCFTOOLS_VIEW ( phased_somatic_germline_vcf ) BCFTOOLS_VIEW.out.vcf .join(BCFTOOLS_VIEW.out.tbi) diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index d3499cf8..d6e7e5b4 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -17,7 +17,7 @@ workflow SMALL_VARIANT_CONSENSUS { fasta // [[:], fasta] _fai // [[:], fai] prioritize_caller // str: which caller's calls take priority ('deepvariant'/'deepsomatic' or 'clair') - combine_method // str: 'consensus' (intersection only) or 'all' (intersection + private calls from priority caller) + combine_method // str: 'consensus' (shared calls only) or 'all' (union of both callers' calls) main: @@ -205,6 +205,9 @@ workflow SMALL_VARIANT_CONSENSUS { BCFTOOLS_ISEC.out.clair_consensus_vcf .set{isec_consensus_vcf} } + else { + error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") + } // BCFTOOLS_ISEC outputs hardcoded names (0002.vcf.gz) inside a prefix directory. // Nextflow stages files using basename only, so both germline and somatic consensus // VCFs would collide as "0002.vcf.gz" in downstream PHASING_HAPLOTYPING:BCFTOOLS_CONCAT. @@ -216,45 +219,56 @@ workflow SMALL_VARIANT_CONSENSUS { } else if (combine_method == 'all') { - // Take the intersection PLUS the private calls from the prioritized caller - // (private calls from the non-priority caller are discarded) + // Union: every variant called by either caller. Variants called by both contribute a + // single record, taken from the prioritized caller; the private calls of BOTH callers + // are kept. This matches `*_var_combine = 'all'` in nextflow_schema.json ("keeps all + // variants from both callers"); `prioritize_caller` only selects whose record is used + // for the shared variants, never which calls are kept. + // The three isec sets are disjoint by construction, so BCFTOOLS_CONCAT needs no -d. if (prioritize_caller in ['deepvariant', 'deepsomatic']) { - // consensus (DeepVariant record) + DeepVariant-private variants + // shared (DeepVariant record) + DeepVariant-private + Clair-private BCFTOOLS_ISEC.out.deepvar_consensus_vcf .join(BCFTOOLS_ISEC.out.deepvar_consensus_tbi) + .join(BCFTOOLS_ISEC.out.deepvar_private_vcf) + .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) .join(BCFTOOLS_ISEC.out.clair_private_vcf) .join(BCFTOOLS_ISEC.out.clair_private_tbi) - .map{ meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> - return[meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi]] + .map{ meta, shared_vcf, shared_tbi, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> + return[meta, [shared_vcf, deepvar_vcf, clair_vcf], [shared_tbi, deepvar_tbi, clair_tbi]] } .set{concat_input} - // concat_input: [meta, [consensus_vcf, private_vcf], [consensus_tbi, private_tbi]] - BCFTOOLS_CONCAT(concat_input) - BCFTOOLS_CONCAT.out.vcf - .set{concat_out} } else if (prioritize_caller == 'clair') { - // consensus (Clair record) + Clair-private variants - BCFTOOLS_ISEC.out.deepvar_private_vcf - .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) - .join(BCFTOOLS_ISEC.out.clair_consensus_vcf) + // shared (Clair record) + DeepVariant-private + Clair-private + BCFTOOLS_ISEC.out.clair_consensus_vcf .join(BCFTOOLS_ISEC.out.clair_consensus_tbi) - .map{ meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> - return[meta, [deepvar_vcf, clair_vcf], [deepvar_tbi, clair_tbi]] + .join(BCFTOOLS_ISEC.out.deepvar_private_vcf) + .join(BCFTOOLS_ISEC.out.deepvar_private_tbi) + .join(BCFTOOLS_ISEC.out.clair_private_vcf) + .join(BCFTOOLS_ISEC.out.clair_private_tbi) + .map{ meta, shared_vcf, shared_tbi, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> + return[meta, [shared_vcf, deepvar_vcf, clair_vcf], [shared_tbi, deepvar_tbi, clair_tbi]] } .set{concat_input} - // concat_input: [meta, [private_vcf, consensus_vcf], [private_tbi, consensus_tbi]] - BCFTOOLS_CONCAT(concat_input) - BCFTOOLS_CONCAT.out.vcf - .set{concat_out} } - // concat_out: [meta, vcf] -- unsorted concatenated VCF (consensus + priority-caller-private) + else { + error("prioritize_caller must be one of [deepvariant, deepsomatic, clair], got '${prioritize_caller}'") + } + // concat_input: [meta, [shared_vcf, deepvar_private_vcf, clair_private_vcf], [tbis...]] + BCFTOOLS_CONCAT(concat_input) + BCFTOOLS_CONCAT.out.vcf + .set{concat_out} + // concat_out: [meta, vcf] -- unsorted union of both callers' calls BCFTOOLS_SORT(concat_out) BCFTOOLS_SORT.out.vcf .set{vcf} BCFTOOLS_SORT.out.tbi .set{tbi} - // vcf/tbi: [meta, vcf/tbi] -- sorted combined VCF + // vcf/tbi: [meta, vcf/tbi] -- sorted union VCF + } + + else { + error("combine_method must be 'consensus' or 'all', got '${combine_method}'") } emit: diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 299bc0c7..050375ed 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -128,7 +128,9 @@ workflow TUMORONLY_SMALLVAR { // DeepVariant emits a record for every site it evaluates, not just its calls, so // most records are RefCall. ClairS-TO needs no equivalent step here because - // VCFSPLIT already restricts it to PASS. The VCF published under + // VCFSPLIT already restricts its SOMATIC split to PASS -- note that its GERMLINE split + // is not PASS-filtered but PASS-rewritten, so a PASS filter would not reduce it and + // germline/somatic origin is carried in INFO by VCFTAG instead. The VCF published under // variants/deepvariant/ is unaffected. DEEPVARIANT_PASS_FILTER ( DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) diff --git a/subworkflows/local/vcf_pass_filter.nf b/subworkflows/local/vcf_pass_filter.nf index 51f0833a..f52e0cb9 100644 --- a/subworkflows/local/vcf_pass_filter.nf +++ b/subworkflows/local/vcf_pass_filter.nf @@ -22,12 +22,13 @@ workflow VCF_PASS_FILTER { main: if (params.smallvar_filter_pass) { - // --write-index is set in conf/modules.config because the module's index output is - // optional, and the join() below would otherwise drop every sample. + // The module declares `emit: index, optional: true`, so the index only exists because + // conf/modules.config puts --write-index=tbi in ext.args for every alias. failOnMismatch + // turns a missing index into an immediate error instead of silently dropping the sample. PASS_FILTER ( vcfs, [], [], [] ) PASS_FILTER.out.vcf - .join(PASS_FILTER.out.index) + .join(PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) .set{ filtered } } else { diff --git a/tests/fixtures/vcftag_input.vcf b/tests/fixtures/vcftag_input.vcf new file mode 100644 index 00000000..34fda4cc --- /dev/null +++ b/tests/fixtures/vcftag_input.vcf @@ -0,0 +1,11 @@ +##fileformat=VCFv4.2 +##INFO= +##INFO= +##FILTER= +##FILTER= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT testsample +chr1 100 . A C 30 PASS CALLER=clairs-to GT:DP 0/1:20 +chr1 200 . G T 20 NonSomatic CALLER=clairs-to;EXISTING GT:DP 0/1:18 +chr1 300 . T A 10 RefCall . GT:DP 0/0:12 +chr1 400 . C G 40 . CALLER=deepsomatic GT:DP 1/1:25 diff --git a/tests/fixtures/vcftag_input.vcf.gz b/tests/fixtures/vcftag_input.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..988819c22c88bc153d91397e8db80263c9cbe483 GIT binary patch literal 505 zcmb2|=3rp}f&Xj_PR>jWCXBHc0<)V91lT`3{}SBtE9{_W^?fZ6S z+fkQE(UJTAUdY?k-QWv4HmA{{%kZ*k5#{^G^y?=~%1eqZH8YXkG$`I+ZUHL{{s zwcAhK9qOH-yzYwWPR861clbT6iWyu#YJ6QT-L}|IDd6RbRxzCyY|j?ol;{oWNh{vM z9p5Nck#g|U$Cv*SC0SN?itn9rblM&<{|5TzIE<&hk1xcXQ_i2S;UPlT%R{p`otT--6BcUf0b|oU_Md!IY&7=E{q;aZGA# zR=D!{^11#z>A&-lz`W};9Sw1slNjynB(jR?) h|F!eEB^Ifge7nt;pScx%4H&2B?v!R=2FEps008J@+w1@U literal 0 HcmV?d00001 diff --git a/tests/fixtures/vcftag_input.vcf.gz.tbi b/tests/fixtures/vcftag_input.vcf.gz.tbi new file mode 100644 index 0000000000000000000000000000000000000000..3e8c3efd859874550e714bc5ac9aa7e101d7798f GIT binary patch literal 112 zcmb2|=3rp}f&Xj_PR>jW!3^AmpHfm%5)u-U5)v9N@&Li9fLT);8x<`VpAgV_sL2v| t Date: Mon, 21 Sep 2026 13:14:12 +0200 Subject: [PATCH 04/25] fix bugs --- CHANGELOG.md | 2 +- docs/usage.md | 10 +++++----- modules/local/vcfsplit/main.nf | 26 ++++++++++++++++++++------ modules/local/vcftag/main.nf | 4 ++-- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3a8f3e..850dab8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so every germline record sitting at a somatic coordinate was retained; on B2037620 it removed none of the 13,708,100 germline records, and `variants/phased/somatic_smallvariants.vcf.gz` ended up holding 4,129,270 germline variants that the report could not distinguish from somatic ones. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) now stamps `INFO/SOMATIC` and `INFO/GERMLINE` on the two arms immediately before they are merged for phasing — the only point at which origin is unambiguous for every caller, since `GERMLINE_CONSENSUS` can emit records that never passed through `VCFSPLIT` — and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. Longphase preserves custom INFO keys, verified on v2.0.1. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits; it still normalises the germline split's `FILTER` to `PASS` so that downstream tools which filter on `PASS` continue to see every record, but that value is no longer destroyed. Germline calls remain published in full under `variants/phased/germline_smallvariants.vcf.gz`, `vep/germline/` and `variants//` (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so every germline record sitting at a somatic coordinate was retained; on B2037620 it removed none of the 13,708,100 germline records, and `variants/phased/somatic_smallvariants.vcf.gz` ended up holding 4,129,270 germline variants that the report could not distinguish from somatic ones. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) now stamps `INFO/SOMATIC` and `INFO/GERMLINE` on the two arms immediately before they are merged for phasing — the only point at which origin is unambiguous for every caller, since `GERMLINE_CONSENSUS` can emit records that never passed through `VCFSPLIT` — and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. Longphase preserves custom INFO keys, verified on v2.0.1. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits (multi-valued `FILTER` is joined with `,`, since `;` separates INFO fields); it still normalises the germline split's `FILTER` to `PASS` so that downstream tools which filter on `PASS` continue to see every record, but that value is no longer destroyed. Germline calls remain published in full under `variants/phased/germline_smallvariants.vcf.gz`, `vep/germline/` and `variants//` (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620), and with `'deepsomatic'` it would have dropped every DeepSomatic-private call (26,560 `PASS` records on the same sample). Both private sets are now kept alongside the shared record, matching `nextflow_schema.json` ("keeps all variants from both callers"); `prioritize_caller_*` selects only whose record represents a shared variant. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate rather than only the variants they call: on a 30x PacBio tumour sample `_somatic.vcf.gz` held 13,684,025 records of which 49,957 were `PASS` (9,349,614 `RefCall`, 4,011,128 `GERMLINE`, 273,326 `PON`), and `_germline.vcf.gz` held 13,684,023 records with 4,991,797 `PASS`. Clair3 and ClairS are far less extreme but still kept their `LowQual` and `NonSomatic` records, so with the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at": the published `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, giving a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all — its circos and small-variant chunks each exceeded V8's 512 MB string limit, so Quarto failed with `failed to allocate string; buffer exceeds maximum length`. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`, and signature fitting is unchanged because `SIGNATURES_BCFTOOLS_VIEW` already applied its own `PASS` filter. Reruns therefore give different `somatic_smallvariants.vcf.gz` and `germline_smallvariants.vcf.gz` content, and VEP and the report see far fewer variants. Set `--smallvar_filter_pass false` for the previous behaviour. The per-caller VCFs published under `variants/deepvariant`, `variants/deepsomatic`, `variants/clair3` and `variants/clairs` are unchanged and remain unfiltered (@robert-a-forsyth). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs `modkit pileup` with `--cpg --modified-bases 5mC` by default; previously it ran with no arguments, and that unfiltered pileup (every modification code at every sequence context) produced 30-45 GB per sample. Reruns therefore give different bedMethyl content: only 5mC rows at CpG sites, so 5hmC and 6mA calls and non-CpG positions are no longer reported. Set `--modkit_args=''` (the `=` form; `--modkit_args ''` reaches the pipeline as `true` and is rejected by parameter validation on Nextflow 25) or an empty `modkit_args` entry in a params file to get the previous unfiltered output (@ljwharbers). diff --git a/docs/usage.md b/docs/usage.md index 3ddb2a4d..87e7cf53 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -398,11 +398,11 @@ every record. Three INFO fields carry this provenance: -| Field | Meaning | -| ----- | ------- | -| `SOMATIC` | Record came from the somatic call set | -| `GERMLINE` | Record came from the germline call set | -| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS` | +| Field | Meaning | +| ------------- | ---------------------------------------------------------------------------------------------------------------- | +| `SOMATIC` | Record came from the somatic call set | +| `GERMLINE` | Record came from the germline call set | +| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS`. Multiple filters are joined with `,` | Germline calls dropped from `variants/phased/somatic_smallvariants.vcf.gz` are not lost: they remain in `variants/phased/germline_smallvariants.vcf.gz`, in diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index b075dcca..d773494a 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -33,10 +33,15 @@ process VCFSPLIT { # Record the caller's original FILTER in INFO. These records are already PASS, but stamping # both splits keeps them symmetric and self-describing alongside the germline arm below. - bcftools view somatic_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + # The header line's double quotes arrive via -v q. Escaped quotes inside a Nextflow script + # block are fragile, and losing the escape silently produces an unparseable VCF header. + # FILTER is ";"-delimited but ";" separates INFO fields, so it is stored as ",". + bcftools view somatic_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } - { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; print } + /^#CHROM/ { print "##INFO="; print; next } + { of = \$7; gsub(/;/, ",", of) + \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of + print } ' | bgzip -c > somatic.vcf.gz tabix -p vcf somatic.vcf.gz @@ -53,13 +58,22 @@ process VCFSPLIT { # indistinguishable from somatic ones once the two sets were merged for phasing; it is kept in # INFO/ORIG_FILTER instead. Germline/somatic provenance itself is stamped later, by # PHASING_HAPLOTYPING:TAG_GERMLINE / TAG_SOMATIC, which covers callers that bypass VCFSPLIT. - bcftools view germline_tmp.vcf.gz | awk 'BEGIN{FS=OFS="\t"} + bcftools view germline_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } - { \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" \$7 : \$8 ";ORIG_FILTER=" \$7; \$7 = "PASS"; print } + /^#CHROM/ { print "##INFO="; print; next } + { of = \$7; gsub(/;/, ",", of) + \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of + \$7 = "PASS" + print } ' | bgzip -c > germline.vcf.gz tabix -p vcf germline.vcf.gz + # Read both headers back. tabix will happily index a VCF whose header htslib cannot parse, so + # without this a malformed header surfaces as a confusing failure in a later process instead + # of here. set -e is in effect, so a bad header fails this task. + bcftools view -h somatic.vcf.gz > /dev/null + bcftools view -h germline.vcf.gz > /dev/null + # Cleanup intermediate files rm indels_pass.vcf.gz snv_pass.vcf.gz somatic_tmp.vcf.gz rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi somatic_tmp.vcf.gz.tbi diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf index 851dff6b..3fd9dc81 100644 --- a/modules/local/vcftag/main.nf +++ b/modules/local/vcftag/main.nf @@ -27,10 +27,10 @@ process VCFTAG { # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and # filter on FILTER, so a FILTER-based label does not survive to where it is needed. # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. - bcftools view ${vcf} | awk -v flag="${flag}" 'BEGIN{FS=OFS="\\t"} + bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } /^#CHROM/ { - print "##INFO=" + print "##INFO=" print next } From 5b38dcff17ab195ce593cab80a3e4b958c08ee1e Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 10:30:03 +0200 Subject: [PATCH 05/25] bump wakhan (bug) --- CHANGELOG.md | 1 + modules/local/wakhan/environment.yml | 2 +- modules/local/wakhan/main.nf | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 850dab8c..48e9a388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3, `src/coverage/processing.py:121` averaged a BAF bin with `haplotype_1_coverage[haplotype_1_position.index(min(l2)):haplotype_1_position.index(max(l2))]`; because `.index()` searches the full position list by value, the slice is empty whenever that list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError: mean requires at least one data point` (it also always dropped the bin's last SNP). Upstream replaced the block in 0.4.4 with a guarded loop that emits `CENTROMERE_SENTINEL` for bins holding fewer than `MIN_SNPS_PER_BAF_BIN` het SNPs. The failure was deterministic, so the retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). - [#193](https://github.com/IntGenomicsLab/lrsomatic/pull/193) - Fixed the documented way of enabling EVE. `--vep_eve https://evemodel.org/api/proteins/bulk/download/` was rejected at launch, because the check for "still needs reshaping" keyed on a `.zip` suffix and that endpoint carries no file extension, so the URL was taken for a finished file and an index demanded for it. The check now asks whether the value already is a prepared (bgzipped) file instead (@AmberVerhasselt). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs a patched modkit 0.6.4 built from [ljwharbers/modkit@pacbio-conflict-fix](https://github.com/ljwharbers/modkit/tree/pacbio-conflict-fix): `ghcr.io/ljwharbers/modkit:0.6.4-pacbiofix-6e0afa2` under Docker, and the native SIF `oras://ghcr.io/ljwharbers/modkit-sif:0.6.4-pacbiofix-6e0afa2` under Singularity/Apptainer (unless `singularity_pull_docker_container` is set, which falls back to the Docker image). It keeps reads whose 5mC and 5hmC probabilities sum above 1, fixes pileup on PacBio-style MM tags, and honours `--phased` and `--modified-bases` in the general pileup workers that modkit uses for PacBio BAMs with 6mA calls (stock modkit wrote empty `_hp1`/`_hp2` files and an `h` row next to every `m` row for those). Stock modkit 0.4.3-0.6.4 silently dropped 32-65 % of reads from recent PacBio HiFi BAMs (Jasmine >= 26.1.3) and returned empty `--cpg` pileups ([nanoporetech/modkit#612](https://github.com/nanoporetech/modkit/issues/612); fixes proposed upstream in [nanoporetech/modkit#720](https://github.com/nanoporetech/modkit/pull/720)). The image is `linux/amd64` only and Conda is not supported for this step: `MODKIT_PILEUP` stops with an error under `-profile conda`/`mamba`, use `--skip_modkit` there. The module should return to the nf-core biocontainer once a modkit release includes the fix (@ljwharbers). - [#181](https://github.com/IntGenomicsLab/lrsomatic/pull/181) - NanoPlot pre- and post-alignment statistics now reach MultiQC. `ch_nanoplot_pre_txt` and `ch_nanoplot_post_txt` were declared empty and mixed into the MultiQC inputs, but never assigned from `NANOPLOT_PRE.out.txt` / `NANOPLOT_POST.out.txt`, so the NanoStat section was silently missing from every report (@ljwharbers). diff --git a/modules/local/wakhan/environment.yml b/modules/local/wakhan/environment.yml index 6b0cdb43..33c3c873 100644 --- a/modules/local/wakhan/environment.yml +++ b/modules/local/wakhan/environment.yml @@ -4,4 +4,4 @@ channels: - conda-forge - bioconda dependencies: - - "bioconda::wakhan=0.4.3" + - "bioconda::wakhan=0.4.4" diff --git a/modules/local/wakhan/main.nf b/modules/local/wakhan/main.nf index ad7aba5e..b461baaf 100644 --- a/modules/local/wakhan/main.nf +++ b/modules/local/wakhan/main.nf @@ -4,8 +4,8 @@ process WAKHAN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/wakhan:0.4.3--pyhdfd78af_0': - 'biocontainers/wakhan:0.4.3--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/wakhan:0.4.4--pyhdfd78af_0': + 'biocontainers/wakhan:0.4.4--pyhdfd78af_0' }" input: tuple val(meta), path(tumor_input), path(tumor_index), path(normal_input), path(normal_index), path(vcf), path(breakpoints) @@ -40,7 +40,7 @@ process WAKHAN { // and LRSOMATICREPORT resolves them by solution_/ path tuple val(meta), path("solution_*", type: 'dir') , emit: solution_dirs, optional: true // WARN: Manually update version information as tool does not provide on CLI - tuple val("${task.process}"), val('wakhan'), val("0.4.3"), topic: versions, emit: versions_wakhan + tuple val("${task.process}"), val('wakhan'), val("0.4.4"), topic: versions, emit: versions_wakhan when: task.ext.when == null || task.ext.when From da99d0bc8b9aa4d135afd155f4387f50b2d7a278 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:38:56 +0200 Subject: [PATCH 06/25] Split multi-allelics before intersection, harden the consensus channel joins BCFTOOLS_NORM ran with -Oz only, so multi-allelic records were never split. BCFTOOLS_ISEC matches on exact CHROM/POS/REF/ALT, so a site one caller reports as A>G,GT and the other as A>G never intersected: in consensus mode the variant was dropped, in union mode it appeared twice. ClairS-TO emits no multi-allelic records while DeepVariant emits ~2.5% (128,115 of 5,058,527 on B1975944), so every DeepVariant multi-allelic site was systematically excluded from the consensus. Adding -m -any splits them first; verified on real DeepSomatic output (45,638 -> 45,677 records over chr1:1-5Mb, 39 split, none left multi-allelic). This also makes the AF declaration mismatch harmless: ClairS-TO declares FORMAT/AF as Number=1 while DeepVariant and DeepSomatic declare Number=A, and bcftools concat only warns before keeping the first file's definition. Once every record is biallelic the two declarations are equivalent. The comment on STANDARDIZE_AF is corrected accordingly: it claimed DeepVariant emits VAF, but all four callers emit FORMAT/AF and none emits VAF, so the rename is a no-op under the default prioritize_caller='clair' and is kept as a guarantee for WAKHAN rather than as a live conversion. Two silent-failure paths in SMALL_VARIANT_CONSENSUS are closed. The caller branch had no `other` arm, so an unrecognised meta.caller removed the sample from every downstream result with a successful exit; it now errors. The DeepVariant/Clair join was a plain join, so a sample present for one caller but not the other was dropped just as quietly; it now uses failOnMismatch and failOnDuplicate, matching PHASING_HAPLOTYPING. Record counts in the merged output change, so the snapshots need regenerating. Co-Authored-By: Claude Opus 5 (1M context) --- conf/modules.config | 7 +++++- subworkflows/local/small_variant_consensus.nf | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index ad58f252..99872d66 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -121,8 +121,13 @@ process { withName: '.*:BCFTOOLS_NORM' { ext.prefix = { "${meta.id}.${meta.caller}_norm" } + // -m -any splits multi-allelic records into biallelic ones. BCFTOOLS_ISEC matches on exact + // CHROM/POS/REF/ALT, so without this a site one caller reports as A>G,GT and the other as + // A>G never intersects: in consensus mode the variant is dropped, in union mode it appears + // twice. ClairS-TO emits no multi-allelic records at all while DeepVariant emits ~2.5%, so + // every DeepVariant multi-allelic site was systematically excluded from the consensus. ext.args = { - "-Oz" + "-m -any -Oz" } publishDir = [ enabled: false diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index f45e959b..43e5114a 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -45,6 +45,16 @@ workflow SMALL_VARIANT_CONSENSUS { // MODULE: STANDARDIZE_AF (BCFTOOLS_ANNOTATE alias, label: process_low) -- rename the AF FORMAT field to the priority caller's: // FORMAT/AF -> FORMAT/VAF when prioritize_caller is 'deepvariant'/'deepsomatic' // FORMAT/VAF -> FORMAT/AF when prioritize_caller is 'clair' + // This guarantees the merged VCF exposes allele frequency under a single FORMAT key, which is + // what WAKHAN consumes. Every caller currently emits FORMAT/AF and none emits VAF (verified + // against Clair3, ClairS-TO, DeepVariant and DeepSomatic output), so under the default + // prioritize_caller='clair' it is a no-op; it is kept as the guarantee, not the mechanism. + // + // The callers do disagree on the AF *declaration*: ClairS-TO says Number=1, DeepVariant and + // DeepSomatic say Number=A. bcftools concat only warns and keeps the first file's definition. + // Renaming cannot fix that and `annotate -h` cannot override an existing FORMAT definition, + // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record + // reaching here carries one ALT and one AF value, making the two declarations equivalent. // if (combine_method == 'all') { normalized_vcfs @@ -105,12 +115,20 @@ workflow SMALL_VARIANT_CONSENSUS { // annotated_vcfs: [meta(+caller), vcf, tbi] -- VCF with CALLER INFO tag // Branch annotated VCFs by caller family for the intersection step + // An unrecognised meta.caller would silently vanish without the `other` arm, taking the whole + // sample out of the results with a successful exit. annotated_vcfs .branch { meta, _vcfs, _tbi -> deepvariant: meta.caller in [ 'deepvariant', 'deepsomatic' ] clair: meta.caller in ['clair3','clairs-to','clairs'] + other: true } .set{annotated_vcfs_branched} + + annotated_vcfs_branched.other + .map { meta, _vcfs, _tbi -> + error("SMALL_VARIANT_CONSENSUS: unrecognised meta.caller '${meta.caller}' for sample '${meta.id}'; expected one of [deepvariant, deepsomatic, clair3, clairs-to, clairs]") + } // annotated_vcfs_branched.deepvariant: [meta(caller=deepvariant/deepsomatic), vcf, tbi] // annotated_vcfs_branched.clair: [meta(caller=clair3/clairs-to/clairs), vcf, tbi] @@ -153,8 +171,10 @@ workflow SMALL_VARIANT_CONSENSUS { // deepvariant_ch: [meta (no caller), vcf, tbi] // Join DeepVariant and Clair VCFs per sample into a single tuple for BCFTOOLS_ISEC + // failOnMismatch: a sample present for one caller but not the other would otherwise be dropped + // from every downstream result while the run still reported success. deepvariant_ch - .join(clair_ch) + .join(clair_ch, failOnMismatch: true, failOnDuplicate: true) .map { meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> def vcfs = [deepvar_vcf, clair_vcf] def tbis = [deepvar_tbi, clair_tbi] From b646c20878318e6627c9ae763085f2869b46dc36 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:42:38 +0200 Subject: [PATCH 07/25] Adjudicate the tumour-only germline arm with DeepSomatic's verdict With --germline_var_keep deepvariant the tumour-only germline arm runs DeepVariant on the tumour BAM. DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is only PASS/RefCall/LowQual/NoCall -- so those calls mix germline and clonal somatic variants and cannot be separated on their own. Published unchanged, germline_smallvariants.vcf.gz and vep/germline/ carried most of the somatic call set. DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic variants"), PON, RefCall or PASS. That verdict is now transferred onto the DeepVariant records as INFO/DS_VERDICT and used to select the germline arm, mirroring how ClairS-TO adjudicates its own calls through NonSomatic and VCFSPLIT. This is a caller's own adjudication rather than positional subtraction of the somatic call set: on B1975944 only 1.14% (57,684) of DeepVariant's 5,058,527 PASS calls carry a positive somatic verdict, against ~87% positional overlap with the union somatic set. Only positively adjudicated germline sites are kept (GERMLINE or PON). RefCall and sites DeepSomatic never evaluated are dropped rather than assumed germline, so the arm is not overpopulated with unadjudicated calls; it keeps 83.1% of the input. The verdict stays in INFO/DS_VERDICT so the decision is auditable in the published VCF. The transfer is three independent bcftools invocations, so it is three aliased instances of the existing upstream modules (DS_VERDICT_QUERY, DS_VERDICT_ANNOTATE, DS_GERMLINE_SELECT) rather than a new bespoke process. DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS over 13.7M records), so transferring it as a string cannot inject the ";" that would break INFO parsing. DEEPSOMATIC now runs before DEEPVARIANT in the subworkflow because the verdict is built from its raw VCF, before the PASS filter discards the GERMLINE records. The deep family must therefore be enabled as a pair: --germline_var_keep deepvariant without deepsomatic in --somatic_var_keep is rejected at launch rather than silently producing an unadjudicated germline arm. Paired mode is untouched -- both Clair3 and DeepVariant already run on the normal BAM there, so the germline arm needs no adjudication. Also corrects the --smallvar_filter_pass docs: VCFTAG normalises FILTER to PASS, so `false` does not restore the pre-filter behaviour as previously claimed. Germline record counts change, so the snapshots need regenerating. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + conf/modules.config | 39 +++++ docs/usage.md | 35 ++++- .../local/tumor_only/tumoronly_smallvar.nf | 148 ++++++++++++------ .../utils_nfcore_lrsomatic_pipeline/main.nf | 24 +++ 5 files changed, 198 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04dd66d4..2710942b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. Note that `--smallvar_filter_pass false` does **not** fully restore the previous behaviour: `VCFTAG` normalises `FILTER` to `PASS`, so the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` no longer removes anything. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). diff --git a/conf/modules.config b/conf/modules.config index 99872d66..1cf9262a 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -166,6 +166,45 @@ process { enabled: false ] } + // GERMLINE VERDICT TRANSFER (tumor-only deep family). These three selectors are matched on the + // alias, so they do not disturb the generic BCFTOOLS_QUERY/ANNOTATE/VIEW entries below. + withName: '.*:DS_VERDICT_QUERY' { + ext.prefix = { "${meta.id}.ds_verdict" } + // FILTER is single-valued in DeepSomatic output (RefCall/GERMLINE/PON/PASS), so it cannot + // inject the ";" that would break INFO parsing once transferred. + ext.args = { + "-f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" + } + publishDir = [ + enabled: false + ] + } + + withName: '.*:DS_VERDICT_ANNOTATE' { + ext.prefix = { "${meta.id}.deepvariant_verdict" } + ext.args = { + '''-h <(echo '##INFO=') \ + -c CHROM,POS,REF,ALT,INFO/DS_VERDICT \ + -Oz \ + -W=tbi''' + } + publishDir = [ + enabled: false + ] + } + + withName: '.*:DS_GERMLINE_SELECT' { + ext.prefix = { "${meta.id}.deepvariant_germline" } + // Keep only positively-adjudicated germline sites. RefCall and sites DeepSomatic never + // evaluated (DS_VERDICT absent) are dropped rather than assumed germline. + ext.args = { + "-i 'INFO/DS_VERDICT=\"GERMLINE\" || INFO/DS_VERDICT=\"PON\"' --output-type z --write-index=tbi" + } + publishDir = [ + enabled: false + ] + } + withName: '.*:BCFTOOLS_QUERY' { ext.args = { "-f '%CHROM\t%POS\t%REF\t%ALT\t${meta.caller}\n'" diff --git a/docs/usage.md b/docs/usage.md index 7e62197c..bce38531 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -135,6 +135,23 @@ For structural variants, the CHM13 panel of normals is a merged panel combining For tumour-only small variants, ClairS-TO separates germline from somatic calls with a panel of normals and with its Verdict module, which tags each call as germline, somatic or subclonal somatic from tumour purity and allele-specific copy number. `--genome CHM13` supplies five CHM13 PON VCFs (gnomAD, dbSNP, 1000 Genomes, CoLoRSdb and ASAP), which **replace** the GRCh38 databases inside the container. Unless `--skip_ascat` is set, purity and copy number come from the pipeline's own ASCAT run (`CLAIRSTO_VERDICT_TAG`); only with `--skip_ascat` does ClairS-TO estimate them itself, from assembly-specific loci, allele and GC content files. A GRCh38 resource set on a CHM13 run leaves germline variants untagged. +When `--germline_var_keep` includes `deepvariant`, the tumour-only germline arm +runs DeepVariant on the **tumour** BAM. DeepVariant is a germline caller with no +somatic discrimination, so on its own those calls mix germline and clonal somatic +variants. The pipeline therefore transfers DeepSomatic's verdict onto them: +DeepSomatic evaluates the same sites and labels each `GERMLINE`, `PON`, `RefCall` +or `PASS`, and that label is recorded in `INFO/DS_VERDICT`. Only positively +adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; +`RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed +germline. On a 30x tumour-only sample this keeps about 83% of DeepVariant's +`PASS` calls and removes roughly 1% that DeepSomatic positively calls somatic. + +Because of this, `deepvariant` and `deepsomatic` must be enabled together: +`--germline_var_keep deepvariant` without `deepsomatic` in `--somatic_var_keep` is +rejected at launch. Note that even after adjudication the tumour-only germline arm +is a tumour-derived proxy, not a call set from normal tissue, and should not be +used for secondary findings without that caveat. + With `--genome CHM13 --skip_ascat` the pipeline builds a CHM13 resource set from the ASCAT files it already downloads, so no extra setup is needed. LogR correction is GC-only, as ClairS-TO recommends for CHM13: no replication timing file is published for the assembly. Without `--skip_ascat` nothing is built, because the tagging comes from ASCAT's own tables. To use a resource set of your own — another assembly, or a CHM13 set carrying an `RT_.txt` for replication timing correction — pass `--clairsto_cna_resources` together with `--skip_ascat`. Without `--skip_ascat` it is ignored, with a warning. Expected layout: @@ -387,11 +404,19 @@ otherwise be "every site every caller looked at", which inflates the phased VCFs by three orders of magnitude and produces a meaningless mutation burden. `--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's -VCF that is handed to the caller consensus, phasing, VEP and the report. Set it to -`false` to restore the previous unfiltered behaviour. In tumor-only mode ClairS-TO -is unaffected by the setting: `VCFSPLIT` already restricts its somatic split to -`PASS`. The per-caller VCFs published under `//variants/` -are never filtered, so no calls are lost from the results directory. +VCF that is handed to the caller consensus, phasing, VEP and the report. In +tumor-only mode ClairS-TO is unaffected by the setting: `VCFSPLIT` already +restricts its somatic split to `PASS`, and its germline split is `PASS`-rewritten +rather than `PASS`-filtered. The per-caller VCFs published under +`//variants/` are never filtered, so no calls are lost +from the results directory. + +Setting `--smallvar_filter_pass false` does **not** fully restore the pre-filter +behaviour. `VCFTAG` normalises `FILTER` to `PASS` on both arms before phasing, so +the later `--apply-filters PASS` on the signature input no longer removes +anything: with the filter off, non-`PASS` records reach SigProfiler that +previously could not. Use it to inspect the unfiltered call set, not to reproduce +older results. `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 2fbd1067..cb89b849 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -11,6 +11,12 @@ include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subwork include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' +// Germline verdict transfer: DeepSomatic adjudicates DeepVariant's tumor-derived germline calls. +// Three independent bcftools invocations, so three aliased instances of the upstream modules. +include { BCFTOOLS_QUERY as DS_VERDICT_QUERY } from '../../../modules/nf-core/bcftools/query/main' +include { BCFTOOLS_ANNOTATE as DS_VERDICT_ANNOTATE } from '../../../modules/nf-core/bcftools/annotate/main' +include { BCFTOOLS_VIEW as DS_GERMLINE_SELECT } from '../../../modules/nf-core/bcftools/view/main' + workflow TUMORONLY_SMALLVAR { @@ -132,6 +138,50 @@ workflow TUMORONLY_SMALLVAR { // clairsto_somatic_ch: [meta(+caller:'clairs-to'), vcf, tbi] -- somatic variants } + // DEEPSOMATIC in tumor-only mode: normal BAM/BAI are empty lists + if(somatic_var_keep.contains('deepsomatic')) { + tumor_bams + .map { meta, tumor_bam, tumor_bai -> + def normal_bam = [] + def normal_bai = [] + return [meta,normal_bam,normal_bai,tumor_bam,tumor_bai] + } + .set{deepsomatic_input_ch} + // deepsomatic_input_ch: [meta, [], [], tumor_bam, tumor_bai] + // empty normal_bam/bai signals tumor-only mode to DEEPSOMATIC subworkflow + + // + // SUBWORKFLOW: DEEPSOMATIC (local) + // Input: [meta, [], [], tumor_bam, tumor_bai] -- tumor-only (no normal) + // [[:],[]] / fasta / fai / [[:],[]] + // Output: .vcf -- [meta, vcf] + // .vcf_index -- [meta, tbi] + // + DEEPSOMATIC ( + deepsomatic_input_ch, + [[:],[]], // intervals (empty = genome-wide) + fasta, + fai, + [[:],[]], // GZI (empty if FASTA is uncompressed) + ds_pon_channel + ) + // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), + // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT + // already restricts it to PASS. The VCF published under variants/deepsomatic/ is + // unaffected. + DEEPSOMATIC_PASS_FILTER ( + DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + ) + + DEEPSOMATIC_PASS_FILTER.out.vcf + .map{ meta, vcf, tbi -> + def new_meta = meta + [caller:'deepsomatic'] + return [new_meta, vcf, tbi] + } + .set{deepsomatic_ch} + // deepsomatic_ch: [meta(+caller:'deepsomatic'), vcf, tbi] + } + // DEEPVARIANT: germline-only variant calling (no somatic mode for tumor-only) if(germline_var_keep.contains('deepvariant')) { @@ -168,13 +218,66 @@ workflow TUMORONLY_SMALLVAR { DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) ) + // GERMLINE VERDICT TRANSFER (tumor-only, deep family) + // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is + // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its + // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the + // germline VCF therefore carries most of the somatic call set. + // + // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic + // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO + // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 + // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% + // (57,684) PASS -- the last being real somatic calls that must not be published as germline. + // + // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and + // unevaluated sites are dropped rather than assumed germline. The verdict stays in + // INFO/DS_VERDICT so the decision is auditable in the published VCF. + // + // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified + // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would + // break INFO parsing. + // + // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) + // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter + // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER + // + DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) + + // + // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) + // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. + // DEEPVARIANT_PASS_FILTER.out.vcf + .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) + .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) + .map { meta, vcf, tbi, annotations, annotations_index -> + def columns = [] // no extra column specs + def header_lines = [] // no extra header lines + def rename_chrs = [] // no chromosome renaming + return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] + } + .set{ ds_verdict_annotate_input } + + DS_VERDICT_ANNOTATE ( ds_verdict_annotate_input ) + + // + // MODULE: DS_GERMLINE_SELECT (BCFTOOLS_VIEW alias, label: process_medium) + // Keeps only the positively-adjudicated germline records (see ext.args in conf/modules.config). + // + DS_GERMLINE_SELECT ( + DS_VERDICT_ANNOTATE.out.vcf.join(DS_VERDICT_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true), + [], [], [] + ) + + DS_GERMLINE_SELECT.out.vcf + .join(DS_GERMLINE_SELECT.out.index, failOnMismatch: true, failOnDuplicate: true) .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] } .set{deepvariant_ch} - // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] + // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] -- germline-adjudicated only } // COMBINE GERMLINE VARIANTS @@ -207,49 +310,6 @@ workflow TUMORONLY_SMALLVAR { .set{germline_vcf} } - // DEEPSOMATIC in tumor-only mode: normal BAM/BAI are empty lists - if(somatic_var_keep.contains('deepsomatic')) { - tumor_bams - .map { meta, tumor_bam, tumor_bai -> - def normal_bam = [] - def normal_bai = [] - return [meta,normal_bam,normal_bai,tumor_bam,tumor_bai] - } - .set{deepsomatic_input_ch} - // deepsomatic_input_ch: [meta, [], [], tumor_bam, tumor_bai] - // empty normal_bam/bai signals tumor-only mode to DEEPSOMATIC subworkflow - - // - // SUBWORKFLOW: DEEPSOMATIC (local) - // Input: [meta, [], [], tumor_bam, tumor_bai] -- tumor-only (no normal) - // [[:],[]] / fasta / fai / [[:],[]] - // Output: .vcf -- [meta, vcf] - // .vcf_index -- [meta, tbi] - // - DEEPSOMATIC ( - deepsomatic_input_ch, - [[:],[]], // intervals (empty = genome-wide) - fasta, - fai, - [[:],[]], // GZI (empty if FASTA is uncompressed) - ds_pon_channel - ) - // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), - // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT - // already restricts it to PASS. The VCF published under variants/deepsomatic/ is - // unaffected. - DEEPSOMATIC_PASS_FILTER ( - DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) - ) - - DEEPSOMATIC_PASS_FILTER.out.vcf - .map{ meta, vcf, tbi -> - def new_meta = meta + [caller:'deepsomatic'] - return [new_meta, vcf, tbi] - } - .set{deepsomatic_ch} - // deepsomatic_ch: [meta(+caller:'deepsomatic'), vcf, tbi] - } // COMBINE SOMATIC VARIATION if (somatic_var_keep.size() > 1) { diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index a64c405c..072a9d5f 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -239,6 +239,30 @@ workflow PIPELINE_COMPLETION { def validateInputParameters() { genomeExistsError() validateReportGenePanels() + validateDeepCallerPairing() +} + +// +// The tumor-only germline arm runs DeepVariant on the TUMOR BAM, which cannot separate germline +// from clonal somatic calls on its own. TUMORONLY_SMALLVAR resolves that by transferring +// DeepSomatic's FILTER verdict (GERMLINE/PON/RefCall/PASS) onto those calls, so DeepVariant +// without DeepSomatic would leave the germline arm unadjudicated -- and would reference +// DEEPSOMATIC.out before it exists. Require the deep family to be fully on or fully off. +// +def callerList(value) { + if (value instanceof List) { + return value + } + return value.toString().tokenize(',').collect { token -> token.trim() } +} + +def validateDeepCallerPairing() { + def germline = callerList(params.germline_var_keep) + def somatic = callerList(params.somatic_var_keep) + + if (germline.contains('deepvariant') && !somatic.contains('deepsomatic')) { + error("--germline_var_keep includes 'deepvariant' but --somatic_var_keep does not include 'deepsomatic'. In tumor-only mode the DeepVariant germline calls are adjudicated with DeepSomatic's verdict, so the two must be enabled together. Add 'deepsomatic' to --somatic_var_keep, or drop 'deepvariant' from --germline_var_keep.") + } } // From 33bb9b2223619acbda5b3aa2f81d8ef4b5dd9f70 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:46:51 +0200 Subject: [PATCH 08/25] Preserve the caller's FILTER through VCFTAG so the PASS filter still bites VCFTAG rewrites FILTER to PASS on both arms before phasing, which is needed so that downstream tools filtering on PASS still see the germline records ClairS-TO marks NonSomatic. But it did so destructively, and every record reaching the published phased VCFs, VEP and the report therefore read FILTER=PASS with the caller's verdict gone. That also silently disabled the signature-stage filter. SIGNATURES_BCFTOOLS_VIEW ran --apply-filters PASS on a call set whose FILTER had already been normalised to PASS, so it could never remove anything: with --smallvar_filter_pass false, RefCall/LowQual/GERMLINE/PON records reached SigProfilerMatrixGenerator that previously could not, and the matrices were computed over mostly-reference sites. The parameter was documented as restoring the previous behaviour, which it did not. VCFTAG now records the caller's FILTER in INFO/ORIG_FILTER before overwriting it, as VCFSPLIT already did for the ClairS-TO arm, and the signature filter tests that field instead. VCFSPLIT's stamp is respected rather than duplicated: both the header line and the per-record field are added only when absent, since a duplicate INFO key makes the record unparseable. Multi-valued FILTER is joined with "," because ";" separates INFO fields, and a FILTER of "." is recorded as "." rather than skipped. bcftools accepts only one of -i/-e, so the ALT="*" exclusion is folded into the same include expression. Verified against the module fixture with bcftools 1.20, including a record that already carried ORIG_FILTER and one with a multi-valued FILTER: exactly one ORIG_FILTER header line, no double stamping, and htslib parses the result. The module test is extended to cover these cases. Note that the module test cannot currently run locally: nf-test 0.9.3 generates `VCFTAG(*input)` and the available Nextflow 26.04.1 rejects the spread operator under its strict syntax, while the pipeline requires >=25.10.4 so 25.04.6 will not run either. This is pre-existing and reproduces unchanged at 5b38dcf. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- conf/modules.config | 10 ++++++++-- docs/usage.md | 13 +++++++------ modules/local/vcftag/main.nf | 21 ++++++++++++++++++++- modules/local/vcftag/tests/main.nf.test | 14 ++++++++++++-- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2710942b..4789d488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). - [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. Note that `--smallvar_filter_pass false` does **not** fully restore the previous behaviour: `VCFTAG` normalises `FILTER` to `PASS`, so the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` no longer removes anything. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now runs [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) v1.6.0 from `ghcr.io/ljwharbers/lrsomatic-report:1.6.0` (`oras://ghcr.io/ljwharbers/lrsomatic-report-sif:1.6.0` under Singularity/Apptainer) instead of the source vendored at `assets/lrsomatic_report`, which is deleted; updating the tool is now a tag bump. The image is `linux/amd64` only and **Conda is not supported for this step**: the module errors under `-profile conda`/`mamba`, `conda` is dropped from the nf-test CI matrix, and `--skip_report` gives a conda run (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 1cf9262a..29b9d235 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -717,8 +717,14 @@ process { } withName : '.*:SIGNATURES_BCFTOOLS_VIEW' { - // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only - ext.args = { "--apply-filters PASS --types snps,mnps,indels --exclude 'ALT=\"*\"' --output-type v" } + // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only. + // The PASS test reads INFO/ORIG_FILTER, not FILTER: VCFTAG normalises FILTER to PASS on + // every record before phasing, so --apply-filters PASS here could never remove anything and + // non-PASS records reached SigProfiler whenever --smallvar_filter_pass was false. + // ORIG_FILTER carries the caller's own verdict, so the filter bites again. "." is kept + // because an unfiltered record is not a rejected one. bcftools allows only one of -i/-e, + // so the ALT="*" exclusion is folded into the same expression. + ext.args = { "-i '(INFO/ORIG_FILTER=\"PASS\" || INFO/ORIG_FILTER=\".\") && ALT!=\"*\"' --types snps,mnps,indels --output-type v" } ext.prefix = { "${meta.id}_somatic_pass" } publishDir = [ enabled: false diff --git a/docs/usage.md b/docs/usage.md index bce38531..cab62881 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -411,12 +411,13 @@ rather than `PASS`-filtered. The per-caller VCFs published under `//variants/` are never filtered, so no calls are lost from the results directory. -Setting `--smallvar_filter_pass false` does **not** fully restore the pre-filter -behaviour. `VCFTAG` normalises `FILTER` to `PASS` on both arms before phasing, so -the later `--apply-filters PASS` on the signature input no longer removes -anything: with the filter off, non-`PASS` records reach SigProfiler that -previously could not. Use it to inspect the unfiltered call set, not to reproduce -older results. +Set it to `false` to restore the previous unfiltered behaviour. `VCFTAG` +normalises `FILTER` to `PASS` on both arms before phasing, so the caller's own +verdict is preserved in `INFO/ORIG_FILTER` and the signature input is filtered on +that field rather than on `FILTER`; without this the `PASS` filter in front of +SigProfiler could never remove anything once `FILTER` had been rewritten. Every +published VCF downstream of phasing therefore reads `FILTER=PASS`, with the +original value available in `INFO/ORIG_FILTER`. `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf index 3fd9dc81..a7d81868 100644 --- a/modules/local/vcftag/main.nf +++ b/modules/local/vcftag/main.nf @@ -27,20 +27,39 @@ process VCFTAG { # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and # filter on FILTER, so a FILTER-based label does not survive to where it is needed. # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. - bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"} + # + # The caller's own FILTER is preserved in INFO/ORIG_FILTER before it is overwritten, so the + # verdict is not destroyed by the normalisation. FILTER is ";"-delimited but ";" separates INFO + # fields, so it is stored as ",". VCFSPLIT already stamps ORIG_FILTER on the ClairS-TO arm; both + # the header line and the per-record field are therefore added only when not already present, + # since a duplicate INFO key would make the record unparseable. + bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"; seen_orig=0} + /^##INFO=" + if (!seen_orig) { + print "##INFO=" + } print next } { + if (\$8 !~ /(^|;)ORIG_FILTER=/) { + of = \$7 + gsub(/;/, ",", of) + \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of + } \$7 = "PASS" \$8 = (\$8 == "." || \$8 == "") ? flag : \$8 ";" flag print } ' | bgzip -c > ${prefix}.vcf.gz + # tabix will index a VCF whose header htslib cannot parse, so read it back here rather than + # letting a malformed header surface as a confusing failure in a later process. + bcftools view -h ${prefix}.vcf.gz > /dev/null + tabix -p vcf ${prefix}.vcf.gz """ diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test index 18b79c41..558f2dd7 100644 --- a/modules/local/vcftag/tests/main.nf.test +++ b/modules/local/vcftag/tests/main.nf.test @@ -42,8 +42,18 @@ nextflow_process { { assert lines.every { it.split('\t')[6] == 'PASS' } }, // a pre-existing INFO field is preserved rather than overwritten { assert lines.any { it.contains('EXISTING') && it.contains('CALLER=clairs-to') } }, - // the record whose INFO was '.' gets the bare flag, not '.;SOMATIC' - { assert lines.any { it.split('\t')[7] == 'SOMATIC' } } + // the record whose INFO was '.' gets no leading '.', just the stamped fields + { assert lines.any { it.split('\t')[7] == 'ORIG_FILTER=RefCall;SOMATIC' } }, + // the caller's FILTER is preserved before being normalised away + { assert header.contains('ID=ORIG_FILTER') }, + { assert lines.every { it.contains('ORIG_FILTER=') } }, + { assert lines.any { it.contains('ORIG_FILTER=PASS') } }, + { assert lines.any { it.contains('ORIG_FILTER=NonSomatic') } }, + { assert lines.any { it.contains('ORIG_FILTER=RefCall') } }, + // a record whose FILTER was '.' records '.' rather than being skipped + { assert lines.any { it.contains('ORIG_FILTER=.') } }, + // ORIG_FILTER is stamped exactly once per record + { assert lines.every { it.split('\t')[7].count('ORIG_FILTER=') == 1 } } ) } } From 33a631badbb087d116190d5ff9c7f560e15749a0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 11:58:50 +0200 Subject: [PATCH 09/25] Make the VCFTAG test runnable The test read its output through path(...).vcf, an accessor provided by an nft-vcf plugin that nf-test.config does not load, so it failed with MissingPropertyException on every run and had never passed. It now uses the built-in linesGzip accessor instead, which needs no plugin. Co-Authored-By: Claude Opus 5 (1M context) --- modules/local/vcftag/tests/main.nf.test | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test index 558f2dd7..3f4a658f 100644 --- a/modules/local/vcftag/tests/main.nf.test +++ b/modules/local/vcftag/tests/main.nf.test @@ -28,8 +28,11 @@ nextflow_process { then { assert process.success - def lines = path(process.out.vcf[0][1]).vcf.variantsAsStrings - def header = path(process.out.vcf[0][1]).vcf.header.getMetaDataInSortedOrder().join('\n') + // linesGzip is built into nf-test; the .vcf accessor needs an nft-vcf plugin that + // nf-test.config does not load, so this reads the records directly. + def all = path(process.out.vcf[0][1]).linesGzip + def lines = all.findAll { line -> !line.startsWith('#') } + def header = all.findAll { line -> line.startsWith('##') }.join('\n') assertAll( // every input record survives -- tagging must not filter From 825af157f55db18e7e0088660be0a544c77808bb Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 12:06:14 +0200 Subject: [PATCH 10/25] Fold STANDARDIZE_AF into BCFTOOLS_ANNOTATE STANDARDIZE_AF and BCFTOOLS_ANNOTATE were two back-to-back bcftools annotate calls on the same file with disjoint options -- one renaming the allele frequency FORMAT key, the next stamping INFO/CALLER. --rename-annots composes with -a/-c/-h, verified against a real VCF, so a single invocation does both and the alias is removed along with its modules.config block. The rename is selected by meta.rename_to, set only when combine_method is 'all'; in 'consensus' mode every surviving record comes from one caller and needs no rename. Ordering is safe because BCFTOOLS_QUERY reads only CHROM/POS/REF/ALT and does not care whether the rename has happened. The rendered ext.args was checked for all three meta cases (VAF, AF, key absent): the escapes reach printf intact and an absent key yields no --rename-annots. The joins around the annotate step now use failOnMismatch/failOnDuplicate for the same reason as the caller join: a missing annotation table should stop the run, not quietly drop the sample. Co-Authored-By: Claude Opus 5 (1M context) --- conf/modules.config | 22 +++++---- subworkflows/local/small_variant_consensus.nf | 45 ++++++++----------- 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 29b9d235..5ca2a6c5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -143,21 +143,19 @@ process { enabled: false ] } - withName: '.*STANDARDIZE_AF' { - ext.prefix = { "${meta.id}.${meta.caller}_standardized" } - ext.args = { - meta.rename_to == 'VAF' - ? "--rename-annots <(printf 'FORMAT/AF\\tFORMAT/VAF\\n') -Oz -W=tbi" - : "--rename-annots <(printf 'FORMAT/VAF\\tFORMAT/AF\\n') -Oz -W=tbi" - } - publishDir = [ - enabled: false - ] - } withName: '.*:BCFTOOLS_ANNOTATE' { ext.prefix = { "${meta.id}.${meta.caller}" } + // Stamps INFO/CALLER and, in 'all' mode, unifies the allele frequency key in the same + // invocation: --rename-annots composes with -a/-c/-h, so this does not need a second + // bcftools annotate pass. meta.rename_to is set only when SMALL_VARIANT_CONSENSUS is + // merging two callers into one VCF. ext.args = { - '''-h <(echo '##INFO=') \ + def rename = meta.rename_to == 'VAF' + ? "--rename-annots <(printf 'FORMAT/AF\\tFORMAT/VAF\\n') " + : meta.rename_to == 'AF' + ? "--rename-annots <(printf 'FORMAT/VAF\\tFORMAT/AF\\n') " + : "" + rename + '''-h <(echo '##INFO=') \ -c CHROM,POS,REF,ALT,INFO/CALLER \ -Oz \ -W=tbi''' diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index 43e5114a..d50ba7f7 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -2,7 +2,6 @@ include { BCFTOOLS_NORM } from '../../modul include { BCFTOOLS_ISEC } from '../../modules/nf-core/bcftools/isec/main' include { BCFTOOLS_QUERY } from '../../modules/nf-core/bcftools/query/main' include { BCFTOOLS_ANNOTATE } from '../../modules/nf-core/bcftools/annotate/main' -include { BCFTOOLS_ANNOTATE as STANDARDIZE_AF } from '../../modules/nf-core/bcftools/annotate/main' include { BCFTOOLS_CONCAT } from '../../modules/nf-core/bcftools/concat/main' include { BCFTOOLS_SORT } from '../../modules/nf-core/bcftools/sort/main' include { BCFTOOLS_SORT as SORT_POST_NORM } from '../../modules/nf-core/bcftools/sort/main' @@ -42,7 +41,7 @@ workflow SMALL_VARIANT_CONSENSUS { // normalized_vcfs: [meta(+caller), vcf.gz, tbi] -- normalised, sorted per-caller VCF // - // MODULE: STANDARDIZE_AF (BCFTOOLS_ANNOTATE alias, label: process_low) -- rename the AF FORMAT field to the priority caller's: + // ALLELE FREQUENCY KEY -- BCFTOOLS_ANNOTATE below renames the AF FORMAT field to the priority caller's: // FORMAT/AF -> FORMAT/VAF when prioritize_caller is 'deepvariant'/'deepsomatic' // FORMAT/VAF -> FORMAT/AF when prioritize_caller is 'clair' // This guarantees the merged VCF exposes allele frequency under a single FORMAT key, which is @@ -56,27 +55,10 @@ workflow SMALL_VARIANT_CONSENSUS { // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record // reaching here carries one ALT and one AF value, making the two declarations equivalent. // - if (combine_method == 'all') { - normalized_vcfs - .map { meta, vcf, tbi -> - def rename_to = prioritize_caller in ['deepvariant', 'deepsomatic'] ? 'VAF' : 'AF' - def new_meta = meta + [rename_to: rename_to] - return [new_meta, vcf, tbi, [], [], [], [], []] - } - .set { standardize_input } - - STANDARDIZE_AF(standardize_input) - - STANDARDIZE_AF.out.vcf - .join(STANDARDIZE_AF.out.tbi) - .map { meta, vcf, tbi -> - def clean_meta = meta.findAll { k, _v -> k != 'rename_to' } - return [clean_meta, vcf, tbi] - } - .set { normalized_vcfs } - // normalized_vcfs: [meta(+caller), vcf, tbi] -- normalised, AF-standardized per-caller VCF - } - // In 'consensus' mode, normalized_vcfs comes from SORT_POST_NORM (post-BCFTOOLS_NORM re-sorting) + // The rename is carried out by BCFTOOLS_ANNOTATE below rather than by a second annotate call: + // meta.rename_to selects the --rename-annots file in conf/modules.config. Only 'all' mode needs + // it, since in 'consensus' mode every surviving record comes from one caller. BCFTOOLS_QUERY + // reads only CHROM/POS/REF/ALT, so it does not care whether the rename has happened yet. // // MODULE: BCFTOOLS_QUERY (label: process_single) @@ -89,13 +71,18 @@ workflow SMALL_VARIANT_CONSENSUS { // Prepare BCFTOOLS_ANNOTATE input: VCF + caller-name annotation file normalized_vcfs - .join(BCFTOOLS_QUERY.out.output) - .join(BCFTOOLS_QUERY.out.index) + .join(BCFTOOLS_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) + .join(BCFTOOLS_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) .map{ meta, vcf, tbi, annotations, annotations_index -> def columns = [] // no extra column specs def header_lines = [] // no extra header lines def rename_chrs = [] // no chromosome renaming - return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] + // 'all' mode merges records from both callers into one VCF, so the allele + // frequency key is unified here; 'consensus' mode needs no rename. + def new_meta = combine_method == 'all' + ? meta + [rename_to: (prioritize_caller in ['deepvariant', 'deepsomatic'] ? 'VAF' : 'AF')] + : meta + return [ new_meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] } .set{annotate_input} // annotate_input: [meta, vcf, tbi, annotations_tsv, annotations_tbi, [], [], []] @@ -110,7 +97,11 @@ workflow SMALL_VARIANT_CONSENSUS { BCFTOOLS_ANNOTATE(annotate_input) BCFTOOLS_ANNOTATE.out.vcf - .join(BCFTOOLS_ANNOTATE.out.tbi) + .join(BCFTOOLS_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .map { meta, vcf, tbi -> + def clean_meta = meta.findAll { k, _v -> k != 'rename_to' } + return [clean_meta, vcf, tbi] + } .set{annotated_vcfs} // annotated_vcfs: [meta(+caller), vcf, tbi] -- VCF with CALLER INFO tag From d4401a46505cd0704a48489cd12893d6736a1198 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 12:08:13 +0200 Subject: [PATCH 11/25] Drop a duplicated CLAIR3 argument and correct a false ordering comment CLAIR3 received --sample_name twice: the module passes it from ext.prefix, which defaults to meta.id, and conf/modules.config passed the identical value again. Harmless with argparse, but it reads as if the name were configurable. Removed the config copy, matching how CLAIRSTO is already handled on dev. The comment in PHASING_HAPLOTYPING claiming the somatic VCF is passed first "(higher priority in phasing)" was false: BCFTOOLS_CONCAT sorts its input file list alphabetically, so with the _germline_tagged/_somatic_tagged prefixes the germline file is always passed first. The ordering is cosmetic under -a, which emits in coordinate order; the comment now says so rather than describing an intent the module discards. Co-Authored-By: Claude Opus 5 (1M context) --- conf/modules.config | 2 +- subworkflows/local/phasing_haplotyping.nf | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 5ca2a6c5..afaeeb06 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -595,7 +595,7 @@ process { } withName: '.*:CLAIR3' { - ext.args = { "--sample_name=${meta.id}" } + // --sample_name is passed by the module itself, from ext.prefix (which defaults to meta.id). publishDir = [ path: { "${params.outdir}/${meta.id}/variants/clair3" }, mode: params.publish_dir_mode, diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index d8f8ce3e..3de385ef 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -163,7 +163,10 @@ workflow PHASING_HAPLOTYPING { tagged_germline_vcf .join(tagged_somatic_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> - def vcfs = [som_vcf, germ_vcf] // somatic first (higher priority in phasing) + // Order here is cosmetic: BCFTOOLS_CONCAT sorts its input file list alphabetically + // (modules/nf-core/bcftools/concat/main.nf), so the germline file is passed first + // regardless. With -a the output is coordinate-ordered either way. + def vcfs = [som_vcf, germ_vcf] def tbis = [som_tbi, germ_tbi] return [ meta, vcfs, tbis] } From d3b45818928f04df4a974c02bc7933055f15d1e0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Tue, 22 Sep 2026 22:39:14 +0200 Subject: [PATCH 12/25] Regenerate snapshots for the germline/somatic tagging processes Picks up TAG_GERMLINE and TAG_SOMATIC, and restores the clairsto and lrsomatic_report versions that the previous snapshot run had rolled back. Co-Authored-By: Claude Opus 5 (1M context) --- tests/clair_only.nf.test.snap | 12 +++++-- tests/consensus.nf.test.snap | 49 +++++++++++++++++---------- tests/deep_only.nf.test.snap | 27 +++++++++++---- tests/default.nf.test.snap | 12 +++++-- tests/union.nf.test.snap | 62 +++++++++++++++++++++-------------- 5 files changed, 108 insertions(+), 54 deletions(-) diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index 1a13f798..3c26ccd1 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -28,7 +28,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -57,7 +57,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -111,6 +111,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -804,6 +810,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T16:39:52.362456747" + "timestamp": "2026-09-22T17:07:33.562660027" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index c7b025c3..3bcf37d7 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -33,7 +33,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -59,6 +59,15 @@ "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, + "DS_GERMLINE_SELECT": { + "bcftools": "1.23.1" + }, + "DS_VERDICT_ANNOTATE": { + "bcftools": 1.22 + }, + "DS_VERDICT_QUERY": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -80,7 +89,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -131,6 +140,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -578,38 +593,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,35824329c737d89e70234e0b8e2de3fa", - "sample1_normal.bam.bai:md5,2eccbe13acc6273ebf6332ec5a5da815", - "sample1_tumor.bam:md5,969daba408c88235a28c6ed38d513965", - "sample1_tumor.bam.bai:md5,4face49fc270117ab49d72d1413a7efb", + "sample1_normal.bam:md5,a524e54ab0685eeb79e461926d8c7906", + "sample1_normal.bam.bai:md5,ba98fb39fc3cc8b7319376b003aa3dcd", + "sample1_tumor.bam:md5,db8b0da3dd1a6f1c38420a4942881119", + "sample1_tumor.bam.bai:md5,1b75ed3238520172019817b88d254f1f", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,b85799ad0a512d6c30ff05f8b5dca54a", - "sample1_whatshap_stats.log:md5,5889131d24524deb76e5d585e3a2dd3f", - "sample1_whatshap_stats.tsv:md5,648f4ba86c78fc80d388c95193e3c4f4", + "sample1_whatshap_stats.gtf:md5,30bde8f88b7d4e88b935e88e00997ce7", + "sample1_whatshap_stats.log:md5,51d522df21c6b526909a5cc9de93eaea", + "sample1_whatshap_stats.tsv:md5,8ce67834b3b7471c04151b55585a56bc", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", - "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", - "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", - "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", + "sample2_normal.bam:md5,0b3633a4d389b4467fb3eac83da94882", + "sample2_normal.bam.bai:md5,d96d0071ab25ca8dc2327acba4395515", + "sample2_tumor.bam:md5,5c549642cc7192289222d558dc3a07a7", + "sample2_tumor.bam.bai:md5,1554e8cc8ced05271f5d5f7ac7819410", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", - "sample2_whatshap_stats.log:md5,e7336c45aa5738a67f032898accb305e", - "sample2_whatshap_stats.tsv:md5,e478af5b3adb92a590a485f444a0dabe", + "sample2_whatshap_stats.gtf:md5,2e5ace4cac0b42bb6132513062781e47", + "sample2_whatshap_stats.log:md5,9aa99ee14473f65071366ed1729ab064", + "sample2_whatshap_stats.tsv:md5,13cbc783b1357e1bd2c6bb51ae6f1267", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -636,6 +651,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:28:32.306705325" + "timestamp": "2026-09-22T17:31:36.878403725" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index dd3cba3d..53251031 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -35,6 +35,15 @@ "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, + "DS_GERMLINE_SELECT": { + "bcftools": "1.23.1" + }, + "DS_VERDICT_ANNOTATE": { + "bcftools": 1.22 + }, + "DS_VERDICT_QUERY": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -104,6 +113,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -561,14 +576,14 @@ "read_qual.txt:md5,8b92ff7dc4536188be159b95525511cd", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample3_tumor.bam:md5,965264ef8436cb887ad02c92d40fb50e", - "sample3_tumor.bam.bai:md5,cfc6329667a3c6c66e3c0ca0ace4c6e9", + "sample3_tumor.bam:md5,9449340acf09825767e1f5b4a3f0e5c2", + "sample3_tumor.bam.bai:md5,3be46bae7402c3865f27881457b7a466", "sample3_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", - "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,061ecb622d83678229f5d0a7b0e95d75", - "sample3_whatshap_stats.tsv:md5,7c7453a5eed17e9ccc405844f3bd511a", + "sample3_whatshap_stats.gtf:md5,5a9b20b6ed25ef2ede4271156a83ce88", + "sample3_whatshap_stats.log:md5,a62c149286176f863d4293d0748a6dec", + "sample3_whatshap_stats.tsv:md5,90c20ce05eef472d9f5bb77754636d31", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -581,6 +596,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:36:01.066378821" + "timestamp": "2026-09-22T17:50:37.140724409" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index c0b8e75a..9310ac78 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -18,7 +18,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -47,7 +47,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -95,6 +95,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -582,6 +588,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:44:15.090049384" + "timestamp": "2026-09-22T18:06:32.118064724" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index 0663c675..d7a68115 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -30,7 +30,7 @@ "clairs": "0.4.4" }, "CLAIRSTO": { - "clairsto": "0.4.2" + "clairsto": "0.5.1" }, "CRAMINO_POST": { "cramino": "1.3.0" @@ -56,6 +56,15 @@ "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, + "DS_GERMLINE_SELECT": { + "bcftools": "1.23.1" + }, + "DS_VERDICT_ANNOTATE": { + "bcftools": 1.22 + }, + "DS_VERDICT_QUERY": { + "bcftools": 1.22 + }, "GERMLINE_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, @@ -77,7 +86,7 @@ "longphase": "2.0.1" }, "LRSOMATICREPORT": { - "lrsomatic_report": "1.3.2" + "lrsomatic_report": "1.6.0" }, "METAEXTRACT": { "samtools": 1.21 @@ -123,14 +132,17 @@ "SORT_POST_NORM": { "bcftools": 1.22 }, - "STANDARDIZE_AF": { - "bcftools": 1.22 - }, "SV_VEP": { "ensemblvep": 115.2, "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -578,52 +590,52 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,271da2f58d613f1f101a12ef06eca766", - "sample1_normal.bam.bai:md5,c51c824e0ac5451f9c10eee78f40cb98", - "sample1_tumor.bam:md5,f5f68131d379181bbc335c9530e9cc7e", - "sample1_tumor.bam.bai:md5,75e62a6480344075038ce6ad2499bfbb", + "sample1_normal.bam:md5,dcb537c163750d41140d0a4da107f18e", + "sample1_normal.bam.bai:md5,12cf13fae8230cfc1cc8888f80102143", + "sample1_tumor.bam:md5,58cf65517c91958f95a1f4be1c75e2df", + "sample1_tumor.bam.bai:md5,e40049da796ad038e7ebeaa9ac070488", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,2b8ff0a9e055c9b4432bc3da9cf98a4d", - "sample1_whatshap_stats.log:md5,70a5270fec94f8b569a4a7bda49751db", - "sample1_whatshap_stats.tsv:md5,1eab3a16f1a51016c3b74f104de152a6", + "sample1_whatshap_stats.gtf:md5,9a7823e6b572036a7e95904b0083b8ca", + "sample1_whatshap_stats.log:md5,05c819627f77305a75d776a27dbf247e", + "sample1_whatshap_stats.tsv:md5,5b9d3f6a6e1e1c67c855f8f9b581bb18", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,467fb8fb5e25e0935baf7732aeafa958", - "sample2_normal.bam.bai:md5,8211da94e88be2653c9e9c7c37f8a585", - "sample2_tumor.bam:md5,7cc3341d659e0f58fe683618a8942f0f", - "sample2_tumor.bam.bai:md5,0b4249a3db1dd784f3936ed26478e1b2", + "sample2_normal.bam:md5,74e92562e0cd406a6282bce98ccc93d2", + "sample2_normal.bam.bai:md5,430c3b069a79227396a6b4f1c37a02ef", + "sample2_tumor.bam:md5,0059b8022c674b5daf2fd001ab69a54e", + "sample2_tumor.bam.bai:md5,e236afb5f7a644827e2b38cd36838d98", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,428701fa4041349a6ba5a65662aec261", - "sample2_whatshap_stats.log:md5,be94280217c27d05773afa1465152c8a", - "sample2_whatshap_stats.tsv:md5,0c97b6033a149898e88aaf9c632b2de4", + "sample2_whatshap_stats.gtf:md5,8107c5dce892aa11145274fd088788f8", + "sample2_whatshap_stats.log:md5,c2b17319f0f9e6a8ec102be86aa833ba", + "sample2_whatshap_stats.tsv:md5,7028c4a428f5d0fd31472439e287e565", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", "read_qual.txt:md5,8b92ff7dc4536188be159b95525511cd", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample3_tumor.bam:md5,116b6944da4aa833a8d21c46b5f5ecfe", - "sample3_tumor.bam.bai:md5,ea9eca53bbaba26d40b791a2ea1aadf6", + "sample3_tumor.bam:md5,94e7f18f0a22d16f0868780df7975447", + "sample3_tumor.bam.bai:md5,a2cc1013d6fde4fec50608fa9681a7a7", "sample3_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample3_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample3_tumor.stats:md5,ecd5ea4fee37379dd5c5ae3e89dfddda", - "sample3_whatshap_stats.gtf:md5,f47156e18c490ff9a4e6efd04d43acc5", - "sample3_whatshap_stats.log:md5,496a531c3c19585603fa7fded0c958ba", - "sample3_whatshap_stats.tsv:md5,fba638c8da0d7a9fdacb11734e219b16", + "sample3_whatshap_stats.gtf:md5,5a9b20b6ed25ef2ede4271156a83ce88", + "sample3_whatshap_stats.log:md5,dc9a34518331417b7b5811462b722791", + "sample3_whatshap_stats.tsv:md5,26a07a6db5f1e5bf7844aa0a8c190cb7", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,56e899f85876cee082788927d0f89c5f", @@ -636,6 +648,6 @@ "nf-test": "0.9.3", "nextflow": "26.04.1" }, - "timestamp": "2026-09-18T17:54:05.409593728" + "timestamp": "2026-09-22T18:43:33.111991495" } } \ No newline at end of file From 25a0191be528d6454b27671e7ec068e9ed2450ec Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 23 Sep 2026 11:53:33 +0200 Subject: [PATCH 13/25] fix report clog --- tests/chm13.nf.test.snap | 15 ++++++-- workflows/lrsomatic.nf | 76 +++++++++++++++++++++++++++++++--------- 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/tests/chm13.nf.test.snap b/tests/chm13.nf.test.snap index a267d392..ad12da16 100644 --- a/tests/chm13.nf.test.snap +++ b/tests/chm13.nf.test.snap @@ -67,6 +67,9 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, + "PASS_FILTER": { + "bcftools": "1.23.1" + }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -101,6 +104,12 @@ "perl-math-cdf": 0.1, "tabix": 1.21 }, + "TAG_GERMLINE": { + "bcftools": 1.2 + }, + "TAG_SOMATIC": { + "bcftools": 1.2 + }, "UNTAR": { "untar": 1.34 }, @@ -136,9 +145,9 @@ } ], "meta": { - "nf-test": "0.9.0", - "nextflow": "26.04.6" + "nf-test": "0.9.3", + "nextflow": "25.10.4" }, - "timestamp": "2026-09-21T11:19:37.348390574" + "timestamp": "2026-09-23T10:46:28.14524359" } } \ No newline at end of file diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index 37bd1c4c..9704b5ed 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -696,8 +696,10 @@ workflow LRSOMATIC { // ascat_tumoronly_ch: [meta, purityploidy, segments] // All ASCAT files per sample for the report module, which globs by suffix + // groupKey: release each sample on its own three emissions, not when ASCAT finishes for all ch_ascat_files = ASCAT.out.segments_raw .mix(ASCAT.out.purityploidy, ASCAT.out.png) + .map { meta, files -> [groupKey(meta, 3), files] } .groupTuple() .map { meta, files -> [meta, files.flatten()] } // ch_ascat_files: [meta, [file, file, ...]] @@ -1255,8 +1257,10 @@ workflow LRSOMATIC { ) // The WAKHAN outputs the report renders: ranked solutions, heatmap, per-solution plots + // groupKey: release each sample on its own three emissions ch_wakhan_files = WAKHAN.out.solutions_ranks .mix(WAKHAN.out.heatmap_html, WAKHAN.out.solution_dirs) + .map { meta, files -> [groupKey(meta, 3), files] } .groupTuple() .map { meta, files -> [meta, files.flatten()] } // solution_dirs contributes a list // ch_wakhan_files: [meta, [file_or_dir, ...]] @@ -1276,13 +1280,19 @@ workflow LRSOMATIC { .set { report_id_meta } // report_id_meta: [id, meta] - ch_somatic_vep_vcf - .map { meta, vcf -> [meta.id, vcf] } - .set { report_vep_ch } + // A skipped module leaves an empty leg, and remainder: true then defers every sample to + // channel close. One [] per sample keeps each leg matched so samples report independently. + def report_empty_slot = { -> report_id_meta.map { id, _meta -> [id, []] } } - ch_sv_vep_vcf - .map { meta, vcf -> [meta.id, vcf] } - .set { report_sv_vep_ch } + def report_vep_ch = params.skip_vep + ? report_empty_slot() + : ch_somatic_vep_vcf.map { meta, vcf -> [meta.id, vcf] } + // report_vep_ch: [id, vcf] + + def report_sv_vep_ch = params.skip_vep + ? report_empty_slot() + : ch_sv_vep_vcf.map { meta, vcf -> [meta.id, vcf] } + // report_sv_vep_ch: [id, vcf] SEVERUS.out.somatic_vcf .map { meta, vcf -> [meta.id, vcf] } @@ -1292,29 +1302,63 @@ workflow LRSOMATIC { .map { meta, vcf, _tbi -> [meta.id, vcf] } .set { report_somatic_ch } - ch_ascat_files - .map { meta, files -> [meta.id, files] } - .set { report_ascat_ch } + def report_ascat_ch = params.skip_ascat + ? report_empty_slot() + : ch_ascat_files.map { meta, files -> [meta.id, files] } + // report_ascat_ch: [id, [files]] + + def report_wakhan_ch = params.skip_wakhan + ? report_empty_slot() + : ch_wakhan_files.map { meta, files -> [meta.id, files] } + // report_wakhan_ch: [id, [files]] - ch_wakhan_files - .map { meta, files -> [meta.id, files] } - .set { report_wakhan_ch } + // One emission per sample per tool, none optional: mosdepth 2, cramino 1, samtools 2. + // Adding another per-sample emission to either mix below must bump this count. + def qc_files_per_sample = params.skip_qc + ? 0 + : (params.skip_mosdepth ? 0 : 2) + (params.skip_cramino ? 0 : 1) + (params.skip_bamstats ? 0 : 2) // Tumor-side QC, keyed by the sample id (= report id) + // groupKey: emit a sample's bundle on its own files; toString() restores a plain String key ch_mosdepth_summary .mix(ch_mosdepth_global, ch_cramino_post_txt, ch_bam_stats, ch_bam_flagstat) .filter { meta, _f -> meta.type == 'tumor' } - .map { meta, f -> [meta.id, f] } + .map { meta, f -> [groupKey(meta.id, qc_files_per_sample), f] } .groupTuple() - .set { report_qc_tumor_ch } + .map { key, files -> [key.toString(), files] } + .set { report_qc_tumor_grouped } + // report_qc_tumor_grouped: [id, [qc_file, ...]] // Normal-side QC (matched mode): a pair shares meta.id, so already keyed by the report id ch_mosdepth_summary .mix(ch_mosdepth_global, ch_cramino_post_txt, ch_bam_stats, ch_bam_flagstat) .filter { meta, _f -> meta.type == 'normal' } - .map { meta, f -> [meta.id, f] } + .map { meta, f -> [groupKey(meta.id, qc_files_per_sample), f] } .groupTuple() - .set { report_qc_normal_ch } + .map { key, files -> [key.toString(), files] } + .set { report_qc_normal_grouped } + // report_qc_normal_grouped: [id, [qc_file, ...]] -- paired samples only + + def report_qc_tumor_ch = qc_files_per_sample == 0 + ? report_empty_slot() + : report_qc_tumor_grouped + + // Normal-side QC covers paired samples only; meta.paired_data gives the tumor-only arm + // its [] up front instead of waiting out channel close for a match that never arrives. + report_id_meta + .branch { _id, meta -> + paired: meta.paired_data + tumor_only: true + } + .set { report_roster } + + def report_qc_normal_ch = qc_files_per_sample == 0 + ? report_empty_slot() + : report_roster.paired + .join(report_qc_normal_grouped) + .map { id, _meta, files -> [id, files] } + .mix(report_roster.tumor_only.map { id, _meta -> [id, []] }) + // report_qc_normal_ch: [id, [files] | []] -- full roster report_id_meta .join(report_vep_ch, remainder: true) From 0a3024ac5e1be231da08f7dd07d770703c26b687 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 23 Sep 2026 12:03:24 +0200 Subject: [PATCH 14/25] Fill in the PR number on this branch's changelog entries Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64799a46..934f4923 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now runs [lrsomatic_report](https://github.com/ljwharbers/lrsomatic_report) v1.6.0 from `ghcr.io/ljwharbers/lrsomatic-report:1.6.0` (`oras://ghcr.io/ljwharbers/lrsomatic-report-sif:1.6.0` under Singularity/Apptainer) instead of the source vendored at `assets/lrsomatic_report`, which is deleted; updating the tool is now a tag bump. The image is `linux/amd64` only and **Conda is not supported for this step**: the module errors under `-profile conda`/`mamba`, `conda` is dropped from the nf-test CI matrix, and `--skip_report` gives a conda run (@ljwharbers). @@ -44,7 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#XXX](https://github.com/IntGenomicsLab/lrsomatic/pull/XXX) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3 `src/coverage/processing.py:121` averaged a BAF bin over a slice built from `.index()` lookups; because `.index()` searches by value the slice is empty whenever the position list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError`. The failure was deterministic, so retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3 `src/coverage/processing.py:121` averaged a BAF bin over a slice built from `.index()` lookups; because `.index()` searches by value the slice is empty whenever the position list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError`. The failure was deterministic, so retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). - [#193](https://github.com/IntGenomicsLab/lrsomatic/pull/193) - `--vep_eve https://evemodel.org/api/proteins/bulk/download/` was rejected at launch because the "needs preparing" check keyed on a `.zip` suffix; it now checks whether the value is already a prepared bgzipped file (@AmberVerhasselt). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs a patched modkit 0.6.4 ([ljwharbers/modkit@pacbio-conflict-fix](https://github.com/ljwharbers/modkit/tree/pacbio-conflict-fix)): `ghcr.io/ljwharbers/modkit:0.6.4-pacbiofix-6e0afa2` under Docker and `oras://ghcr.io/ljwharbers/modkit-sif:0.6.4-pacbiofix-6e0afa2` under Singularity/Apptainer. Stock modkit 0.4.3-0.6.4 dropped 32-65 % of reads from recent PacBio HiFi BAMs and returned empty `--cpg` pileups ([nanoporetech/modkit#612](https://github.com/nanoporetech/modkit/issues/612), fix proposed in [nanoporetech/modkit#720](https://github.com/nanoporetech/modkit/pull/720)), and ignored `--phased`/`--modified-bases` for PacBio BAMs with 6mA calls. The image is `linux/amd64` only and Conda is not supported (use `--skip_modkit` there); return to the biocontainer once a release includes the fix (@ljwharbers). From d5e610ebd30703777b1263ac968b6d74c05f0409 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Wed, 23 Sep 2026 13:03:18 +0200 Subject: [PATCH 15/25] Satisfy the pre-commit hooks: call the report slot closure explicitly Nextflow's strict syntax resolves `report_empty_slot()` against declared functions, not local closure variables, so it read as undefined. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- workflows/lrsomatic.nf | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c85f93e8..ce751704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the *other* caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#201](https://github.com/IntGenomicsLab/lrsomatic/pull/201) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now pull the fork image from Docker Hub: `oras://docker.io/ljwharbers/clairs-to-sif:0.5.1-verdict-chm13-c0687e8-flat` under Singularity/Apptainer and `docker.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8-flat` otherwise, instead of `ghcr.io/ljwharbers/clairs-to`. The `-cpu` SIF on ghcr failed with `PROTOCOL_ERROR` on slow links: ghcr redirects every blob download to an Azure URL that expires at the next 5-minute mark and resets a stream still open then, and Apptainer resumes neither an `oras://` nor a `docker://` download. Docker Hub's download URLs are valid for 50 minutes and only checked when the request starts. `-flat` is the same software copied into an empty image in a few layers (3.3 GB instead of 7 GB); the software and its outputs are unchanged. `docs/usage.md` describes `pullTimeout`, Docker Hub's anonymous pull limit, pre-pulling, and how to recover the remaining `oras://ghcr.io` SIFs resumably (@ljwharbers). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index 9704b5ed..acf98c0d 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -1285,12 +1285,12 @@ workflow LRSOMATIC { def report_empty_slot = { -> report_id_meta.map { id, _meta -> [id, []] } } def report_vep_ch = params.skip_vep - ? report_empty_slot() + ? report_empty_slot.call() : ch_somatic_vep_vcf.map { meta, vcf -> [meta.id, vcf] } // report_vep_ch: [id, vcf] def report_sv_vep_ch = params.skip_vep - ? report_empty_slot() + ? report_empty_slot.call() : ch_sv_vep_vcf.map { meta, vcf -> [meta.id, vcf] } // report_sv_vep_ch: [id, vcf] @@ -1303,12 +1303,12 @@ workflow LRSOMATIC { .set { report_somatic_ch } def report_ascat_ch = params.skip_ascat - ? report_empty_slot() + ? report_empty_slot.call() : ch_ascat_files.map { meta, files -> [meta.id, files] } // report_ascat_ch: [id, [files]] def report_wakhan_ch = params.skip_wakhan - ? report_empty_slot() + ? report_empty_slot.call() : ch_wakhan_files.map { meta, files -> [meta.id, files] } // report_wakhan_ch: [id, [files]] @@ -1340,7 +1340,7 @@ workflow LRSOMATIC { // report_qc_normal_grouped: [id, [qc_file, ...]] -- paired samples only def report_qc_tumor_ch = qc_files_per_sample == 0 - ? report_empty_slot() + ? report_empty_slot.call() : report_qc_tumor_grouped // Normal-side QC covers paired samples only; meta.paired_data gives the tumor-only arm @@ -1353,7 +1353,7 @@ workflow LRSOMATIC { .set { report_roster } def report_qc_normal_ch = qc_files_per_sample == 0 - ? report_empty_slot() + ? report_empty_slot.call() : report_roster.paired .join(report_qc_normal_grouped) .map { id, _meta, files -> [id, files] } From 27728f812ab733876e1804451cdc9046820ee8aa Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:33:33 +0200 Subject: [PATCH 16/25] Move the INFO/SOMATIC filter of local BCFTOOLS_VIEW into ext.args Drop the hard-coded -i from the module script and set it in the PHASING_HAPLOTYPING:BCFTOOLS_VIEW config block. Refresh meta.yml: drop the removed targets inputs, generic description, document versions output/topic. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 1 + modules/local/bcftools/view/main.nf | 1 - modules/local/bcftools/view/meta.yml | 35 +++++++++++++++++++--------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 9372bad1..ca462202 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -460,6 +460,7 @@ process { } withName: '.*:PHASING_HAPLOTYPING:BCFTOOLS_VIEW' { ext.prefix = { "somatic_smallvariants" } + ext.args = { "-i 'INFO/SOMATIC=1'" } publishDir = [ path: { "${params.outdir}/${meta.id}/variants/phased" }, mode: params.publish_dir_mode, diff --git a/modules/local/bcftools/view/main.nf b/modules/local/bcftools/view/main.nf index ae81084e..63905dd2 100644 --- a/modules/local/bcftools/view/main.nf +++ b/modules/local/bcftools/view/main.nf @@ -23,7 +23,6 @@ process BCFTOOLS_VIEW { def prefix = task.ext.prefix ?: "${meta.id}" """ bcftools view \\ - -i 'INFO/SOMATIC=1' \\ -Oz \\ -W=tbi \\ ${args} \\ diff --git a/modules/local/bcftools/view/meta.yml b/modules/local/bcftools/view/meta.yml index 28c0fc48..29e3646b 100644 --- a/modules/local/bcftools/view/meta.yml +++ b/modules/local/bcftools/view/meta.yml @@ -1,5 +1,5 @@ name: bcftools_view -description: Filter VCF to positions defined by a targets file using bcftools view -T +description: Filter a VCF with bcftools view, with the filter expression supplied via ext.args; outputs a bgzipped VCF and tbi index keywords: - filtering - VCF @@ -25,14 +25,6 @@ input: type: file description: Tabix index of the input VCF pattern: "*.tbi" - - targets: - type: file - description: VCF file used as position filter (-T) - pattern: "*.{vcf.gz,vcf,bcf}" - - targets_tbi: - type: file - description: Tabix index of the targets VCF - pattern: "*.tbi" output: vcf: - - meta: @@ -50,7 +42,28 @@ output: type: file description: Tabix index of filtered VCF pattern: "*.tbi" + versions_bcftools: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version | sed '1!d; s/^.*bcftools //'": + type: string + description: The command used to generate the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version | sed '1!d; s/^.*bcftools //'": + type: string + description: The command used to generate the version of the tool authors: - - "@rforsyth" + - "@robert-a-forsyth" maintainers: - - "@rforsyth" + - "@robert-a-forsyth" From c691a0cffa1a5f18ec1719fea697349ecb482570 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:35:21 +0200 Subject: [PATCH 17/25] Leave FILTER untouched in VCFTAG VCFTAG now only stamps the provenance flag, so --smallvar_filter_pass false keeps each caller's FILTER. ORIG_FILTER is stamped only on the ClairS-TO germline split, the one arm rewritten to PASS, and SIGNATURES_BCFTOOLS_VIEW goes back to --apply-filters PASS with --exclude 'ALT="*"'. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 +-- conf/modules.config | 10 ++----- docs/usage.md | 21 ++++++------- modules/local/vcfsplit/main.nf | 34 +++++---------------- modules/local/vcftag/main.nf | 25 ++-------------- modules/local/vcftag/tests/main.nf.test | 35 ++++++++++------------ subworkflows/local/phasing_haplotyping.nf | 4 +-- tests/fixtures/vcftag_input.vcf | 2 ++ tests/fixtures/vcftag_input.vcf.gz | Bin 505 -> 546 bytes tests/fixtures/vcftag_input.vcf.gz.tbi | Bin 112 -> 112 bytes 10 files changed, 42 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce751704..c810c559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` additionally records each record's original `FILTER` in `INFO/ORIG_FILTER` on both splits. Germline calls remain published in full (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` records each ClairS-TO germline record's original `FILTER` in `INFO/ORIG_FILTER` before rewriting it to `PASS`. Germline calls remain published in full (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. `VCFTAG` normalises `FILTER` to `PASS` before phasing, which would have made the later `SIGNATURES_BCFTOOLS_VIEW --apply-filters PASS` a no-op and let non-`PASS` records reach SigProfiler whenever the parameter was `false`; the caller's verdict is now preserved in `INFO/ORIG_FILTER` and the signature input is filtered on that instead, so `false` behaves as documented. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). - [#201](https://github.com/IntGenomicsLab/lrsomatic/pull/201) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now pull the fork image from Docker Hub: `oras://docker.io/ljwharbers/clairs-to-sif:0.5.1-verdict-chm13-c0687e8-flat` under Singularity/Apptainer and `docker.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8-flat` otherwise, instead of `ghcr.io/ljwharbers/clairs-to`. The `-cpu` SIF on ghcr failed with `PROTOCOL_ERROR` on slow links: ghcr redirects every blob download to an Azure URL that expires at the next 5-minute mark and resets a stream still open then, and Apptainer resumes neither an `oras://` nor a `docker://` download. Docker Hub's download URLs are valid for 50 minutes and only checked when the request starts. `-flat` is the same software copied into an empty image in a few layers (3.3 GB instead of 7 GB); the software and its outputs are unchanged. `docs/usage.md` describes `pullTimeout`, Docker Hub's anonymous pull limit, pre-pulling, and how to recover the remaining `oras://ghcr.io` SIFs resumably (@ljwharbers). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index ca462202..6cd1516f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -753,14 +753,8 @@ process { } withName : '.*:SIGNATURES_BCFTOOLS_VIEW' { - // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only. - // The PASS test reads INFO/ORIG_FILTER, not FILTER: VCFTAG normalises FILTER to PASS on - // every record before phasing, so --apply-filters PASS here could never remove anything and - // non-PASS records reached SigProfiler whenever --smallvar_filter_pass was false. - // ORIG_FILTER carries the caller's own verdict, so the filter bites again. "." is kept - // because an unfiltered record is not a rejected one. bcftools allows only one of -i/-e, - // so the ALT="*" exclusion is folded into the same expression. - ext.args = { "-i '(INFO/ORIG_FILTER=\"PASS\" || INFO/ORIG_FILTER=\".\") && ALT!=\"*\"' --types snps,mnps,indels --output-type v" } + // SigProfilerMatrixGenerator ignores FILTER and reads plain-text VCF only + ext.args = { "--apply-filters PASS --types snps,mnps,indels --exclude 'ALT=\"*\"' --output-type v" } ext.prefix = { "${meta.id}_somatic_pass" } publishDir = [ enabled: false diff --git a/docs/usage.md b/docs/usage.md index 00a0c28e..b9ab4571 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -425,13 +425,10 @@ rather than `PASS`-filtered. The per-caller VCFs published under `//variants/` are never filtered, so no calls are lost from the results directory. -Set it to `false` to restore the previous unfiltered behaviour. `VCFTAG` -normalises `FILTER` to `PASS` on both arms before phasing, so the caller's own -verdict is preserved in `INFO/ORIG_FILTER` and the signature input is filtered on -that field rather than on `FILTER`; without this the `PASS` filter in front of -SigProfiler could never remove anything once `FILTER` had been rewritten. Every -published VCF downstream of phasing therefore reads `FILTER=PASS`, with the -original value available in `INFO/ORIG_FILTER`. +Set it to `false` to restore the previous unfiltered behaviour: each caller's +records are passed on with their original `FILTER`. Only the ClairS-TO germline +split is normalised to `PASS`, with its original value kept in +`INFO/ORIG_FILTER`. `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only @@ -453,11 +450,11 @@ every record. Three INFO fields carry this provenance: -| Field | Meaning | -| ------------- | ---------------------------------------------------------------------------------------------------------------- | -| `SOMATIC` | Record came from the somatic call set | -| `GERMLINE` | Record came from the germline call set | -| `ORIG_FILTER` | The `FILTER` value in the ClairS-TO output, before normalisation to `PASS`. Multiple filters are joined with `,` | +| Field | Meaning | +| ------------- | --------------------------------------------------------------------------------------------------------------------- | +| `SOMATIC` | Record came from the somatic call set | +| `GERMLINE` | Record came from the germline call set | +| `ORIG_FILTER` | Original `FILTER` of ClairS-TO germline records, before normalisation to `PASS`. Multiple filters are joined with `,` | Germline calls dropped from `variants/phased/somatic_smallvariants.vcf.gz` are not lost: they remain in `variants/phased/germline_smallvariants.vcf.gz`, in diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index d773494a..2b627223 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -28,21 +28,7 @@ process VCFSPLIT { bcftools view -i 'FILTER="PASS"' $snv_vcf | bgzip -c > snv_pass.vcf.gz tabix -p vcf indels_pass.vcf.gz tabix -p vcf snv_pass.vcf.gz - bcftools concat -a -Oz -o somatic_tmp.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz - tabix -p vcf somatic_tmp.vcf.gz - - # Record the caller's original FILTER in INFO. These records are already PASS, but stamping - # both splits keeps them symmetric and self-describing alongside the germline arm below. - # The header line's double quotes arrive via -v q. Escaped quotes inside a Nextflow script - # block are fragile, and losing the escape silently produces an unparseable VCF header. - # FILTER is ";"-delimited but ";" separates INFO fields, so it is stored as ",". - bcftools view somatic_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} - /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } - { of = \$7; gsub(/;/, ",", of) - \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of - print } - ' | bgzip -c > somatic.vcf.gz + bcftools concat -a -Oz -o somatic.vcf.gz indels_pass.vcf.gz snv_pass.vcf.gz tabix -p vcf somatic.vcf.gz bcftools view -i 'FILTER~"NonSomatic" || INFO/Verdict_Germline=1' $indel_vcf | bgzip -c > indels_filtered.vcf.gz @@ -52,15 +38,11 @@ process VCFSPLIT { bcftools concat -a -Oz -o germline_tmp.vcf.gz indels_filtered.vcf.gz snv_filtered.vcf.gz tabix -p vcf germline_tmp.vcf.gz - # FILTER is normalised to PASS so that downstream tools which filter on PASS -- implicitly or - # otherwise -- see every germline record. The caller's original FILTER (typically NonSomatic) - # would otherwise be destroyed here, which is what previously made germline records - # indistinguishable from somatic ones once the two sets were merged for phasing; it is kept in - # INFO/ORIG_FILTER instead. Germline/somatic provenance itself is stamped later, by - # PHASING_HAPLOTYPING:TAG_GERMLINE / TAG_SOMATIC, which covers callers that bypass VCFSPLIT. + # Normalise FILTER to PASS and keep the original in INFO/ORIG_FILTER (";" stored as ","). + # The header's double quotes arrive via -v q, avoiding fragile escapes in the script block. bcftools view germline_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } - /^#CHROM/ { print "##INFO="; print; next } + /^#CHROM/ { print "##INFO="; print; next } { of = \$7; gsub(/;/, ",", of) \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of \$7 = "PASS" @@ -68,15 +50,13 @@ process VCFSPLIT { ' | bgzip -c > germline.vcf.gz tabix -p vcf germline.vcf.gz - # Read both headers back. tabix will happily index a VCF whose header htslib cannot parse, so - # without this a malformed header surfaces as a confusing failure in a later process instead - # of here. set -e is in effect, so a bad header fails this task. + # Fail here, not downstream, if either header does not parse. bcftools view -h somatic.vcf.gz > /dev/null bcftools view -h germline.vcf.gz > /dev/null # Cleanup intermediate files - rm indels_pass.vcf.gz snv_pass.vcf.gz somatic_tmp.vcf.gz - rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi somatic_tmp.vcf.gz.tbi + rm indels_pass.vcf.gz snv_pass.vcf.gz + rm indels_pass.vcf.gz.tbi snv_pass.vcf.gz.tbi """ stub: diff --git a/modules/local/vcftag/main.nf b/modules/local/vcftag/main.nf index a7d81868..390122c6 100644 --- a/modules/local/vcftag/main.nf +++ b/modules/local/vcftag/main.nf @@ -23,41 +23,22 @@ process VCFTAG { script: prefix = task.ext.prefix ?: "${meta.id}_${flag.toLowerCase()}" """ - # Stamp a constant INFO flag recording which arm this record came from, and normalise FILTER - # to PASS. Provenance has to live in INFO rather than FILTER: downstream steps rewrite and - # filter on FILTER, so a FILTER-based label does not survive to where it is needed. + # Stamp a constant INFO flag marking the call set; FILTER is left as the caller emitted it. # bcftools annotate cannot set a constant INFO field without an annotation file, hence awk. - # - # The caller's own FILTER is preserved in INFO/ORIG_FILTER before it is overwritten, so the - # verdict is not destroyed by the normalisation. FILTER is ";"-delimited but ";" separates INFO - # fields, so it is stored as ",". VCFSPLIT already stamps ORIG_FILTER on the ClairS-TO arm; both - # the header line and the per-record field are therefore added only when not already present, - # since a duplicate INFO key would make the record unparseable. - bcftools view ${vcf} | awk -v flag="${flag}" -v q='"' 'BEGIN{FS=OFS="\t"; seen_orig=0} - /^##INFO=" - if (!seen_orig) { - print "##INFO=" - } print next } { - if (\$8 !~ /(^|;)ORIG_FILTER=/) { - of = \$7 - gsub(/;/, ",", of) - \$8 = (\$8 == "." || \$8 == "") ? "ORIG_FILTER=" of : \$8 ";ORIG_FILTER=" of - } - \$7 = "PASS" \$8 = (\$8 == "." || \$8 == "") ? flag : \$8 ";" flag print } ' | bgzip -c > ${prefix}.vcf.gz - # tabix will index a VCF whose header htslib cannot parse, so read it back here rather than - # letting a malformed header surface as a confusing failure in a later process. + # Fail here, not downstream, if the header does not parse. bcftools view -h ${prefix}.vcf.gz > /dev/null tabix -p vcf ${prefix}.vcf.gz diff --git a/modules/local/vcftag/tests/main.nf.test b/modules/local/vcftag/tests/main.nf.test index 3f4a658f..c0ec94a6 100644 --- a/modules/local/vcftag/tests/main.nf.test +++ b/modules/local/vcftag/tests/main.nf.test @@ -10,7 +10,7 @@ nextflow_process { // Runs for real (no -stub): the tagging is an awk program embedded in the Nextflow script // block, so the escaping only holds if it is actually executed. - test("stamps the flag, declares its header and normalises FILTER") { + test("stamps the flag and declares its header without touching FILTER") { when { process { @@ -34,29 +34,24 @@ nextflow_process { def lines = all.findAll { line -> !line.startsWith('#') } def header = all.findAll { line -> line.startsWith('##') }.join('\n') + def filters = lines.collectEntries { l -> def f = l.split('\t'); [ (f[1]): f[6] ] } + def infos = lines.collectEntries { l -> def f = l.split('\t'); [ (f[1]): f[7] ] } + assertAll( // every input record survives -- tagging must not filter - { assert lines.size() == 4 }, + { assert lines.size() == 5 }, // the flag is declared, so bcftools can query it downstream { assert header.contains('ID=SOMATIC') }, - // every record carries the flag, including the one whose INFO was '.' - { assert lines.every { it.contains('SOMATIC') } }, - // FILTER is normalised to PASS on all records, including NonSomatic and RefCall - { assert lines.every { it.split('\t')[6] == 'PASS' } }, - // a pre-existing INFO field is preserved rather than overwritten - { assert lines.any { it.contains('EXISTING') && it.contains('CALLER=clairs-to') } }, - // the record whose INFO was '.' gets no leading '.', just the stamped fields - { assert lines.any { it.split('\t')[7] == 'ORIG_FILTER=RefCall;SOMATIC' } }, - // the caller's FILTER is preserved before being normalised away - { assert header.contains('ID=ORIG_FILTER') }, - { assert lines.every { it.contains('ORIG_FILTER=') } }, - { assert lines.any { it.contains('ORIG_FILTER=PASS') } }, - { assert lines.any { it.contains('ORIG_FILTER=NonSomatic') } }, - { assert lines.any { it.contains('ORIG_FILTER=RefCall') } }, - // a record whose FILTER was '.' records '.' rather than being skipped - { assert lines.any { it.contains('ORIG_FILTER=.') } }, - // ORIG_FILTER is stamped exactly once per record - { assert lines.every { it.split('\t')[7].count('ORIG_FILTER=') == 1 } } + // the flag is stamped exactly once per record, including the one whose INFO was '.' + { assert infos.values().every { it.split(';').count('SOMATIC') == 1 } }, + { assert infos['300'] == 'SOMATIC' }, + // FILTER is left exactly as the caller emitted it + { assert filters == [ '100':'PASS', '200':'NonSomatic', '300':'RefCall', '400':'.', '500':'LowQual;RefCall' ] }, + // pre-existing INFO is preserved, with the flag appended + { assert infos['200'] == 'CALLER=clairs-to;EXISTING;SOMATIC' }, + { assert infos['500'] == 'CALLER=clair3;EXISTING;SOMATIC' }, + // VCFTAG no longer adds ORIG_FILTER + { assert !all.any { it.contains('ORIG_FILTER') } } ) } } diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 3de385ef..40fb011e 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -144,8 +144,8 @@ workflow PHASING_HAPLOTYPING { // MODULE: VCFTAG (label: process_single), aliased TAG_SOMATIC / TAG_GERMLINE // Stamp each arm with an INFO provenance flag before the merge. This is the only point where // germline-vs-somatic origin is unambiguous for every caller: GERMLINE_CONSENSUS can emit - // records that never passed through VCFSPLIT, so tagging earlier would leave holes. After the - // merge the two populations are otherwise indistinguishable -- both carry FILTER=PASS. + // records that never passed through VCFSPLIT, so tagging earlier would leave holes. FILTER + // cannot tell the arms apart after the merge, so the flag carries provenance. // LongPhase preserves custom INFO keys, so the flags survive phasing (verified on v2.0.1). // TAG_SOMATIC ( somatic_vcf, 'SOMATIC' ) diff --git a/tests/fixtures/vcftag_input.vcf b/tests/fixtures/vcftag_input.vcf index 34fda4cc..a1945635 100644 --- a/tests/fixtures/vcftag_input.vcf +++ b/tests/fixtures/vcftag_input.vcf @@ -2,6 +2,7 @@ ##INFO= ##INFO= ##FILTER= +##FILTER= ##FILTER= ##contig= #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT testsample @@ -9,3 +10,4 @@ chr1 100 . A C 30 PASS CALLER=clairs-to GT:DP 0/1:20 chr1 200 . G T 20 NonSomatic CALLER=clairs-to;EXISTING GT:DP 0/1:18 chr1 300 . T A 10 RefCall . GT:DP 0/0:12 chr1 400 . C G 40 . CALLER=deepsomatic GT:DP 1/1:25 +chr1 500 . G A 5 LowQual;RefCall CALLER=clair3;EXISTING GT:DP 0/1:9 diff --git a/tests/fixtures/vcftag_input.vcf.gz b/tests/fixtures/vcftag_input.vcf.gz index 988819c22c88bc153d91397e8db80263c9cbe483..c956c3c5d4ce5c621969ebe1ab7248a6926ccd8f 100644 GIT binary patch literal 546 zcmb2|=3rp}f&Xj_PR>jW?u@Y)4(2sE2(Uf){7Xc7=c<)Q%yNT#IGR))S2*~t*XG;t za+7ztyKesd2Gf;uo*Deyf5v`VXc6PmOQz20&sm=zJtG$xCc)nMS!$ik`CcXaxkr{) z@!y*1|7jbey7Spo)5u0mll%$VFMG;;iq$5H`EXo#{Iveaf1b@YJ=e{jJv_zI#5=+5 zut?nwM?LK*uCT9u0&Y9!e+ycZvMj4@&kp`$UVdjw9naTKS6HoOaiP{*TFo&aH0b76 zk+c@0YjFYZ7YWl`v{yF%@Q#)IPn(GGpG~WLH5~TLGPDk}{ecSJ= U?1oZT&Pri%y)*+eI8lHI0I~Ds^Z)<= literal 505 zcmb2|=3rp}f&Xj_PR>jWCXBHc0<)V91lT`3{}SBtE9{_W^?fZ6S z+fkQE(UJTAUdY?k-QWv4HmA{{%kZ*k5#{^G^y?=~%1eqZH8YXkG$`I+ZUHL{{s zwcAhK9qOH-yzYwWPR861clbT6iWyu#YJ6QT-L}|IDd6RbRxzCyY|j?ol;{oWNh{vM z9p5Nck#g|U$Cv*SC0SN?itn9rblM&<{|5TzIE<&hk1xcXQ_i2S;UPlT%R{p`otT--6BcUf0b|oU_Md!IY&7=E{q;aZGA# zR=D!{^11#z>A&-lz`W};9Sw1slNjynB(jR?) h|F!eEB^Ifge7nt;pScx%4H&2B?v!R=2FEps008J@+w1@U diff --git a/tests/fixtures/vcftag_input.vcf.gz.tbi b/tests/fixtures/vcftag_input.vcf.gz.tbi index 3e8c3efd859874550e714bc5ac9aa7e101d7798f..75130768821ae4097ed26672d63124118d62a5d6 100644 GIT binary patch delta 34 qcmXRYm|!OLfN`P3jRUXtsvR_0)V0}Y0aM6c0S1PU?tIRP0SW*L?G51o delta 34 qcmXRYm|!NAz_n1~T~p?IPlMhSk92reFof!}Ffc^zjTE04pa1~jl?&_u From 75036a0e7e44db77439b08b0d88e1b1345c6a586 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:37:24 +0200 Subject: [PATCH 18/25] Prefilter DS_VERDICT_QUERY to the non-PASS/RefCall DeepSomatic rows Only GERMLINE/PON verdicts are consumed downstream, so the verdict table no longer carries the ~9M RefCall and PASS rows (13.4M -> 4.3M on B1975944). The filter excludes PASS/RefCall rather than including GERMLINE/PON, because bcftools errors on a FILTER ID the header does not declare. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 11 +++++------ subworkflows/local/tumor_only/tumoronly_smallvar.nf | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 6cd1516f..c1031053 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -168,10 +168,10 @@ process { // alias, so they do not disturb the generic BCFTOOLS_QUERY/ANNOTATE/VIEW entries below. withName: '.*:DS_VERDICT_QUERY' { ext.prefix = { "${meta.id}.ds_verdict" } - // FILTER is single-valued in DeepSomatic output (RefCall/GERMLINE/PON/PASS), so it cannot - // inject the ";" that would break INFO parsing once transferred. + // Keep only non-PASS/RefCall rows (GERMLINE/PON in practice). Excludes rather than includes, + // since naming an undeclared FILTER (PON is absent without a PON) makes bcftools fail. ext.args = { - "-f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" + "-e 'FILTER=\"PASS\" || FILTER=\"RefCall\"' -f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" } publishDir = [ enabled: false @@ -181,7 +181,7 @@ process { withName: '.*:DS_VERDICT_ANNOTATE' { ext.prefix = { "${meta.id}.deepvariant_verdict" } ext.args = { - '''-h <(echo '##INFO=') \ + '''-h <(echo '##INFO=') \ -c CHROM,POS,REF,ALT,INFO/DS_VERDICT \ -Oz \ -W=tbi''' @@ -193,8 +193,7 @@ process { withName: '.*:DS_GERMLINE_SELECT' { ext.prefix = { "${meta.id}.deepvariant_germline" } - // Keep only positively-adjudicated germline sites. RefCall and sites DeepSomatic never - // evaluated (DS_VERDICT absent) are dropped rather than assumed germline. + // Keep only DeepSomatic-adjudicated germline sites; all others lack DS_VERDICT or fail this test. ext.args = { "-i 'INFO/DS_VERDICT=\"GERMLINE\" || INFO/DS_VERDICT=\"PON\"' --output-type z --write-index=tbi" } diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index cb89b849..fefab476 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -231,8 +231,8 @@ workflow TUMORONLY_SMALLVAR { // (57,684) PASS -- the last being real somatic calls that must not be published as germline. // // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and - // unevaluated sites are dropped rather than assumed germline. The verdict stays in - // INFO/DS_VERDICT so the decision is auditable in the published VCF. + // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts + // are transferred to INFO/DS_VERDICT. // // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would @@ -240,7 +240,7 @@ workflow TUMORONLY_SMALLVAR { // // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter - // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER + // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER, non-PASS/RefCall rows only // DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) From 18023f2bbad64f05939bdfacb13d240cd3d6e865 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:39:10 +0200 Subject: [PATCH 19/25] Rejoin split multi-allelics after the caller consensus BCFTOOLS_NORM splits multi-allelics (-m -any) so isec can match per ALT, which left two records at one POS in the VCF handed to LongPhase and Wakhan. BCFTOOLS_NORM_REJOIN (-m +any) now rejoins them at the end of SMALL_VARIANT_CONSENSUS on both the consensus and 'all' paths. The final file keeps its _{germline,somatic}_sorted name; the sort steps now write *_split_sorted. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 20 ++++++++++--- subworkflows/local/small_variant_consensus.nf | 28 +++++++++++++++++-- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c1031053..a71c5c53 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -126,6 +126,7 @@ process { // A>G never intersects: in consensus mode the variant is dropped, in union mode it appears // twice. ClairS-TO emits no multi-allelic records at all while DeepVariant emits ~2.5%, so // every DeepVariant multi-allelic site was systematically excluded from the consensus. + // BCFTOOLS_NORM_REJOIN rejoins the split records after isec, before phasing. ext.args = { "-m -any -Oz" } @@ -133,6 +134,17 @@ process { enabled: false ] } + // Rejoin split multi-allelics; LongPhase keys variants by position and cannot take two records at one POS. + withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_NORM_REJOIN' { + ext.prefix = { "${meta.id}_germline_sorted" } + ext.args = { '-m +any --output-type z --write-index=tbi' } + publishDir = [ enabled: false ] + } + withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_NORM_REJOIN' { + ext.prefix = { "${meta.id}_somatic_sorted" } + ext.args = { '-m +any --output-type z --write-index=tbi' } + publishDir = [ enabled: false ] + } withName: '.*:BCFTOOLS_ISEC' { ext.prefix = { "${meta.id}_isec" } @@ -553,26 +565,26 @@ process { ] } withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_SORT' { - ext.prefix = { "${meta.id}_germline_sorted" } + ext.prefix = { "${meta.id}_germline_split_sorted" } ext.args = {'-Oz -W=tbi'} publishDir = [ enabled: false ] } withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_SORT' { - ext.prefix = { "${meta.id}_somatic_sorted" } + ext.prefix = { "${meta.id}_somatic_split_sorted" } ext.args = {'-Oz -W=tbi'} publishDir = [ enabled: false ] } withName: '.*:GERMLINE_CONSENSUS:BCFTOOLS_SORT_CONSENSUS' { - ext.prefix = { "${meta.id}_germline_sorted" } + ext.prefix = { "${meta.id}_germline_split_sorted" } ext.args = { '-Oz -W=tbi' } publishDir = [ enabled: false ] } withName: '.*:SOMATIC_CONSENSUS:BCFTOOLS_SORT_CONSENSUS' { - ext.prefix = { "${meta.id}_somatic_sorted" } + ext.prefix = { "${meta.id}_somatic_split_sorted" } ext.args = { '-Oz -W=tbi' } publishDir = [ enabled: false ] } diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index d50ba7f7..4717cb69 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -1,4 +1,5 @@ include { BCFTOOLS_NORM } from '../../modules/nf-core/bcftools/norm/main' +include { BCFTOOLS_NORM as BCFTOOLS_NORM_REJOIN } from '../../modules/nf-core/bcftools/norm/main' include { BCFTOOLS_ISEC } from '../../modules/nf-core/bcftools/isec/main' include { BCFTOOLS_QUERY } from '../../modules/nf-core/bcftools/query/main' include { BCFTOOLS_ANNOTATE } from '../../modules/nf-core/bcftools/annotate/main' @@ -21,7 +22,7 @@ workflow SMALL_VARIANT_CONSENSUS { main: // - // MODULE: BCFTOOLS_NORM (label: process_medium) -- left-align and normalise; sorted after, since left-alignment can reorder records + // MODULE: BCFTOOLS_NORM (label: process_medium) -- left-align and split multi-allelics for isec; rejoined before phasing // Input: [meta, vcf, tbi] -- per-caller VCF // Output: .vcf -- [meta, vcf] -- left-aligned, normalised VCF (unsorted) // @@ -54,6 +55,8 @@ workflow SMALL_VARIANT_CONSENSUS { // Renaming cannot fix that and `annotate -h` cannot override an existing FORMAT definition, // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record // reaching here carries one ALT and one AF value, making the two declarations equivalent. + // After BCFTOOLS_NORM_REJOIN, consensus files keep the source caller's own header (lossless round + // trip); 'all' mode takes the Number=A header of isec's 0000.vcf.gz, which concat lists first. // // The rename is carried out by BCFTOOLS_ANNOTATE below rather than by a second annotate call: // meta.rename_to selects the --rename-annots file in conf/modules.config. Only 'all' mode needs @@ -266,8 +269,27 @@ workflow SMALL_VARIANT_CONSENSUS { error("combine_method must be 'consensus' or 'all', got '${combine_method}'") } + // + // MODULE: BCFTOOLS_NORM_REJOIN (BCFTOOLS_NORM alias) -- rejoin split sites (-m +any) so LongPhase and Wakhan see one record per position + // Input: [meta, vcf, tbi] -- sorted consensus/union VCF + // Output: .vcf -- [meta, vcf.gz] + // .tbi -- [meta, tbi] + // + BCFTOOLS_NORM_REJOIN( + vcf.join(tbi, failOnMismatch: true, failOnDuplicate: true), + fasta + ) + + BCFTOOLS_NORM_REJOIN.out.vcf + .join(BCFTOOLS_NORM_REJOIN.out.tbi, failOnMismatch: true, failOnDuplicate: true) + .multiMap { meta, rejoined_vcf, rejoined_tbi -> + vcf: [meta, rejoined_vcf] + tbi: [meta, rejoined_tbi] + } + .set { rejoined } + emit: - vcf // [meta, vcf] -- final consensus/combined VCF - tbi // [meta, tbi] + vcf = rejoined.vcf // [meta, vcf] -- final consensus/combined VCF, multi-allelics rejoined + tbi = rejoined.tbi // [meta, tbi] } From 6a3e297208bb874fe60dcce485400c5bfdd0bf07 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:41:27 +0200 Subject: [PATCH 20/25] Join ASCAT and Wakhan report files instead of counting with groupKey groupKey(meta, 3) counted optional outputs, so a sample missing one was silently dropped at channel close. ASCAT now joins its required outputs and adds segments_raw with remainder: true. Wakhan always writes a solution_* link for every row of solutions_ranks.tsv, so solution_dirs is now required and the three outputs are joined. Co-Authored-By: Claude Opus 5.5 (1M context) --- modules/local/wakhan/main.nf | 2 +- workflows/lrsomatic.nf | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/local/wakhan/main.nf b/modules/local/wakhan/main.nf index b461baaf..517ae995 100644 --- a/modules/local/wakhan/main.nf +++ b/modules/local/wakhan/main.nf @@ -38,7 +38,7 @@ process WAKHAN { tuple val(meta), path("solutions_ranks.tsv") , emit: solutions_ranks // Whole directories, not the plots inside: every solution's plot has the same basename, // and LRSOMATICREPORT resolves them by solution_/ path - tuple val(meta), path("solution_*", type: 'dir') , emit: solution_dirs, optional: true + tuple val(meta), path("solution_*", type: 'dir') , emit: solution_dirs // WARN: Manually update version information as tool does not provide on CLI tuple val("${task.process}"), val('wakhan'), val("0.4.4"), topic: versions, emit: versions_wakhan diff --git a/workflows/lrsomatic.nf b/workflows/lrsomatic.nf index acf98c0d..03517069 100644 --- a/workflows/lrsomatic.nf +++ b/workflows/lrsomatic.nf @@ -696,12 +696,11 @@ workflow LRSOMATIC { // ascat_tumoronly_ch: [meta, purityploidy, segments] // All ASCAT files per sample for the report module, which globs by suffix - // groupKey: release each sample on its own three emissions, not when ASCAT finishes for all - ch_ascat_files = ASCAT.out.segments_raw - .mix(ASCAT.out.purityploidy, ASCAT.out.png) - .map { meta, files -> [groupKey(meta, 3), files] } - .groupTuple() - .map { meta, files -> [meta, files.flatten()] } + // Joined per sample; segments_raw is optional, so it arrives as null when absent + ch_ascat_files = ASCAT.out.purityploidy + .join(ASCAT.out.png) + .join(ASCAT.out.segments_raw, remainder: true) + .map { meta, purityploidy, png, segments_raw -> [meta, [purityploidy, png, segments_raw ?: []].flatten()] } // ch_ascat_files: [meta, [file, file, ...]] } @@ -1257,12 +1256,11 @@ workflow LRSOMATIC { ) // The WAKHAN outputs the report renders: ranked solutions, heatmap, per-solution plots - // groupKey: release each sample on its own three emissions + // Joined per sample; all three outputs are required ch_wakhan_files = WAKHAN.out.solutions_ranks - .mix(WAKHAN.out.heatmap_html, WAKHAN.out.solution_dirs) - .map { meta, files -> [groupKey(meta, 3), files] } - .groupTuple() - .map { meta, files -> [meta, files.flatten()] } // solution_dirs contributes a list + .join(WAKHAN.out.heatmap_html) + .join(WAKHAN.out.solution_dirs) + .map { meta, ranks, heatmap, dirs -> [meta, [ranks, heatmap, dirs].flatten()] } // dirs may be a list // ch_wakhan_files: [meta, [file_or_dir, ...]] } From d653bcbd7a6eb19ecc8629efb88f5b32a2c51fa4 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:43:32 +0200 Subject: [PATCH 21/25] Inline the PASS filter as BCFTOOLS_VIEW aliases at each caller Drop the one-module VCF_PASS_FILTER subworkflow; each call site now picks between the filtered copy and its input on params.smallvar_filter_pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- conf/modules.config | 42 ++++--------------- .../local/paired/paired_smallvar_germline.nf | 34 ++++++++------- .../local/paired/paired_smallvar_somatic.nf | 34 ++++++++------- .../local/tumor_only/tumoronly_smallvar.nf | 38 ++++++++--------- subworkflows/local/vcf_pass_filter.nf | 41 ------------------ tests/union.nf.test | 5 +-- 6 files changed, 63 insertions(+), 131 deletions(-) delete mode 100644 subworkflows/local/vcf_pass_filter.nf diff --git a/conf/modules.config b/conf/modules.config index a71c5c53..58d2116f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -772,47 +772,19 @@ process { ] } - // VCF_PASS_FILTER copies, one alias per small variant caller. DeepVariant and - // DeepSomatic emit every evaluated site rather than only their calls, and Clair3/ClairS - // keep their LowQual and NonSomatic records; these filtered copies are what the - // consensus, phasing, VEP and report steps consume. The per-caller VCFs under - // variants// are published from the caller processes and are left untouched. - // ClairS-TO has no entry here: VCFSPLIT already restricts its SOMATIC split to PASS. - // Note this is not true of the germline split, whose FILTER is rewritten to PASS rather - // than filtered; germline/somatic origin is carried in INFO by VCFTAG instead. - // --write-index is required because the module's index output is optional, and the - // downstream join() would otherwise drop every sample. - withName: '.*:DEEPVARIANT_PASS_FILTER:PASS_FILTER' { + // PASS-only per-caller copies fed to consensus/phasing/VEP; ClairS-TO is covered by VCFSPLIT. + // --write-index=tbi is required: the module's index output is optional and the join needs it. + withName: '.*:(CLAIR3|CLAIRS|DEEPVARIANT|DEEPSOMATIC)_PASS_FILTER' { ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_deepvariant_pass" } publishDir = [ enabled: false ] } - withName: '.*:DEEPSOMATIC_PASS_FILTER:PASS_FILTER' { - ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_deepsomatic_pass" } - publishDir = [ - enabled: false - ] - } - - withName: '.*:CLAIR3_PASS_FILTER:PASS_FILTER' { - ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_clair3_pass" } - publishDir = [ - enabled: false - ] - } - - withName: '.*:CLAIRS_PASS_FILTER:PASS_FILTER' { - ext.args = '--apply-filters PASS --output-type z --write-index=tbi' - ext.prefix = { "${meta.id}_clairs_pass" } - publishDir = [ - enabled: false - ] - } + withName: '.*:CLAIR3_PASS_FILTER' { ext.prefix = { "${meta.id}_clair3_pass" } } + withName: '.*:CLAIRS_PASS_FILTER' { ext.prefix = { "${meta.id}_clairs_pass" } } + withName: '.*:DEEPVARIANT_PASS_FILTER' { ext.prefix = { "${meta.id}_deepvariant_pass" } } + withName: '.*:DEEPSOMATIC_PASS_FILTER' { ext.prefix = { "${meta.id}_deepsomatic_pass" } } withName : '.*:SIGPROFILER_MATRIXGENERATOR' { ext.args = { params.sigprofiler_matrix_args ?: '' } diff --git a/subworkflows/local/paired/paired_smallvar_germline.nf b/subworkflows/local/paired/paired_smallvar_germline.nf index 798e5b3f..3164e33b 100644 --- a/subworkflows/local/paired/paired_smallvar_germline.nf +++ b/subworkflows/local/paired/paired_smallvar_germline.nf @@ -1,11 +1,11 @@ // IMPORT MODULES +include { BCFTOOLS_VIEW as CLAIR3_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' include { CLAIR3 } from '../../../modules/local/clair3/main.nf' // IMPORT SUBWORKFLOWS include { DEEPVARIANT } from '../../../subworkflows/nf-core/deepvariant/main.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' -include { VCF_PASS_FILTER as CLAIR3_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' -include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_GERMLINE { @@ -75,14 +75,15 @@ workflow PAIRED_SMALLVAR_GERMLINE { fai ) - // Clair3's merge_output.vcf.gz keeps LowQual and RefCall records; restrict them the - // same way DeepVariant's output is restricted so both callers enter the union on - // equal terms. The VCF published under variants/clair3/ is unaffected. - CLAIR3_PASS_FILTER ( - CLAIR3.out.vcf.join(CLAIR3.out.tbi) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def clair3_vcf = CLAIR3.out.vcf.join(CLAIR3.out.tbi) + if (params.smallvar_filter_pass) { + CLAIR3_PASS_FILTER ( clair3_vcf, [], [], [] ) + clair3_vcf = CLAIR3_PASS_FILTER.out.vcf + .join(CLAIR3_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - CLAIR3_PASS_FILTER.out.vcf + clair3_vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clair3'] return [new_meta, vcf, tbi] @@ -127,14 +128,15 @@ workflow PAIRED_SMALLVAR_GERMLINE { [[:],[]] // GFF annotation (not used) ) - // DeepVariant emits a record for every site it evaluates, not just its calls, so - // most records are RefCall. Without this the union is "every site DeepVariant - // looked at". The VCF published under variants/deepvariant/ is unaffected. - DEEPVARIANT_PASS_FILTER ( - DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepvariant_vcf = DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_vcf, [], [], [] ) + deepvariant_vcf = DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DEEPVARIANT_PASS_FILTER.out.vcf + deepvariant_vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/paired/paired_smallvar_somatic.nf b/subworkflows/local/paired/paired_smallvar_somatic.nf index 953ed01d..ab296b99 100644 --- a/subworkflows/local/paired/paired_smallvar_somatic.nf +++ b/subworkflows/local/paired/paired_smallvar_somatic.nf @@ -1,4 +1,6 @@ // IMPORT MODULES +include { BCFTOOLS_VIEW as CLAIRS_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' include { CLAIRS } from '../../../modules/local/clairs/main.nf' include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat' include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort' @@ -6,8 +8,6 @@ include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/so // IMPORT SUBWORKFLOWS include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' -include { VCF_PASS_FILTER as CLAIRS_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' -include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' workflow PAIRED_SMALLVAR_SOMATIC { @@ -73,14 +73,15 @@ workflow PAIRED_SMALLVAR_SOMATIC { BCFTOOLS_CONCAT.out.vcf ) - // The concatenated ClairS VCF still carries its LowQual and NonSomatic calls; - // restrict them the same way DeepSomatic's output is restricted so both callers - // enter the union on equal terms. variants/clairs/ is unaffected. - CLAIRS_PASS_FILTER ( - BCFTOOLS_SORT.out.vcf.join(BCFTOOLS_SORT.out.tbi) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def clairs_vcf = BCFTOOLS_SORT.out.vcf.join(BCFTOOLS_SORT.out.tbi) + if (params.smallvar_filter_pass) { + CLAIRS_PASS_FILTER ( clairs_vcf, [], [], [] ) + clairs_vcf = CLAIRS_PASS_FILTER.out.vcf + .join(CLAIRS_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - CLAIRS_PASS_FILTER.out.vcf + clairs_vcf .map { meta, vcf , tbi -> def new_meta = meta + [caller:'clairs'] return [new_meta, vcf, tbi] @@ -117,14 +118,15 @@ workflow PAIRED_SMALLVAR_SOMATIC { ds_pon_channel ) - // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), - // not just its calls. Without this the union is "every site DeepSomatic looked - // at". The VCF published under variants/deepsomatic/ is unaffected. - DEEPSOMATIC_PASS_FILTER ( - DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepsomatic_vcf = DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_vcf, [], [], [] ) + deepsomatic_vcf = DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DEEPSOMATIC_PASS_FILTER.out.vcf + deepsomatic_vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index fefab476..e4f90f5c 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -1,4 +1,6 @@ // IMPORT MODULES +include { BCFTOOLS_VIEW as DEEPVARIANT_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as DEEPSOMATIC_PASS_FILTER } from '../../../modules/nf-core/bcftools/view/main' include { CLAIRSTO } from '../../../modules/local/clairsto/main.nf' include { CLAIRSTO_VERDICT_TAG } from '../../../modules/local/clairsto/verdict_tag/main.nf' include { VCFSPLIT } from '../../../modules/local/vcfsplit/main.nf' @@ -8,8 +10,6 @@ include { DEEPVARIANT } from '../../../subwork include { DEEPSOMATIC } from '../../../subworkflows/local/deepsomatic.nf' include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' -include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' -include { VCF_PASS_FILTER as DEEPSOMATIC_PASS_FILTER } from '../../../subworkflows/local/vcf_pass_filter.nf' // Germline verdict transfer: DeepSomatic adjudicates DeepVariant's tumor-derived germline calls. // Three independent bcftools invocations, so three aliased instances of the upstream modules. @@ -165,15 +165,15 @@ workflow TUMORONLY_SMALLVAR { [[:],[]], // GZI (empty if FASTA is uncompressed) ds_pon_channel ) - // DeepSomatic emits a record for every site it evaluates (RefCall/GERMLINE/PON), - // not just its calls. ClairS-TO needs no equivalent step here because VCFSPLIT - // already restricts it to PASS. The VCF published under variants/deepsomatic/ is - // unaffected. - DEEPSOMATIC_PASS_FILTER ( - DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepsomatic_vcf = DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPSOMATIC_PASS_FILTER ( deepsomatic_vcf, [], [], [] ) + deepsomatic_vcf = DEEPSOMATIC_PASS_FILTER.out.vcf + .join(DEEPSOMATIC_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DEEPSOMATIC_PASS_FILTER.out.vcf + deepsomatic_vcf .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepsomatic'] return [new_meta, vcf, tbi] @@ -208,15 +208,13 @@ workflow TUMORONLY_SMALLVAR { [[:],[]] // GFF annotation (not used) ) - // DeepVariant emits a record for every site it evaluates, not just its calls, so - // most records are RefCall. ClairS-TO needs no equivalent step here because - // VCFSPLIT already restricts its SOMATIC split to PASS -- note that its GERMLINE split - // is not PASS-filtered but PASS-rewritten, so a PASS filter would not reduce it and - // germline/somatic origin is carried in INFO by VCFTAG instead. The VCF published under - // variants/deepvariant/ is unaffected. - DEEPVARIANT_PASS_FILTER ( - DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) - ) + // PASS-only copy for downstream steps; published VCFs are untouched. + def deepvariant_vcf = DEEPVARIANT.out.vcf.join(DEEPVARIANT.out.vcf_index) + if (params.smallvar_filter_pass) { + DEEPVARIANT_PASS_FILTER ( deepvariant_vcf, [], [], [] ) + deepvariant_vcf = DEEPVARIANT_PASS_FILTER.out.vcf + .join(DEEPVARIANT_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) + } // GERMLINE VERDICT TRANSFER (tumor-only, deep family) // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is @@ -248,7 +246,7 @@ workflow TUMORONLY_SMALLVAR { // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. // - DEEPVARIANT_PASS_FILTER.out.vcf + deepvariant_vcf .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) .map { meta, vcf, tbi, annotations, annotations_index -> diff --git a/subworkflows/local/vcf_pass_filter.nf b/subworkflows/local/vcf_pass_filter.nf deleted file mode 100644 index f52e0cb9..00000000 --- a/subworkflows/local/vcf_pass_filter.nf +++ /dev/null @@ -1,41 +0,0 @@ -// IMPORT MODULES -// Aliased so the software-versions report keeps this separate from the local -// BCFTOOLS_VIEW used by PHASING_HAPLOTYPING, which is a different bcftools build. -include { BCFTOOLS_VIEW as PASS_FILTER } from '../../modules/nf-core/bcftools/view/main' - -// -// SUBWORKFLOW: VCF_PASS_FILTER -// Restrict a per-caller VCF to its PASS records before it is handed to the caller -// consensus, phasing, VEP and the report. DeepVariant and DeepSomatic emit a record for -// every site they evaluate (RefCall/GERMLINE/PON), and Clair3/ClairS keep their LowQual -// and NonSomatic calls, so without this the union is "every site every caller looked at". -// The per-caller VCFs published under variants// are produced elsewhere and are -// never filtered, so no calls are lost from the results directory. -// -// Include once per caller under an alias so conf/modules.config can give each one its own -// prefix, e.g. include { VCF_PASS_FILTER as DEEPVARIANT_PASS_FILTER }. -// -workflow VCF_PASS_FILTER { - - take: - vcfs // [meta, vcf, tbi] - - main: - if (params.smallvar_filter_pass) { - // The module declares `emit: index, optional: true`, so the index only exists because - // conf/modules.config puts --write-index=tbi in ext.args for every alias. failOnMismatch - // turns a missing index into an immediate error instead of silently dropping the sample. - PASS_FILTER ( vcfs, [], [], [] ) - - PASS_FILTER.out.vcf - .join(PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) - .set{ filtered } - } - else { - vcfs - .set{ filtered } - } - - emit: - vcf = filtered // [meta, vcf, tbi] -} diff --git a/tests/union.nf.test b/tests/union.nf.test index aa14800b..8acb4956 100644 --- a/tests/union.nf.test +++ b/tests/union.nf.test @@ -134,12 +134,11 @@ nextflow_pipeline { { assert workflow.success }, // ── No PASS filter process may run ─────────────────────────── - // VCF_PASS_FILTER passes its input straight through when the param is - // false, so PASS_FILTER never executes and never reports a version. + // With the param false no _PASS_FILTER process runs or reports a version. { def versions = file("$outputDir/pipeline_info/lrsomatic_software_mqc_versions.yml") assert versions.exists() - assert !versions.text.contains('PASS_FILTER') + assert !versions.text.contains('_PASS_FILTER') }, // ── Phased VCFs still exist and have data ──────────────────── From c91a23e70fd4ca9b82d4291ca9e2deaf5cc64ac9 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:45:10 +0200 Subject: [PATCH 22/25] Add VCFTAG meta.yml and fix var_keep defaults in usage.md Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/usage.md | 4 +- modules/local/vcftag/meta.yml | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 modules/local/vcftag/meta.yml diff --git a/docs/usage.md b/docs/usage.md index b9ab4571..afe0e4ee 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -401,8 +401,8 @@ These options control how variants from multiple callers are filtered and merged | Parameter | Description | | ------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| `--germline_var_keep` | Expression or threshold for retaining germline variants after calling. Default = `null` | -| `--somatic_var_keep` | Expression or threshold for retaining somatic variants after calling. Default = `null` | +| `--germline_var_keep` | Comma-separated germline callers to run: `deepvariant`, `clair`. Default = `clair` | +| `--somatic_var_keep` | Comma-separated somatic callers to run: `deepsomatic`, `clair`. Default = `clair` | | `--germline_var_combine` | How to combine germline caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | | `--somatic_var_combine` | How to combine somatic caller outputs: `consensus` (shared calls only) or `all` (union). Default = `all` | | `--prioritize_caller_germline` | Whose record to use for variants called by both germline callers: `deepvariant` or `clair`. Default = `clair` | diff --git a/modules/local/vcftag/meta.yml b/modules/local/vcftag/meta.yml new file mode 100644 index 00000000..648ca689 --- /dev/null +++ b/modules/local/vcftag/meta.yml @@ -0,0 +1,75 @@ +name: vcftag +description: Stamp a constant INFO flag, named by the flag input, on every record of a VCF; FILTER is left untouched +keywords: + - vcf + - annotation + - INFO + - variant calling +tools: + - bcftools: + description: Tools for variant calling and manipulating VCFs and BCFs + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + tool_dev_url: https://github.com/samtools/bcftools + doi: "10.1093/gigascience/giab008" + licence: ["MIT"] + identifier: biotools:bcftools +input: + - - meta: + type: map + description: Groovy Map containing sample information e.g. [ id:'test' ] + - vcf: + type: file + description: Input VCF file to tag + pattern: "*.vcf.gz" + - tbi: + type: file + description: Tabix index of the input VCF + pattern: "*.tbi" + - flag: + type: string + description: | + Name of the INFO flag added to every record (e.g. `GERMLINE`); + its lowercase form is also used in the default output prefix +output: + vcf: + - - meta: + type: map + description: Groovy Map containing sample information + - ${prefix}.vcf.gz: + type: file + description: VCF with the INFO flag added to every record + pattern: "*.vcf.gz" + tbi: + - - meta: + type: map + description: Groovy Map containing sample information + - ${prefix}.vcf.gz.tbi: + type: file + description: Tabix index of the tagged VCF + pattern: "*.vcf.gz.tbi" + versions_bcftools: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version |& sed '1!d ; s/bcftools //'": + type: string + description: The command used to generate the version of the tool +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - bcftools: + type: string + description: The tool name + - "bcftools --version |& sed '1!d ; s/bcftools //'": + type: string + description: The command used to generate the version of the tool +authors: + - "@robert-a-forsyth" +maintainers: + - "@robert-a-forsyth" From 5fae9328ba7a34cd02a1c091813342cd08c0a687 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:46:56 +0200 Subject: [PATCH 23/25] Allow deepvariant without deepsomatic; skip the verdict transfer then Tumour-only DeepVariant germline calls are adjudicated with DeepSomatic's verdict only when deepsomatic is selected; otherwise they pass through. Drops validateDeepCallerPairing, which rejected valid paired-only runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- docs/usage.md | 8 +- .../local/tumor_only/tumoronly_smallvar.nf | 108 +++++++++--------- .../utils_nfcore_lrsomatic_pipeline/main.nf | 24 ---- 4 files changed, 62 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c810c559..4fded0de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. `deepvariant` and `deepsomatic` must now be enabled together and a mismatched pair is rejected at launch. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. The verdict is applied only when `deepsomatic` is also selected; with `deepvariant` alone the germline calls are used unfiltered by verdict and may include somatic variants. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` records each ClairS-TO germline record's original `FILTER` in `INFO/ORIG_FILTER` before rewriting it to `PASS`. Germline calls remain published in full (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). - [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). diff --git a/docs/usage.md b/docs/usage.md index afe0e4ee..0c686ce9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -138,7 +138,7 @@ For tumour-only small variants, ClairS-TO separates germline from somatic calls When `--germline_var_keep` includes `deepvariant`, the tumour-only germline arm runs DeepVariant on the **tumour** BAM. DeepVariant is a germline caller with no somatic discrimination, so on its own those calls mix germline and clonal somatic -variants. The pipeline therefore transfers DeepSomatic's verdict onto them: +variants. When DeepSomatic also runs, the pipeline transfers its verdict onto them: DeepSomatic evaluates the same sites and labels each `GERMLINE`, `PON`, `RefCall` or `PASS`, and that label is recorded in `INFO/DS_VERDICT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; @@ -146,9 +146,9 @@ adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; germline. On a 30x tumour-only sample this keeps about 83% of DeepVariant's `PASS` calls and removes roughly 1% that DeepSomatic positively calls somatic. -Because of this, `deepvariant` and `deepsomatic` must be enabled together: -`--germline_var_keep deepvariant` without `deepsomatic` in `--somatic_var_keep` is -rejected at launch. Note that even after adjudication the tumour-only germline arm +This restriction to sites DeepSomatic calls `GERMLINE` or `PON` applies only when +`deepsomatic` is also in `--somatic_var_keep`; without it, the DeepVariant germline +calls are used without any verdict filter and may include somatic variants. Note that even after adjudication the tumour-only germline arm is a tumour-derived proxy, not a call set from normal tissue, and should not be used for secondary findings without that caveat. diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index e4f90f5c..5ec14050 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -216,66 +216,72 @@ workflow TUMORONLY_SMALLVAR { .join(DEEPVARIANT_PASS_FILTER.out.index, failOnMismatch: true, failOnDuplicate: true) } - // GERMLINE VERDICT TRANSFER (tumor-only, deep family) - // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is - // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its - // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the - // germline VCF therefore carries most of the somatic call set. - // - // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic - // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO - // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 - // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% - // (57,684) PASS -- the last being real somatic calls that must not be published as germline. - // - // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and - // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts - // are transferred to INFO/DS_VERDICT. - // - // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified - // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would - // break INFO parsing. - // - // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) - // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter - // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER, non-PASS/RefCall rows only - // - DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) + // Keep only DeepSomatic-adjudicated germline sites; skipped when deepsomatic isn't selected. + def deepvariant_germline = deepvariant_vcf + if (somatic_var_keep.contains('deepsomatic')) { + // GERMLINE VERDICT TRANSFER (tumor-only, deep family) + // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is + // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its + // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the + // germline VCF therefore carries most of the somatic call set. + // + // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic + // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO + // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 + // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% + // (57,684) PASS -- the last being real somatic calls that must not be published as germline. + // + // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and + // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts + // are transferred to INFO/DS_VERDICT. + // + // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified + // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would + // break INFO parsing. + // + // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) + // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter + // Output: .output/.index -- [meta, tsv.gz/tbi] -- CHROM POS REF ALT FILTER, non-PASS/RefCall rows only + // + DS_VERDICT_QUERY ( DEEPSOMATIC.out.vcf.join(DEEPSOMATIC.out.vcf_index), [], [], [] ) - // - // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) - // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. - // - deepvariant_vcf - .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) - .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) - .map { meta, vcf, tbi, annotations, annotations_index -> - def columns = [] // no extra column specs - def header_lines = [] // no extra header lines - def rename_chrs = [] // no chromosome renaming - return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] - } - .set{ ds_verdict_annotate_input } + // + // MODULE: DS_VERDICT_ANNOTATE (BCFTOOLS_ANNOTATE alias, label: process_medium) + // Stamps INFO/DS_VERDICT on each DeepVariant record from the DeepSomatic verdict table. + // + deepvariant_vcf + .join(DS_VERDICT_QUERY.out.output, failOnMismatch: true, failOnDuplicate: true) + .join(DS_VERDICT_QUERY.out.index, failOnMismatch: true, failOnDuplicate: true) + .map { meta, vcf, tbi, annotations, annotations_index -> + def columns = [] // no extra column specs + def header_lines = [] // no extra header lines + def rename_chrs = [] // no chromosome renaming + return [ meta, vcf, tbi, annotations, annotations_index, columns, header_lines, rename_chrs ] + } + .set{ ds_verdict_annotate_input } - DS_VERDICT_ANNOTATE ( ds_verdict_annotate_input ) + DS_VERDICT_ANNOTATE ( ds_verdict_annotate_input ) - // - // MODULE: DS_GERMLINE_SELECT (BCFTOOLS_VIEW alias, label: process_medium) - // Keeps only the positively-adjudicated germline records (see ext.args in conf/modules.config). - // - DS_GERMLINE_SELECT ( - DS_VERDICT_ANNOTATE.out.vcf.join(DS_VERDICT_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true), - [], [], [] - ) + // + // MODULE: DS_GERMLINE_SELECT (BCFTOOLS_VIEW alias, label: process_medium) + // Keeps only the positively-adjudicated germline records (see ext.args in conf/modules.config). + // + DS_GERMLINE_SELECT ( + DS_VERDICT_ANNOTATE.out.vcf.join(DS_VERDICT_ANNOTATE.out.tbi, failOnMismatch: true, failOnDuplicate: true), + [], [], [] + ) + + deepvariant_germline = DS_GERMLINE_SELECT.out.vcf + .join(DS_GERMLINE_SELECT.out.index, failOnMismatch: true, failOnDuplicate: true) + } - DS_GERMLINE_SELECT.out.vcf - .join(DS_GERMLINE_SELECT.out.index, failOnMismatch: true, failOnDuplicate: true) + deepvariant_germline .map{ meta, vcf, tbi -> def new_meta = meta + [caller:'deepvariant'] return [new_meta, vcf, tbi] } .set{deepvariant_ch} - // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] -- germline-adjudicated only + // deepvariant_ch: [meta(+caller:'deepvariant'), vcf, tbi] -- germline-adjudicated if deepsomatic ran } // COMBINE GERMLINE VARIANTS diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index 072a9d5f..a64c405c 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -239,30 +239,6 @@ workflow PIPELINE_COMPLETION { def validateInputParameters() { genomeExistsError() validateReportGenePanels() - validateDeepCallerPairing() -} - -// -// The tumor-only germline arm runs DeepVariant on the TUMOR BAM, which cannot separate germline -// from clonal somatic calls on its own. TUMORONLY_SMALLVAR resolves that by transferring -// DeepSomatic's FILTER verdict (GERMLINE/PON/RefCall/PASS) onto those calls, so DeepVariant -// without DeepSomatic would leave the germline arm unadjudicated -- and would reference -// DEEPSOMATIC.out before it exists. Require the deep family to be fully on or fully off. -// -def callerList(value) { - if (value instanceof List) { - return value - } - return value.toString().tokenize(',').collect { token -> token.trim() } -} - -def validateDeepCallerPairing() { - def germline = callerList(params.germline_var_keep) - def somatic = callerList(params.somatic_var_keep) - - if (germline.contains('deepvariant') && !somatic.contains('deepsomatic')) { - error("--germline_var_keep includes 'deepvariant' but --somatic_var_keep does not include 'deepsomatic'. In tumor-only mode the DeepVariant germline calls are adjudicated with DeepSomatic's verdict, so the two must be enabled together. Add 'deepsomatic' to --somatic_var_keep, or drop 'deepvariant' from --germline_var_keep.") - } } // From 00a9312d32cacd5881ea3246af145bd8e62e2ea0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 11:50:44 +0200 Subject: [PATCH 24/25] Cut long comments, sample numbers and CHANGELOG write-ups to one or two lines Addresses review point 9 on #200. Comments in the consensus, verdict transfer, phasing, VCFSPLIT and modules.config blocks now say what each step does in a line or two; sample IDs and cohort counts are removed from source and docs. The #200 CHANGELOG entries are rewritten to match the length of the others, with short Fixed lines for the multi-allelic rejoin and the per-sample report. No code changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +++-- conf/modules.config | 27 ++++------- docs/usage.md | 48 +++++++------------ modules/local/vcfsplit/main.nf | 3 +- subworkflows/local/phasing_haplotyping.nf | 16 ++----- subworkflows/local/small_variant_consensus.nf | 33 +++---------- .../local/tumor_only/tumoronly_smallvar.nf | 21 +------- 7 files changed, 44 insertions(+), 116 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fded0de..51bfabd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The tumour-only germline arm is now adjudicated with DeepSomatic's verdict instead of being published as-is. With `--germline_var_keep deepvariant` the germline calls come from DeepVariant run on the **tumour** BAM; DeepVariant has no somatic discrimination (its FILTER vocabulary is only `PASS`/`RefCall`/`LowQual`/`NoCall`), so those calls mix germline and clonal somatic variants and most of the somatic call set was being published as germline. DeepSomatic evaluates the same sites and does emit a verdict -- `FILTER=GERMLINE` ("Non somatic variants"), `PON`, `RefCall` or `PASS` -- which is now transferred onto the DeepVariant records as `INFO/DS_VERDICT`, mirroring how ClairS-TO adjudicates its own calls through `NonSomatic` and `VCFSPLIT`. Only positively adjudicated germline sites (`GERMLINE` or `PON`) are kept; `RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed germline. On B1975944 DeepVariant's 5,058,527 `PASS` calls resolve to 77.8% `GERMLINE`, 11.4% `RefCall`, 5.3% `PON`, 4.3% unevaluated and 1.14% (57,684) `PASS`, so the germline arm loses about 17% of its records including the 57,684 that DeepSomatic positively calls somatic. The verdict is applied only when `deepsomatic` is also selected; with `deepvariant` alone the germline calls are used unfiltered by verdict and may include somatic variants. The arm remains a tumour-derived proxy rather than a call set from normal tissue (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The somatic arm is now recovered from the phased germline+somatic VCF by provenance rather than by position. `PHASING_HAPLOTYPING:BCFTOOLS_VIEW` previously used the somatic VCF as a `-T` targets file, which matches on `CHROM`/`POS` only, so germline records sitting at a somatic coordinate were retained; on B2037620 it removed none of the 13,708,100 germline records. A new `VCFTAG` module (aliased `TAG_SOMATIC`/`TAG_GERMLINE`) stamps `INFO/SOMATIC` and `INFO/GERMLINE` immediately before the arms are merged for phasing -- the only point at which origin is unambiguous for every caller -- and the somatic arm is selected with `-i 'INFO/SOMATIC=1'`. `VCFSPLIT` records each ClairS-TO germline record's original `FILTER` in `INFO/ORIG_FILTER` before rewriting it to `PASS`. Germline calls remain published in full (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now produces the union it is documented to produce. Both branches of `SMALL_VARIANT_CONSENSUS` concatenated the shared record from the prioritized caller with the private calls of only the _other_ caller, so the prioritized caller's own private calls were always discarded: with the default `prioritize_caller_somatic = 'clair'` that silently dropped ClairS-TO's private calls (99 on B2037620). Both private sets are now kept alongside the shared record. Invalid `combine_method`/`prioritize_caller` values now raise a clear error instead of leaving the output channel undefined (@robert-a-forsyth). -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Small variant caller output is now restricted to `PASS` records before it is used downstream, controlled by the new `--smallvar_filter_pass` parameter (`true` by default). DeepVariant and DeepSomatic emit a record for every site they evaluate: on a 30x PacBio tumour sample the somatic VCF held 13,684,025 records of which 49,957 were `PASS`. With the default `*_var_combine = 'all'` the union was effectively "every site every caller looked at" -- `variants/phased/somatic_smallvariants.vcf.gz` reached 27,368,030 records for ~4.1 M unique SNVs, a coding TMB around 406 mut/Mb, and `LRSOMATICREPORT` could not render it at all. DeepVariant, DeepSomatic, Clair3 and ClairS now each pass through a `PASS`-only copy before the caller consensus, phasing, VEP and the report see them; ClairS-TO is unchanged because `VCFSPLIT` already restricted it to `PASS`. The per-caller VCFs under `variants//` are unchanged and remain unfiltered (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - In tumour-only mode with both `deepvariant` and `deepsomatic` selected, DeepVariant's germline calls are now kept only where DeepSomatic's verdict is `GERMLINE` or `PON` (`INFO/DS_VERDICT`); with `deepvariant` alone they are unfiltered and may include somatic variants (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The phased somatic VCF is now split from the merged germline+somatic VCF by an `INFO/SOMATIC` provenance flag (new `VCFTAG` module) instead of by position, so germline records at somatic coordinates no longer leak into it; ClairS-TO germline records keep their original `FILTER` in `INFO/ORIG_FILTER` (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Added `--smallvar_filter_pass` (default `true`), which restricts each small variant caller's output to `PASS` records before the caller consensus, phasing, VEP and the report; the published per-caller VCFs are unchanged (@robert-a-forsyth). - [#201](https://github.com/IntGenomicsLab/lrsomatic/pull/201) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now pull the fork image from Docker Hub: `oras://docker.io/ljwharbers/clairs-to-sif:0.5.1-verdict-chm13-c0687e8-flat` under Singularity/Apptainer and `docker.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8-flat` otherwise, instead of `ghcr.io/ljwharbers/clairs-to`. The `-cpu` SIF on ghcr failed with `PROTOCOL_ERROR` on slow links: ghcr redirects every blob download to an Azure URL that expires at the next 5-minute mark and resets a stream still open then, and Apptainer resumes neither an `oras://` nor a `docker://` download. Docker Hub's download URLs are valid for 50 minutes and only checked when the request starts. `-flat` is the same software copied into an empty image in a few layers (3.3 GB instead of 7 GB); the software and its outputs are unchanged. `docs/usage.md` describes `pullTimeout`, Docker Hub's anonymous pull limit, pre-pulling, and how to recover the remaining `oras://ghcr.io` SIFs resumably (@ljwharbers). - [#199](https://github.com/IntGenomicsLab/lrsomatic/pull/199) - `CLAIRSTO` and `CLAIRSTO_VERDICT_TAG` now run the `-cpu` rebuild of the fork image (`0.5.1-verdict-chm13-c0687e8-cpu`), which swaps PyTorch's CUDA build for the CPU build of the same version. The software is otherwise unchanged, but the Apptainer SIF drops from 6.53 GB to 3.46 GB. The old image could not be pulled on a normal VSC link: Apptainer fetches an `oras://` SIF as a single unresumable stream, and the signed blob URL ghcr redirects to expires on a 15-minute wall-clock boundary, so 6.53 GB needed 7.3 MB/s sustained and was otherwise cut mid-transfer with `PROTOCOL_ERROR` (@ljwharbers). - [#197](https://github.com/IntGenomicsLab/lrsomatic/pull/197) - `CLAIRSTO` now runs `ghcr.io/ljwharbers/clairs-to:0.5.1-verdict-chm13-c0687e8` (ClairS-TO 0.5.1) instead of `docker.io/hkubal/clairs-to:v0.4.2`: a fork that lets Verdict read its CNA resources from `--cna_resource_dir`, fixes four places where Verdict's Python port of ASCAT departed from R, and disables Verdict with a warning when its resources cannot be read. **GRCh38 results move as well as CHM13 ones.** Revert to the upstream image once HKU-BAL/ClairS-TO carries these changes. The module also selects the SIF under `-profile apptainer` and sets explicit output prefixes (@ljwharbers). @@ -45,7 +44,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, which fixes a crash that aborted the whole run. In 0.4.3 `src/coverage/processing.py:121` averaged a BAF bin over a slice built from `.index()` lookups; because `.index()` searches by value the slice is empty whenever the position list is not strictly ascending, and `statistics.mean([])` raises `StatisticsError`. The failure was deterministic, so retries could not clear it, and since `report_input_ch` joins on the Wakhan channel a single failed sample suppressed `LRSOMATICREPORT` for the entire batch (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `*_var_combine = 'all'` now keeps both callers' private calls; the prioritized caller's own private calls were previously dropped. Invalid `combine_method`/`prioritize_caller` values now raise an error (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - The caller consensus now splits multi-allelic records before intersecting, so they match across callers, and rejoins them before phasing (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - `LRSOMATICREPORT` now renders each sample as soon as its own inputs are ready instead of waiting for the whole batch, including samples without ASCAT's optional raw segments output (@robert-a-forsyth). +- [#200](https://github.com/IntGenomicsLab/lrsomatic/pull/200) - Bumped `WAKHAN` from 0.4.3 to 0.4.4, fixing a deterministic `StatisticsError` crash in BAF binning that aborted the run (@robert-a-forsyth). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). - [#193](https://github.com/IntGenomicsLab/lrsomatic/pull/193) - `--vep_eve https://evemodel.org/api/proteins/bulk/download/` was rejected at launch because the "needs preparing" check keyed on a `.zip` suffix; it now checks whether the value is already a prepared bgzipped file (@AmberVerhasselt). - [#188](https://github.com/IntGenomicsLab/lrsomatic/pull/188) - `MODKIT_PILEUP` now runs a patched modkit 0.6.4 ([ljwharbers/modkit@pacbio-conflict-fix](https://github.com/ljwharbers/modkit/tree/pacbio-conflict-fix)): `ghcr.io/ljwharbers/modkit:0.6.4-pacbiofix-6e0afa2` under Docker and `oras://ghcr.io/ljwharbers/modkit-sif:0.6.4-pacbiofix-6e0afa2` under Singularity/Apptainer. Stock modkit 0.4.3-0.6.4 dropped 32-65 % of reads from recent PacBio HiFi BAMs and returned empty `--cpg` pileups ([nanoporetech/modkit#612](https://github.com/nanoporetech/modkit/issues/612), fix proposed in [nanoporetech/modkit#720](https://github.com/nanoporetech/modkit/pull/720)), and ignored `--phased`/`--modified-bases` for PacBio BAMs with 6mA calls. The image is `linux/amd64` only and Conda is not supported (use `--skip_modkit` there); return to the biocontainer once a release includes the fix (@ljwharbers). diff --git a/conf/modules.config b/conf/modules.config index 58d2116f..c87d5dff 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -121,12 +121,8 @@ process { withName: '.*:BCFTOOLS_NORM' { ext.prefix = { "${meta.id}.${meta.caller}_norm" } - // -m -any splits multi-allelic records into biallelic ones. BCFTOOLS_ISEC matches on exact - // CHROM/POS/REF/ALT, so without this a site one caller reports as A>G,GT and the other as - // A>G never intersects: in consensus mode the variant is dropped, in union mode it appears - // twice. ClairS-TO emits no multi-allelic records at all while DeepVariant emits ~2.5%, so - // every DeepVariant multi-allelic site was systematically excluded from the consensus. - // BCFTOOLS_NORM_REJOIN rejoins the split records after isec, before phasing. + // Split multi-allelics so BCFTOOLS_ISEC, which matches exact REF/ALT, can intersect them per ALT. + // BCFTOOLS_NORM_REJOIN rejoins them before phasing. ext.args = { "-m -any -Oz" } @@ -157,10 +153,7 @@ process { } withName: '.*:BCFTOOLS_ANNOTATE' { ext.prefix = { "${meta.id}.${meta.caller}" } - // Stamps INFO/CALLER and, in 'all' mode, unifies the allele frequency key in the same - // invocation: --rename-annots composes with -a/-c/-h, so this does not need a second - // bcftools annotate pass. meta.rename_to is set only when SMALL_VARIANT_CONSENSUS is - // merging two callers into one VCF. + // Stamp INFO/CALLER; in 'all' mode (meta.rename_to set) also unify the FORMAT allele frequency key. ext.args = { def rename = meta.rename_to == 'VAF' ? "--rename-annots <(printf 'FORMAT/AF\\tFORMAT/VAF\\n') " @@ -176,12 +169,10 @@ process { enabled: false ] } - // GERMLINE VERDICT TRANSFER (tumor-only deep family). These three selectors are matched on the - // alias, so they do not disturb the generic BCFTOOLS_QUERY/ANNOTATE/VIEW entries below. + // GERMLINE VERDICT TRANSFER (tumor-only): DeepSomatic's verdict filters DeepVariant's germline calls. withName: '.*:DS_VERDICT_QUERY' { ext.prefix = { "${meta.id}.ds_verdict" } - // Keep only non-PASS/RefCall rows (GERMLINE/PON in practice). Excludes rather than includes, - // since naming an undeclared FILTER (PON is absent without a PON) makes bcftools fail. + // Exclude PASS/RefCall rather than include GERMLINE/PON: bcftools fails on an undeclared FILTER. ext.args = { "-e 'FILTER=\"PASS\" || FILTER=\"RefCall\"' -f '%CHROM\t%POS\t%REF\t%ALT\t%FILTER\n'" } @@ -454,9 +445,7 @@ process { enabled: false ] } - // VCFTAG stamps an INFO provenance flag on each arm before germline and somatic are merged - // for somatic phasing, so PHASING_HAPLOTYPING:BCFTOOLS_VIEW can select the somatic arm by - // origin instead of by position. Intermediates only; the published VCFs come from Longphase. + // Provenance flags stamped before the phasing merge, so the somatic arm is selected by origin. withName: '.*:TAG_SOMATIC' { ext.prefix = { "${meta.id}_somatic_tagged" } publishDir = [ @@ -772,8 +761,8 @@ process { ] } - // PASS-only per-caller copies fed to consensus/phasing/VEP; ClairS-TO is covered by VCFSPLIT. - // --write-index=tbi is required: the module's index output is optional and the join needs it. + // PASS-only per-caller copies for downstream steps; ClairS-TO is covered by VCFSPLIT. + // --write-index=tbi: the module's index output is optional and the join needs it. withName: '.*:(CLAIR3|CLAIRS|DEEPVARIANT|DEEPSOMATIC)_PASS_FILTER' { ext.args = '--apply-filters PASS --output-type z --write-index=tbi' publishDir = [ diff --git a/docs/usage.md b/docs/usage.md index 0c686ce9..427a12a8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -136,21 +136,14 @@ For structural variants, the CHM13 panel of normals is a merged panel combining For tumour-only small variants, ClairS-TO separates germline from somatic calls with a panel of normals and with its Verdict module, which tags each call as germline, somatic or subclonal somatic from tumour purity and allele-specific copy number. `--genome CHM13` supplies five CHM13 PON VCFs (gnomAD, dbSNP, 1000 Genomes, CoLoRSdb and ASAP), which **replace** the GRCh38 databases inside the container. Unless `--skip_ascat` is set, purity and copy number come from the pipeline's own ASCAT run (`CLAIRSTO_VERDICT_TAG`); only with `--skip_ascat` does ClairS-TO estimate them itself, from assembly-specific loci, allele and GC content files. A GRCh38 resource set on a CHM13 run leaves germline variants untagged. When `--germline_var_keep` includes `deepvariant`, the tumour-only germline arm -runs DeepVariant on the **tumour** BAM. DeepVariant is a germline caller with no -somatic discrimination, so on its own those calls mix germline and clonal somatic -variants. When DeepSomatic also runs, the pipeline transfers its verdict onto them: -DeepSomatic evaluates the same sites and labels each `GERMLINE`, `PON`, `RefCall` -or `PASS`, and that label is recorded in `INFO/DS_VERDICT`. Only positively -adjudicated germline sites (`GERMLINE` or `PON`) are kept in the germline arm; -`RefCall` and sites DeepSomatic never evaluated are dropped rather than assumed -germline. On a 30x tumour-only sample this keeps about 83% of DeepVariant's -`PASS` calls and removes roughly 1% that DeepSomatic positively calls somatic. - -This restriction to sites DeepSomatic calls `GERMLINE` or `PON` applies only when -`deepsomatic` is also in `--somatic_var_keep`; without it, the DeepVariant germline -calls are used without any verdict filter and may include somatic variants. Note that even after adjudication the tumour-only germline arm -is a tumour-derived proxy, not a call set from normal tissue, and should not be -used for secondary findings without that caveat. +runs DeepVariant on the **tumour** BAM, so on its own those calls mix germline and +clonal somatic variants. When `deepsomatic` is also in `--somatic_var_keep`, the +pipeline records DeepSomatic's verdict at each site in `INFO/DS_VERDICT` and keeps +only sites it calls `GERMLINE` or `PON`; `RefCall` and sites DeepSomatic never +evaluated are dropped. Without `deepsomatic`, the DeepVariant germline calls are +used without a verdict filter and may include somatic variants. Either way the +tumour-only germline arm is a tumour-derived proxy, not a call set from normal +tissue. With `--genome CHM13 --skip_ascat` the pipeline builds a CHM13 resource set from the ASCAT files it already downloads, so no extra setup is needed. LogR correction is GC-only, as ClairS-TO recommends for CHM13: no replication timing file is published for the assembly. Without `--skip_ascat` nothing is built, because the tagging comes from ASCAT's own tables. @@ -410,12 +403,10 @@ These options control how variants from multiple callers are filtered and merged | `--smallvar_filter_pass` | Keep only PASS records from each small variant caller downstream. Default = `true` | DeepVariant and DeepSomatic emit a record for every site they evaluate, not only -for the variants they call: on a 30x PacBio tumour sample a DeepSomatic VCF holds -around 13.7 M records of which roughly 50 k are `PASS`, the rest being `RefCall`, -`GERMLINE` or `PON`. Clair3 and ClairS are far less extreme but still keep their -`LowQual` and `NonSomatic` records. With `*_var_combine = 'all'` the union would -otherwise be "every site every caller looked at", which inflates the phased VCFs -by three orders of magnitude and produces a meaningless mutation burden. +for the variants they call, so most of their records are `RefCall`, `GERMLINE` or +`PON`; Clair3 and ClairS also keep their `LowQual` and `NonSomatic` records. Without +a `PASS` filter, `*_var_combine = 'all'` would carry all of these into the phased +VCFs and the mutation burden. `--smallvar_filter_pass` (`true` by default) restricts the copy of each caller's VCF that is handed to the caller consensus, phasing, VEP and the report. In @@ -433,20 +424,17 @@ split is normalised to `PASS`, with its original value kept in `consensus` keeps only variants called by both callers; `all` keeps the union, i.e. every variant called by either. In both modes `--prioritize_caller_*` chooses only whose record represents a variant that both callers found -- it never decides which -variants are kept. +variants are kept. Multi-allelic records are split so they can be matched across +callers, and rejoined before phasing. #### Germline and somatic provenance Germline and somatic small variants are merged into one VCF for somatic phasing, because Longphase needs all variant sites in a single file to produce consistent -phase blocks. The somatic arm is then recovered from the phased result. - -That recovery selects on an `INFO/SOMATIC` flag stamped on each arm before the merge, -not on position. A positional restriction cannot separate the two populations: a -germline record at the same coordinate as a somatic call is indistinguishable from -it, and `FILTER` is no help either, since `VCFSPLIT` normalises the ClairS-TO -germline split to `PASS` so that downstream tools which filter on `PASS` still see -every record. +phase blocks. The somatic arm is then recovered from the phased result by an +`INFO/SOMATIC` flag stamped on each arm before the merge, not by position, since a +germline record at the same coordinate as a somatic call would otherwise be kept. +Tagging leaves `FILTER` unchanged. Three INFO fields carry this provenance: diff --git a/modules/local/vcfsplit/main.nf b/modules/local/vcfsplit/main.nf index 2b627223..55cb6300 100644 --- a/modules/local/vcfsplit/main.nf +++ b/modules/local/vcfsplit/main.nf @@ -38,8 +38,7 @@ process VCFSPLIT { bcftools concat -a -Oz -o germline_tmp.vcf.gz indels_filtered.vcf.gz snv_filtered.vcf.gz tabix -p vcf germline_tmp.vcf.gz - # Normalise FILTER to PASS and keep the original in INFO/ORIG_FILTER (";" stored as ","). - # The header's double quotes arrive via -v q, avoiding fragile escapes in the script block. + # Normalise FILTER to PASS, keeping the original in INFO/ORIG_FILTER (";" stored as ","). bcftools view germline_tmp.vcf.gz | awk -v q='"' 'BEGIN{FS=OFS="\t"} /^##/ { print; next } /^#CHROM/ { print "##INFO="; print; next } diff --git a/subworkflows/local/phasing_haplotyping.nf b/subworkflows/local/phasing_haplotyping.nf index 40fb011e..0f101586 100644 --- a/subworkflows/local/phasing_haplotyping.nf +++ b/subworkflows/local/phasing_haplotyping.nf @@ -142,11 +142,7 @@ workflow PHASING_HAPLOTYPING { // // MODULE: VCFTAG (label: process_single), aliased TAG_SOMATIC / TAG_GERMLINE - // Stamp each arm with an INFO provenance flag before the merge. This is the only point where - // germline-vs-somatic origin is unambiguous for every caller: GERMLINE_CONSENSUS can emit - // records that never passed through VCFSPLIT, so tagging earlier would leave holes. FILTER - // cannot tell the arms apart after the merge, so the flag carries provenance. - // LongPhase preserves custom INFO keys, so the flags survive phasing (verified on v2.0.1). + // Stamp each arm with an INFO provenance flag before the merge; LongPhase keeps it through phasing. // TAG_SOMATIC ( somatic_vcf, 'SOMATIC' ) TAG_GERMLINE( germline_vcf, 'GERMLINE' ) @@ -163,9 +159,7 @@ workflow PHASING_HAPLOTYPING { tagged_germline_vcf .join(tagged_somatic_vcf) .map { meta, germ_vcf, germ_tbi, som_vcf, som_tbi -> - // Order here is cosmetic: BCFTOOLS_CONCAT sorts its input file list alphabetically - // (modules/nf-core/bcftools/concat/main.nf), so the germline file is passed first - // regardless. With -a the output is coordinate-ordered either way. + // Order is cosmetic: BCFTOOLS_CONCAT sorts its inputs by name. def vcfs = [som_vcf, germ_vcf] def tbis = [som_tbi, germ_tbi] return [ meta, vcfs, tbis] @@ -278,11 +272,7 @@ workflow PHASING_HAPLOTYPING { // // MODULE: BCFTOOLS_VIEW (label: process_medium) - // Reduce the phased somatic+germline VCF to the somatic arm, selecting on the INFO/SOMATIC flag - // stamped before the merge -- by provenance, not position. The previous `-T ` - // targets file matched CHROM/POS only, so germline records co-located with a somatic call were - // retained and became indistinguishable downstream. PS/HP tags on somatic variants survive; - // germline records are dropped here but stay published under variants/phased/ and vep/germline/. + // Keep the somatic arm by its INFO/SOMATIC flag, not by position; PS/HP tags survive. // Input: [meta, phased_combined_vcf, phased_combined_tbi] // Output: .vcf -- [meta, vcf.gz] -- phased somatic-only VCF // .tbi -- [meta, tbi] diff --git a/subworkflows/local/small_variant_consensus.nf b/subworkflows/local/small_variant_consensus.nf index 4717cb69..6444a492 100644 --- a/subworkflows/local/small_variant_consensus.nf +++ b/subworkflows/local/small_variant_consensus.nf @@ -45,23 +45,7 @@ workflow SMALL_VARIANT_CONSENSUS { // ALLELE FREQUENCY KEY -- BCFTOOLS_ANNOTATE below renames the AF FORMAT field to the priority caller's: // FORMAT/AF -> FORMAT/VAF when prioritize_caller is 'deepvariant'/'deepsomatic' // FORMAT/VAF -> FORMAT/AF when prioritize_caller is 'clair' - // This guarantees the merged VCF exposes allele frequency under a single FORMAT key, which is - // what WAKHAN consumes. Every caller currently emits FORMAT/AF and none emits VAF (verified - // against Clair3, ClairS-TO, DeepVariant and DeepSomatic output), so under the default - // prioritize_caller='clair' it is a no-op; it is kept as the guarantee, not the mechanism. - // - // The callers do disagree on the AF *declaration*: ClairS-TO says Number=1, DeepVariant and - // DeepSomatic say Number=A. bcftools concat only warns and keeps the first file's definition. - // Renaming cannot fix that and `annotate -h` cannot override an existing FORMAT definition, - // but it is harmless because BCFTOOLS_NORM now splits multi-allelics (-m -any): every record - // reaching here carries one ALT and one AF value, making the two declarations equivalent. - // After BCFTOOLS_NORM_REJOIN, consensus files keep the source caller's own header (lossless round - // trip); 'all' mode takes the Number=A header of isec's 0000.vcf.gz, which concat lists first. - // - // The rename is carried out by BCFTOOLS_ANNOTATE below rather than by a second annotate call: - // meta.rename_to selects the --rename-annots file in conf/modules.config. Only 'all' mode needs - // it, since in 'consensus' mode every surviving record comes from one caller. BCFTOOLS_QUERY - // reads only CHROM/POS/REF/ALT, so it does not care whether the rename has happened yet. + // Only 'all' mode renames: it merges both callers, so the merged VCF needs one AF key for WAKHAN. // // MODULE: BCFTOOLS_QUERY (label: process_single) @@ -80,8 +64,7 @@ workflow SMALL_VARIANT_CONSENSUS { def columns = [] // no extra column specs def header_lines = [] // no extra header lines def rename_chrs = [] // no chromosome renaming - // 'all' mode merges records from both callers into one VCF, so the allele - // frequency key is unified here; 'consensus' mode needs no rename. + // 'all' mode merges both callers, so unify the AF key; 'consensus' needs no rename. def new_meta = combine_method == 'all' ? meta + [rename_to: (prioritize_caller in ['deepvariant', 'deepsomatic'] ? 'VAF' : 'AF')] : meta @@ -109,8 +92,7 @@ workflow SMALL_VARIANT_CONSENSUS { // annotated_vcfs: [meta(+caller), vcf, tbi] -- VCF with CALLER INFO tag // Branch annotated VCFs by caller family for the intersection step - // An unrecognised meta.caller would silently vanish without the `other` arm, taking the whole - // sample out of the results with a successful exit. + // `other` errors on an unrecognised meta.caller instead of silently dropping the sample. annotated_vcfs .branch { meta, _vcfs, _tbi -> deepvariant: meta.caller in [ 'deepvariant', 'deepsomatic' ] @@ -165,8 +147,7 @@ workflow SMALL_VARIANT_CONSENSUS { // deepvariant_ch: [meta (no caller), vcf, tbi] // Join DeepVariant and Clair VCFs per sample into a single tuple for BCFTOOLS_ISEC - // failOnMismatch: a sample present for one caller but not the other would otherwise be dropped - // from every downstream result while the run still reported success. + // failOnMismatch: a sample missing one caller would otherwise be dropped silently. deepvariant_ch .join(clair_ch, failOnMismatch: true, failOnDuplicate: true) .map { meta, deepvar_vcf, deepvar_tbi, clair_vcf, clair_tbi -> @@ -219,10 +200,8 @@ workflow SMALL_VARIANT_CONSENSUS { } else if (combine_method == 'all') { - // Union: all variants from both callers. Shared variants contribute a single record, taken - // from the prioritized caller; both callers' private calls are kept. prioritize_caller only - // selects whose record is used for shared variants, never which calls are kept. - // The three isec sets are disjoint by construction, so BCFTOOLS_CONCAT needs no -d. + // Union: shared calls (prioritized caller's record) plus both callers' private calls. + // The three isec sets are disjoint, so BCFTOOLS_CONCAT needs no -d. if (prioritize_caller in ['deepvariant', 'deepsomatic']) { // shared (DeepVariant record) + DeepVariant-private + Clair-private BCFTOOLS_ISEC.out.deepvar_consensus_vcf diff --git a/subworkflows/local/tumor_only/tumoronly_smallvar.nf b/subworkflows/local/tumor_only/tumoronly_smallvar.nf index 5ec14050..ab726ceb 100644 --- a/subworkflows/local/tumor_only/tumoronly_smallvar.nf +++ b/subworkflows/local/tumor_only/tumoronly_smallvar.nf @@ -12,7 +12,6 @@ include { SMALL_VARIANT_CONSENSUS as GERMLINE_CONSENSUS } from '../../../subwork include { SMALL_VARIANT_CONSENSUS as SOMATIC_CONSENSUS } from '../../../subworkflows/local/small_variant_consensus.nf' // Germline verdict transfer: DeepSomatic adjudicates DeepVariant's tumor-derived germline calls. -// Three independent bcftools invocations, so three aliased instances of the upstream modules. include { BCFTOOLS_QUERY as DS_VERDICT_QUERY } from '../../../modules/nf-core/bcftools/query/main' include { BCFTOOLS_ANNOTATE as DS_VERDICT_ANNOTATE } from '../../../modules/nf-core/bcftools/annotate/main' include { BCFTOOLS_VIEW as DS_GERMLINE_SELECT } from '../../../modules/nf-core/bcftools/view/main' @@ -219,25 +218,7 @@ workflow TUMORONLY_SMALLVAR { // Keep only DeepSomatic-adjudicated germline sites; skipped when deepsomatic isn't selected. def deepvariant_germline = deepvariant_vcf if (somatic_var_keep.contains('deepsomatic')) { - // GERMLINE VERDICT TRANSFER (tumor-only, deep family) - // DeepVariant is a germline caller with no somatic discrimination -- its FILTER vocabulary is - // only PASS/RefCall/LowQual/NoCall -- and here it is run on the TUMOR BAM, so on its own its - // calls are "germline or clonal somatic" and cannot be told apart. Published unchanged, the - // germline VCF therefore carries most of the somatic call set. - // - // DeepSomatic evaluates the same sites and does emit a verdict: FILTER=GERMLINE ("Non somatic - // variants"), PON, RefCall or PASS. That verdict is transferred here, exactly as ClairS-TO - // adjudicates its own calls via NonSomatic and VCFSPLIT. On B1975944 DeepVariant's 5,058,527 - // PASS calls resolve to 77.8% GERMLINE, 11.4% RefCall, 5.3% PON, 4.3% unevaluated and 1.14% - // (57,684) PASS -- the last being real somatic calls that must not be published as germline. - // - // Only positively-adjudicated germline sites are kept (GERMLINE or PON); RefCall and - // unevaluated sites are dropped rather than assumed germline. Only GERMLINE/PON verdicts - // are transferred to INFO/DS_VERDICT. - // - // DeepSomatic FILTER is single-valued in practice (RefCall/GERMLINE/PON/PASS only, verified - // over 13.7M records), so transferring it as a plain string cannot inject the ";" that would - // break INFO parsing. + // GERMLINE VERDICT TRANSFER: DeepVariant on the tumor BAM cannot tell germline from somatic. // // MODULE: DS_VERDICT_QUERY (BCFTOOLS_QUERY alias, label: process_single) // Input: [meta, deepsomatic_vcf, tbi] -- the RAW DeepSomatic VCF, before its PASS filter From ff88896d455992c705b9520ba384ecec87334df0 Mon Sep 17 00:00:00 2001 From: robert-a-forsyth Date: Fri, 25 Sep 2026 12:57:41 +0200 Subject: [PATCH 25/25] Regenerate pipeline snapshots after the review fixes PASS_FILTER is now reported per alias, BCFTOOLS_NORM_REJOIN is new, and the multi-caller modes phase from rejoined multi-allelics, so union/consensus haplotagged BAMs and WhatsHap stats change. Snapshots regenerated with Nextflow 25.10.4. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/chm13.nf.test.snap | 11 +++++--- tests/clair_only.nf.test.snap | 13 +++++---- tests/consensus.nf.test.snap | 44 +++++++++++++++++++----------- tests/deep_only.nf.test.snap | 13 +++++---- tests/default.nf.test.snap | 13 +++++---- tests/union.nf.test.snap | 50 ++++++++++++++++++++++------------- 6 files changed, 90 insertions(+), 54 deletions(-) diff --git a/tests/chm13.nf.test.snap b/tests/chm13.nf.test.snap index ad12da16..4e196867 100644 --- a/tests/chm13.nf.test.snap +++ b/tests/chm13.nf.test.snap @@ -14,6 +14,9 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, @@ -23,6 +26,9 @@ "CLAIRSTO_CNA_RESOURCES": { "coreutils": 9.5 }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -67,9 +73,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -148,6 +151,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-23T10:46:28.14524359" + "timestamp": "2026-09-25T12:34:22.954806822" } } \ No newline at end of file diff --git a/tests/clair_only.nf.test.snap b/tests/clair_only.nf.test.snap index a7f1925e..557714f7 100644 --- a/tests/clair_only.nf.test.snap +++ b/tests/clair_only.nf.test.snap @@ -24,12 +24,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -74,9 +80,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -840,8 +843,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T17:07:33.562660027" + "timestamp": "2026-09-25T12:28:35.531071958" } } \ No newline at end of file diff --git a/tests/consensus.nf.test.snap b/tests/consensus.nf.test.snap index 889b9399..03293479 100644 --- a/tests/consensus.nf.test.snap +++ b/tests/consensus.nf.test.snap @@ -14,6 +14,9 @@ "BCFTOOLS_NORM": { "bcftools": 1.22 }, + "BCFTOOLS_NORM_REJOIN": { + "bcftools": 1.22 + }, "BCFTOOLS_QUERY": { "bcftools": 1.22 }, @@ -29,12 +32,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -47,6 +56,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -56,6 +68,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -106,9 +121,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -625,10 +637,10 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,a524e54ab0685eeb79e461926d8c7906", - "sample1_normal.bam.bai:md5,ba98fb39fc3cc8b7319376b003aa3dcd", - "sample1_tumor.bam:md5,db8b0da3dd1a6f1c38420a4942881119", - "sample1_tumor.bam.bai:md5,1b75ed3238520172019817b88d254f1f", + "sample1_normal.bam:md5,5d3f0615b0d8a9748e6b6bc7d76ab577", + "sample1_normal.bam.bai:md5,3ef784f2536c7506a50d0ed20c3d78fb", + "sample1_tumor.bam:md5,7317ffe61fe3c4f73961db60329adfa9", + "sample1_tumor.bam.bai:md5,6aaeaebe40c422358bb55296c508db2b", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", @@ -636,18 +648,18 @@ "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", "sample1_whatshap_stats.gtf:md5,30bde8f88b7d4e88b935e88e00997ce7", - "sample1_whatshap_stats.log:md5,51d522df21c6b526909a5cc9de93eaea", - "sample1_whatshap_stats.tsv:md5,8ce67834b3b7471c04151b55585a56bc", + "sample1_whatshap_stats.log:md5,a3cf683c728ce63a5c2be33edd20f9cc", + "sample1_whatshap_stats.tsv:md5,cca55ca6f99eb0857759a4007c10b346", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,0b3633a4d389b4467fb3eac83da94882", + "sample2_normal.bam:md5,ece242f245fc4e2210e13e6de93b8fdc", "sample2_normal.bam.bai:md5,d96d0071ab25ca8dc2327acba4395515", - "sample2_tumor.bam:md5,5c549642cc7192289222d558dc3a07a7", - "sample2_tumor.bam.bai:md5,1554e8cc8ced05271f5d5f7ac7819410", + "sample2_tumor.bam:md5,0c538dc0e27566677313d1f9374fa2b1", + "sample2_tumor.bam.bai:md5,ab2fca59e6729e011c30310eaee2ef1c", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", @@ -655,8 +667,8 @@ "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", "sample2_whatshap_stats.gtf:md5,2e5ace4cac0b42bb6132513062781e47", - "sample2_whatshap_stats.log:md5,9aa99ee14473f65071366ed1729ab064", - "sample2_whatshap_stats.tsv:md5,13cbc783b1357e1bd2c6bb51ae6f1267", + "sample2_whatshap_stats.log:md5,0979359e459a14037a704733f32d3b95", + "sample2_whatshap_stats.tsv:md5,8c974019af5c839c72604e7526ae8a3d", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -681,8 +693,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T17:31:36.878403725" + "timestamp": "2026-09-25T12:21:59.090749802" } } \ No newline at end of file diff --git a/tests/deep_only.nf.test.snap b/tests/deep_only.nf.test.snap index 6d2a1148..9a25aad9 100644 --- a/tests/deep_only.nf.test.snap +++ b/tests/deep_only.nf.test.snap @@ -23,6 +23,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -32,6 +35,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -82,9 +88,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -626,8 +629,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T17:50:37.140724409" + "timestamp": "2026-09-25T12:56:04.936282901" } } \ No newline at end of file diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 4ee27de3..62439fb0 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -14,12 +14,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -64,9 +70,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -618,8 +621,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T18:06:32.118064724" + "timestamp": "2026-09-25T11:57:04.096725193" } } \ No newline at end of file diff --git a/tests/union.nf.test.snap b/tests/union.nf.test.snap index de00b2c4..42bea83c 100644 --- a/tests/union.nf.test.snap +++ b/tests/union.nf.test.snap @@ -14,6 +14,9 @@ "BCFTOOLS_NORM": { "bcftools": 1.22 }, + "BCFTOOLS_NORM_REJOIN": { + "bcftools": 1.22 + }, "BCFTOOLS_QUERY": { "bcftools": 1.22 }, @@ -26,12 +29,18 @@ "CLAIR3": { "clair3": "1.2.0" }, + "CLAIR3_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CLAIRS": { "clairs": "0.4.4" }, "CLAIRSTO": { "clairsto": "0.5.1" }, + "CLAIRS_PASS_FILTER": { + "bcftools": "1.23.1" + }, "CRAMINO_POST": { "cramino": "1.3.0" }, @@ -44,6 +53,9 @@ "DEEPSOMATIC_MAKEEXAMPLES": { "deepsomatic": "1.7.0" }, + "DEEPSOMATIC_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPSOMATIC_POSTPROCESSVARIANTS": { "deepsomatic": "1.7.0" }, @@ -53,6 +65,9 @@ "DEEPVARIANT_MAKEEXAMPLES": { "deepvariant": "1.9.0" }, + "DEEPVARIANT_PASS_FILTER": { + "bcftools": "1.23.1" + }, "DEEPVARIANT_POSTPROCESSVARIANTS": { "deepvariant": "1.9.0" }, @@ -103,9 +118,6 @@ "NANOPLOT_PRE": { "nanoplot": "1.46.1" }, - "PASS_FILTER": { - "bcftools": "1.23.1" - }, "SAMTOOLS_FAIDX": { "samtools": "1.22.1" }, @@ -622,38 +634,38 @@ "sample3/vep/somatic/sample3_SOMATIC_VEP.vcf.gz_summary.html" ], [ - "sample1_normal.bam:md5,dcb537c163750d41140d0a4da107f18e", - "sample1_normal.bam.bai:md5,12cf13fae8230cfc1cc8888f80102143", - "sample1_tumor.bam:md5,58cf65517c91958f95a1f4be1c75e2df", - "sample1_tumor.bam.bai:md5,e40049da796ad038e7ebeaa9ac070488", + "sample1_normal.bam:md5,f0809c8be190289f21339696883dea6f", + "sample1_normal.bam.bai:md5,cbff669c086761e893a84ba3ec032b5f", + "sample1_tumor.bam:md5,6267bf3e86a69536e45968b908d1cc6f", + "sample1_tumor.bam.bai:md5,1ec6d68c92182ff9051b8707c4975445", "sample1_normal.flagstat:md5,1c41ea9923945501eb7e41f83a90502d", "sample1_normal.idxstats:md5,902e503387799123ea59255e3fca172c", "sample1_normal.stats:md5,a8b3fba9c54efbc0934d6eacc1807140", "sample1_tumor.flagstat:md5,8ff32d733c62c4910bf185ef24bf27cf", "sample1_tumor.idxstats:md5,2de140e61f9e86c9c10af20dd565cc93", "sample1_tumor.stats:md5,1c60a1d249d2e503b0678c72e851ea93", - "sample1_whatshap_stats.gtf:md5,9a7823e6b572036a7e95904b0083b8ca", - "sample1_whatshap_stats.log:md5,05c819627f77305a75d776a27dbf247e", - "sample1_whatshap_stats.tsv:md5,5b9d3f6a6e1e1c67c855f8f9b581bb18", + "sample1_whatshap_stats.gtf:md5,f5d331899db63b2ae21e51436bad2ddd", + "sample1_whatshap_stats.log:md5,111538b7e305189fb6de95ea978d06c8", + "sample1_whatshap_stats.tsv:md5,fa0fd5ce2b0919098ffa7d960cf177de", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,47cb0e0bbe71abdbf4f40217dfda43f9", "read_qual.txt:md5,78247dfa2ea336eac0e128eba5e9eef4", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", - "sample2_normal.bam:md5,74e92562e0cd406a6282bce98ccc93d2", - "sample2_normal.bam.bai:md5,430c3b069a79227396a6b4f1c37a02ef", - "sample2_tumor.bam:md5,0059b8022c674b5daf2fd001ab69a54e", - "sample2_tumor.bam.bai:md5,e236afb5f7a644827e2b38cd36838d98", + "sample2_normal.bam:md5,c18bbb1bc05b0ec830fa1ac3c6ef542f", + "sample2_normal.bam.bai:md5,de77684ef264476b3e0531b61ca23740", + "sample2_tumor.bam:md5,8b8cb5ac7668b8ac2f4097f584481d35", + "sample2_tumor.bam.bai:md5,3b46456b0e7b00688519dfb25c8e3c88", "sample2_normal.flagstat:md5,714d0cc0c213e2640e54a16f3d0e6e7e", "sample2_normal.idxstats:md5,72eb83bb11748dc863fef1a0a5497e4b", "sample2_normal.stats:md5,20c47cb94f9ac739d69c57be6daf82c5", "sample2_tumor.flagstat:md5,4344a8745efef9cc2a017024218d61c6", "sample2_tumor.idxstats:md5,69467fc02c83a30084736aeea8b785fb", "sample2_tumor.stats:md5,8635df10132c85a13f2d9878b7cf90a2", - "sample2_whatshap_stats.gtf:md5,8107c5dce892aa11145274fd088788f8", - "sample2_whatshap_stats.log:md5,c2b17319f0f9e6a8ec102be86aa833ba", - "sample2_whatshap_stats.tsv:md5,7028c4a428f5d0fd31472439e287e565", + "sample2_whatshap_stats.gtf:md5,35cd28699c298d99d01cee1c24c6d61b", + "sample2_whatshap_stats.log:md5,9308d0bd8dc9a4a86359f1ec926e6cb1", + "sample2_whatshap_stats.tsv:md5,b81282a609eca306912d4f7b7fdfeebb", "breakpoint_clusters.tsv:md5,d36a70de292ee130ef30da4a58bced18", "breakpoint_clusters_list.tsv:md5,0c0ce62e329f8de492487e8414c30a50", "breakpoints_double.csv:md5,48baac86492026a4a7947bc708c47e6e", @@ -678,8 +690,8 @@ ], "meta": { "nf-test": "0.9.3", - "nextflow": "26.04.1" + "nextflow": "25.10.4" }, - "timestamp": "2026-09-22T18:43:33.111991495" + "timestamp": "2026-09-25T12:04:10.544092583" } } \ No newline at end of file