diff --git a/CHANGELOG.md b/CHANGELOG.md index b80c26bb..fb2c964c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#206](https://github.com/IntGenomicsLab/lrsomatic/pull/206) - A remote (http, https or ftp) ClinVar is now downloaded once per run by the new `VEPPLUGIN_CLINVAR` step in `PREPARE_VEP_PLUGINS`, instead of being staged by Nextflow as a foreign file; local and cloud-storage paths are staged as before. `GERMLINE_VEP` and `SOMATIC_VEP` re-checked the foreign file on its host for every sample, and NCBI answered the burst from a multi-sample GRCh38 run with HTTP 503, so `SOMATIC_VEP` failed with `Can't stage file ...clinvar_20260829.vcf.gz`; `-resume` could not recover, since the failed check changed the staging cache key. The download is checked against the new `--vep_clinvar_md5` and `--vep_clinvar_tbi_md5`, set by default to the checksums NCBI (GRCh38, VCF only) and Ensembl (CHM13, VCF and index) publish, so the pinned release cannot change silently. Resuming a run that already finished re-runs `GERMLINE_VEP` and `SOMATIC_VEP` once, since ClinVar now comes from a task rather than the stage cache. The ClinVar sizes in `docs/usage.md` are also corrected, and `docs/output.md` now documents `vep_plugins/` (@AmberVerhasselt). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `CLAIRS` no longer runs with `--haplotagged_tumor_bam_provided_so_skip_intermediate_phasing_and_haplotagging`. Since somatic calling was moved ahead of `PHASING_HAPLOTYPING` (v1.1.0), ClairS has received the untagged minimap2 BAM, so the flag made it skip its own phasing and haplotagging and call every paired sample without haplotype information: the full-alignment model saw no `HP` tags and the haplotype filtering step had nothing to filter on, the same as `--disable_phasing`. ClairS now runs Clair3 on the normal and tumour BAMs and phases the tumour itself. **Paired somatic calls change** (fewer false positives expected), and `CLAIRS` takes longer and uses more work-directory space (@ljwharbers). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `docs/output.md` now lists the ClairS SNV output as `snvs.vcf.gz`, the name the pipeline publishes, instead of `snv.vcf.gz` (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). diff --git a/conf/igenomes.config b/conf/igenomes.config index 71ff9f59..430e65a2 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -28,9 +28,12 @@ params.genomes = [ vep_alphamissense_tbi : "https://g-608c0c.273595.03c0.data.globus.org/VEP_plugins/AlphaMissense_hg38.tsv.gz.tbi", // A dated release rather than the rolling vcf_GRCh38/clinvar.vcf.gz, whose VCF and // separately-downloaded index both move weekly: two runs would otherwise annotate - // differently, and one straddling an update could pair a new VCF with an old index + // differently, and one straddling an update could pair a new VCF with an old index. + // Downloaded once per run by VEPPLUGIN_CLINVAR, which checks it against NCBI's published MD5 vep_clinvar : "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz", vep_clinvar_tbi : "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz.tbi", + vep_clinvar_md5 : "dffae274fd559ee764d1de44f36df2cf", + // No vep_clinvar_tbi_md5: NCBI publishes no checksum for its indexes // CADD is opt-in: the SNV table alone is 81 GB, and Nextflow's foreign-file cache is // per session, so every non-resumed run would re-stage it. Enable with --vep_cadd_snv // and --vep_cadd_indel (each with its _tbi); see docs/usage.md. @@ -60,6 +63,9 @@ params.genomes = [ vep_polyphen_sift_db : "https://ftp.ensembl.org/pub/release-115/variation/pangenomes/Human/homo_sapiens_pangenome_PolyPhen_SIFT_20240502.db", vep_clinvar : "https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/ensembl/variation/2022_10/vcf/2024_07/clinvar_20240624_GCA_009914755.4.vcf.gz", vep_clinvar_tbi : "https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/ensembl/variation/2022_10/vcf/2024_07/clinvar_20240624_GCA_009914755.4.vcf.gz.tbi", + // From the CHECKSUMS file Ensembl publishes alongside the VCF and its index + vep_clinvar_md5 : "98bde67975c5f0b95230672f642a3c07", + vep_clinvar_tbi_md5 : "50b634a69c36e2479cc2c8f71a8eafd6", // SigProfilerMatrixGenerator CHM13-T2T payload (SigProfilerSuite/SigProfilerMatrixGenerator#250); // not yet hosted on the AlexandrovLab FTP, so it is fetched from the IntGenomicsLab Globus collection sigprofiler_genome : "CHM13-T2T", diff --git a/conf/modules.config b/conf/modules.config index bd0ef4f1..54494857 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -675,12 +675,18 @@ process { time = { 24.h * task.attempt } } + // The wget container carries no CA bundle, as for WGET above; the pinned MD5 still checks the download + withName : '.*:VEPPLUGIN_CLINVAR' { + ext.args = { "--no-check-certificate" } + } + // Published so a later run can skip both the download and the reshaping by pointing - // --vep_revel / --vep_eve (and their _tbi) at these files - withName : '.*:VEPPLUGIN_(REVEL|EVE)' { + // --vep_revel / --vep_eve / --vep_clinvar (and their _tbi) at these files + withName : '.*:VEPPLUGIN_(REVEL|EVE|CLINVAR)' { publishDir = [ mode: params.publish_dir_mode, - path: { "${params.outdir}/vep_plugins" } + path: { "${params.outdir}/vep_plugins" }, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } diff --git a/docs/output.md b/docs/output.md index b392805f..54fb647c 100644 --- a/docs/output.md +++ b/docs/output.md @@ -497,6 +497,31 @@ Phased variant calls produced by Longphase. Present in all samples. +#### `vep_plugins` + +
+Output files + +``` +├── vep_plugins +│ ├── clinvar_20260829.vcf.gz +│ ├── clinvar_20260829.vcf.gz.tbi +│ ├── revel_grch38.tsv.gz +│ ├── revel_grch38.tsv.gz.tbi +│ ├── eve_merged.vcf.gz +│ ├── eve_merged.vcf.gz.tbi +``` + +| File | Description | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------- | +| `clinvar_.vcf.gz{,.tbi}` | ClinVar release and index as downloaded and MD5-checked by `VEPPLUGIN_CLINVAR`; only for a remote `--vep_clinvar` | +| `revel_grch38.tsv.gz{,.tbi}` | REVEL scores re-sorted on GRCh38 and indexed by `VEPPLUGIN_REVEL` | +| `eve_merged.vcf.gz{,.tbi}` | Per-protein EVE VCFs merged and indexed by `VEPPLUGIN_EVE`; only with the opt-in `--vep_eve` | + +Pass these to a later run as `--vep_clinvar`/`--vep_revel`/`--vep_eve` with their `_tbi` to skip the download and preparation. On CHM13 the ClinVar file is the CHM13-lifted Ensembl release instead. + +
+ #### Plugin fields in the `CSQ` annotation The germline and somatic VCFs carry these extra subfields inside VEP's `CSQ` INFO annotation, on diff --git a/docs/usage.md b/docs/usage.md index c9fe4d26..7d822ecb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -223,6 +223,8 @@ opt-in. See [VEP plugins](#vep-plugins) for sizes, licence terms and per-assembl | `--vep_polyphen_sift_db` | Ensembl pangenome PolyPhen/SIFT SQLite database, for the `PolyPhen_SIFT` plugin. Needed on CHM13 only | | `--vep_clinvar` | ClinVar VCF, added as a VEP `--custom` annotation | | `--vep_clinvar_tbi` | Index for `--vep_clinvar`. Required whenever `--vep_clinvar` is set | +| `--vep_clinvar_md5` | Expected MD5 of a remote `--vep_clinvar`, checked after download. Dropped when `--vep_clinvar` is overridden | +| `--vep_clinvar_tbi_md5` | Expected MD5 of the downloaded `--vep_clinvar_tbi`. Set on CHM13 only; dropped when either ClinVar file is overridden | | `--vep_clinvar_fields` | Comma-separated ClinVar INFO fields to carry through. Default = `"CLNSIG,CLNREVSTAT,CLNDN"` | | `--vep_cadd_snv` | CADD SNV score file, for the `CADD` plugin. No default — 81 GB, so opt-in; prefer a local path. GRCh38 only | | `--vep_cadd_snv_tbi` | Index for `--vep_cadd_snv`. Required whenever `--vep_cadd_snv` is set | @@ -468,7 +470,7 @@ Plugins are applied to the germline and somatic VEP runs, not to the structural- | **SIFT** | already in the VEP cache, no file needed | `PolyPhen_SIFT` plugin | CHM13: 13 GB database | | **PolyPhen** | already in the VEP cache, no file needed | `PolyPhen_SIFT` plugin | as above, the same database | | **AlphaMissense** | `AlphaMissense` plugin | `AlphaMissenseProtein` plugin | 613 MB (GRCh38) / 1.1 GB (CHM13) | -| **ClinVar** | `--custom` annotation | `--custom`, CHM13-lifted VCF | 105 MB (GRCh38) / 190 MB (CHM13) | +| **ClinVar** | `--custom` annotation | `--custom`, CHM13-lifted VCF | 193 MB (GRCh38) / 99 MB (CHM13) | | **CADD** | `CADD` plugin, opt-in | not available — see below | none — `--vep_cadd_snv` enables it (81 GB + 1.2 GB) | | **REVEL** | `REVEL` plugin | not available — see below | 667 MB release zip | | **EVE** | `EVE` plugin, opt-in | not available — see below | none — `--vep_eve` enables it (9.6 GB) | @@ -527,6 +529,15 @@ Whether an index is required depends on the shape of what you supply: - **AlphaMissense, ClinVar and CADD** are used exactly as given, so their `_tbi` parameter is always required alongside them. Overriding a data file drops the default index: supply both, or neither. +- **A remote ClinVar** (http, https or ftp) is downloaded once per run by `VEPPLUGIN_CLINVAR`, with + its index, rather than staged by `GERMLINE_VEP` and `SOMATIC_VEP` for every sample: NCBI answers + the burst of requests a multi-sample run sends with HTTP 503. The index must then be a URL too. The + download is checked against `--vep_clinvar_md5` (and `--vep_clinvar_tbi_md5` where set), so a + release re-published under the same name fails the run instead of changing the annotation. Both + defaults carry the MD5 their host publishes, and the CHM13 default also pins its index; NCBI + publishes no index checksum. With your own URL, pass its MD5 too, or the pipeline warns that the + release is not verified. Local and cloud-storage (`s3://`, `gs://`, `az://`) paths are staged as + given. The files are published to `/vep_plugins/` for reuse as local paths. - **REVEL and EVE** ship as zip archives. Pass a `.zip` and the pipeline unpacks and reshapes it; pass a prepared file and its index to use it directly. Remote zips are fetched with `wget` rather than staged by Nextflow, and EVE's 9.6 GB archive serves slowly, so expect hours. @@ -538,8 +549,8 @@ Whether an index is required depends on the shape of what you supply: | AlphaMissense (GRCh38) | `https://storage.googleapis.com/dm_alphamissense/AlphaMissense_hg38.tsv.gz`, with an index we host | 613 MB | no, the index is fetched ready | CC BY 4.0 | | AlphaMissense (protein) | a gene-symbol-keyed table we host, built from the AlphaMissense protein-space release | 1.1 GB | no, fetched ready — see `CITATIONS.md` | CC BY 4.0 | | Pangenome PolyPhen/SIFT | `https://ftp.ensembl.org/pub/release-115/variation/pangenomes/Human/homo_sapiens_pangenome_PolyPhen_SIFT_20240502.db` | 13 GB | no, it is an SQLite database | Ensembl / EMBL-EBI open | -| ClinVar (GRCh38) | `clinvar_20260829.vcf.gz` under `https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/` (+ `.tbi`) | 105 MB | no, `.tbi` is published | public domain | -| ClinVar (CHM13) | `clinvar_20240624_GCA_009914755.4.vcf.gz` under `https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/` | 190 MB | no, `.tbi` is published | public domain | +| ClinVar (GRCh38) | `clinvar_20260829.vcf.gz` under `https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/` (+ `.tbi`) | 193 MB | downloaded once, MD5-checked | public domain | +| ClinVar (CHM13) | `clinvar_20240624_GCA_009914755.4.vcf.gz` under `https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/` | 99 MB | downloaded once, MD5-checked | public domain | | CADD v1.7 SNVs (opt-in) | `https://krishna.gs.washington.edu/download/CADD/v1.7/GRCh38/whole_genome_SNVs.tsv.gz` (+ `.tbi`) | 81 GB | no, `.tbi` is published | free for non-commercial use | | CADD v1.7 indels (opt-in) | `https://krishna.gs.washington.edu/download/CADD/v1.7/GRCh38/gnomad.genomes.r4.0.indel.tsv.gz` (+ `.tbi`) | 1.2 GB | no, `.tbi` is published | free for non-commercial use | | REVEL v1.3 | `https://rothsj06.dmz.hpc.mssm.edu/revel-v1.3_all_chromosomes.zip` | 667 MB | unpacked, re-sorted on GRCh38, indexed | free for non-commercial use | diff --git a/modules/local/vepplugin/clinvar/environment.yml b/modules/local/vepplugin/clinvar/environment.yml new file mode 100644 index 00000000..daa07275 --- /dev/null +++ b/modules/local/vepplugin/clinvar/environment.yml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + # renovate: datasource=conda depName=conda-forge/wget + - conda-forge::wget=1.21.4 diff --git a/modules/local/vepplugin/clinvar/main.nf b/modules/local/vepplugin/clinvar/main.nf new file mode 100644 index 00000000..ff2947cf --- /dev/null +++ b/modules/local/vepplugin/clinvar/main.nf @@ -0,0 +1,68 @@ +process VEPPLUGIN_CLINVAR { + tag "${vcf_url.toString().tokenize('/').last()}" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/3b/3b54fa9135194c72a18d00db6b399c03248103f87e43ca75e4b50d61179994b3/data' + : 'community.wave.seqera.io/library/wget:1.21.4--8b0fcde81c17be5e'}" + + input: + tuple val(vcf_url), val(tbi_url), val(md5), val(tbi_md5) + + output: + path "${vcf_name}{,.tbi}", emit: files + // versions.yml rather than an eval() topic: eval outputs are numbered pipeline-wide, so adding + // one shifts the cache key of every other task that has one and breaks -resume of existing runs + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + vcf_name = vcf_url.toString().tokenize('/').last() + def sums = [ md5 ? "${md5} ${vcf_name}" : null, tbi_md5 ? "${tbi_md5} ${vcf_name}.tbi" : null ].findAll() + def check = sums ? "printf '%s\\n' ${sums.collect { line -> "'${line}'" }.join(' ')} | md5sum -c -" : '' + """ + # Retry a transient 503 from NCBI: a linear backoff of 1 s, 2 s, ... up to 10 s, about 45 s over 10 tries + wget \\ + --no-verbose \\ + --tries=10 \\ + --waitretry=10 \\ + --retry-on-http-error=429,500,502,503,504 \\ + ${args} \\ + -O ${vcf_name} \\ + ${vcf_url} + + # Saved next to the VCF under the name VEP looks for, whatever the host calls it + wget \\ + --no-verbose \\ + --tries=10 \\ + --waitretry=10 \\ + --retry-on-http-error=429,500,502,503,504 \\ + ${args} \\ + -O ${vcf_name}.tbi \\ + ${tbi_url} + + # Pinned checksums keep the release fixed: a host that re-publishes under the same name fails here + ${check} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wget: \$(wget --version | head -1 | cut -d ' ' -f 3) + END_VERSIONS + """ + + stub: + vcf_name = vcf_url.toString().tokenize('/').last() + """ + echo "" | gzip > ${vcf_name} + touch ${vcf_name}.tbi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wget: \$(wget --version | head -1 | cut -d ' ' -f 3) + END_VERSIONS + """ +} diff --git a/modules/local/vepplugin/clinvar/meta.yml b/modules/local/vepplugin/clinvar/meta.yml new file mode 100644 index 00000000..c35d7bd4 --- /dev/null +++ b/modules/local/vepplugin/clinvar/meta.yml @@ -0,0 +1,51 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "vepplugin_clinvar" +description: Download a dated ClinVar VCF and its index once per run, checked against a pinned MD5, for the VEP --custom annotation +keywords: + - vep + - clinvar + - download + - annotation +tools: + - "wget": + description: "GNU Wget is a free software package for retrieving files using HTTP, HTTPS, FTP and FTPS" + homepage: "https://www.gnu.org/software/wget/" + documentation: "https://www.gnu.org/software/wget/manual/wget.html" + licence: ["GPL-3.0-or-later"] + identifier: "" + +input: + - - vcf_url: + type: string + description: URL of the ClinVar VCF; its basename becomes the output name + - tbi_url: + type: string + description: URL of the tabix index published alongside the VCF + - md5: + type: string + description: | + Expected MD5 of the VCF. When set, a download that does not match fails + the task. May be null, in which case the release is not verified. + - tbi_md5: + type: string + description: | + Expected MD5 of the index, checked the same way. May be null, as for + NCBI, which publishes no checksum for its indexes. + +output: + files: + - "${vcf_name}{,.tbi}": + type: file + description: The ClinVar VCF and its tabix index + pattern: "*.vcf.gz{,.tbi}" + versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@AmberVerhasselt" +maintainers: + - "@AmberVerhasselt" diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test b/modules/local/vepplugin/clinvar/tests/main.nf.test new file mode 100644 index 00000000..e1ed2a25 --- /dev/null +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test @@ -0,0 +1,41 @@ +nextflow_process { + + name "Test Process VEPPLUGIN_CLINVAR" + script "../main.nf" + process "VEPPLUGIN_CLINVAR" + + // "small" is what .github/workflows/nf-test.yml selects on for pull_request + tag "small" + tag "modules" + tag "modules_local" + tag "vepplugin_clinvar" + + // Stub only: a real run downloads the full ClinVar release + test("clinvar release - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz', + 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz.tbi', + 'dffae274fd559ee764d1de44f36df2cf', + null // NCBI publishes no checksum for its index + ] + """ + } + } + + then { + assert process.success + assertAll( + // the VCF keeps its release name, and the index sits beside it under the name VEP looks for + { assert process.out.files[0].collect { f -> file(f).name }.sort() == [ 'clinvar_20260829.vcf.gz', 'clinvar_20260829.vcf.gz.tbi' ] }, + { assert snapshot(process.out).match() } + ) + } + + } +} diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test.snap b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap new file mode 100644 index 00000000..d1a484af --- /dev/null +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap @@ -0,0 +1,31 @@ +{ + "clinvar release - stub": { + "content": [ + { + "0": [ + [ + "clinvar_20260829.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "clinvar_20260829.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,ca06b557f10c3af9a0e088e9a76b0580" + ], + "files": [ + [ + "clinvar_20260829.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "clinvar_20260829.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,ca06b557f10c3af9a0e088e9a76b0580" + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-25T16:19:22.283693138" + } +} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 696d5161..628b5ca3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -58,6 +58,8 @@ params { vep_polyphen_sift_db = null vep_clinvar = null vep_clinvar_tbi = null + vep_clinvar_md5 = null + vep_clinvar_tbi_md5 = null vep_clinvar_fields = "CLNSIG,CLNREVSTAT,CLNDN" vep_cadd_snv = null vep_cadd_snv_tbi = null diff --git a/nextflow_schema.json b/nextflow_schema.json index 713e2baa..47356879 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -283,6 +283,20 @@ "fa_icon": "fas fa-file", "help_text": "Required whenever --vep_clinvar is set: ClinVar publishes a .tbi alongside every VCF, so the pipeline does not build one." }, + "vep_clinvar_md5": { + "type": "string", + "description": "Expected MD5 of the --vep_clinvar VCF, checked after the pipeline downloads it.", + "fa_icon": "fas fa-fingerprint", + "pattern": "^[0-9a-fA-F]{32}$", + "help_text": "A remote --vep_clinvar (http, https or ftp) is downloaded once per run by VEPPLUGIN_CLINVAR, rather than staged by GERMLINE_VEP and SOMATIC_VEP for every sample, and a VCF that does not match this MD5 fails the run, so the release cannot change silently. The GRCh38 and CHM13 defaults carry the MD5 their hosts publish; overriding --vep_clinvar drops it. A local or cloud-storage --vep_clinvar is staged as given, and takes no MD5." + }, + "vep_clinvar_tbi_md5": { + "type": "string", + "description": "Expected MD5 of the --vep_clinvar_tbi index, checked after the pipeline downloads it.", + "fa_icon": "fas fa-fingerprint", + "pattern": "^[0-9a-fA-F]{32}$", + "help_text": "Checked alongside --vep_clinvar_md5. The CHM13 default carries the MD5 Ensembl publishes; GRCh38 has none, since NCBI publishes no checksum for its indexes. Overriding --vep_clinvar or --vep_clinvar_tbi drops the default." + }, "vep_clinvar_fields": { "type": "string", "description": "Comma-separated ClinVar INFO fields to carry into the annotation.", diff --git a/subworkflows/local/prepare_vep_plugins.nf b/subworkflows/local/prepare_vep_plugins.nf index d445a7f6..bfa31096 100644 --- a/subworkflows/local/prepare_vep_plugins.nf +++ b/subworkflows/local/prepare_vep_plugins.nf @@ -1,5 +1,6 @@ // -// Reshape the VEP plugin releases that ship as zip archives (REVEL and EVE) +// Reshape the VEP plugin releases that ship as zip archives (REVEL and EVE), and download a remote +// ClinVar once so the VEP tasks never stage it from its host themselves // include { UNZIP as UNZIP_REVEL } from '../../modules/nf-core/unzip/main.nf' @@ -8,6 +9,7 @@ include { WGET as WGET_REVEL } from '../../modules/nf-core/wget/main' include { WGET as WGET_EVE } from '../../modules/nf-core/wget/main' include { VEPPLUGIN_REVEL } from '../../modules/local/vepplugin/revel/main.nf' include { VEPPLUGIN_EVE } from '../../modules/local/vepplugin/eve/main.nf' +include { VEPPLUGIN_CLINVAR } from '../../modules/local/vepplugin/clinvar/main.nf' workflow PREPARE_VEP_PLUGINS { @@ -82,6 +84,24 @@ workflow PREPARE_VEP_PLUGINS { ch_versions = ch_versions.mix(UNZIP_EVE.out.versions) } + // + // MODULE: VEPPLUGIN_CLINVAR (label: process_single) + // Input: the ClinVar VCF and index URLs, and the MD5s pinning them (either may be null) + // Output: .files -- the VCF and its index, under the VCF's own basename + // One download per run: a foreign file is re-checked on its host by GERMLINE_VEP and SOMATIC_VEP + // for every sample, and NCBI answers the burst a multi-sample run sends with 503 + // + if (prepare.containsKey('vep_clinvar')) { + def clinvar = prepare['vep_clinvar'] + + VEPPLUGIN_CLINVAR ( + channel.value([ clinvar.vcf, clinvar.tbi, clinvar.md5, clinvar.tbi_md5 ]) + ) + + staged << VEPPLUGIN_CLINVAR.out.files + ch_versions = ch_versions.mix(VEPPLUGIN_CLINVAR.out.versions) + } + // Value channel read by both VEP tasks; ifEmpty carries the no-plugins case, since collect() emits nothing then ch_extra_files = staged .inject(channel.empty()) { acc, ch -> acc.mix(ch) } diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index a64c405c..bc8c7da8 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -408,6 +408,47 @@ def vepPluginNeedsPrep(data_param) { !value.toString().toLowerCase().endsWith('.gz') } +// +// Whether a value is a URL wget can fetch. Cloud and file:// URIs are left to Nextflow's own +// filesystem providers, which stage them without a request per task +// +def isFetchableUrl(value) { + return value && value.toString() ==~ /(?i)^(https?|ftp):\/\/.*/ +} + +// +// Whether a resource is downloaded once by a prep task instead of staged as a foreign file +// +// Only a remote ClinVar is: a foreign file is re-checked on its host by GERMLINE_VEP and +// SOMATIC_VEP for every sample, and NCBI answers the burst a multi-sample run sends with 503, +// which fails the staging. +// +def vepPluginNeedsFetch(data_param) { + return data_param == 'vep_clinvar' && isFetchableUrl(vepPluginResource(data_param)) +} + +// +// The expected MD5 of a fetched resource. Overriding the data file drops the default MD5, +// which belongs to a different release +// +def vepPluginMd5(data_param) { + def md5_param = "${data_param}_md5".toString() + return params[md5_param] ?: (params[data_param] ? null : getGenomeAttribute(md5_param)) +} + +// +// The expected MD5 of a fetched resource's index. Overriding the data file or the index drops +// the default, which belongs to a different file +// +def vepPluginIndexMd5(data_param) { + def index_param = vepPluginIndexParams()[data_param] + if (!index_param) { + return null + } + def md5_param = "${index_param}_md5".toString() + return params[md5_param] ?: ((params[data_param] || params[index_param]) ? null : getGenomeAttribute(md5_param)) +} + // // The filename a prep task writes, referenced by the VEP argument since plugins stage into the task root // @@ -474,6 +515,20 @@ def validateVepPluginParams() { if (params.vep_custom && !(params.vep_args =~ /--custom file=/)) { error("--vep_custom: needs a matching '--custom file=...' entry in --vep_args, which is where the staged file is substituted in. Add one, e.g. --vep_args '${params.vep_args} --custom file=placeholder,short_name=MyTrack,format=vcf,type=exact,coords=0'.") } + + // The MD5s are checked by the download task, so a ClinVar that is staged instead would silently skip them + ['vep_clinvar_md5', 'vep_clinvar_tbi_md5'].each { md5_param -> + if (params[md5_param] && !vepPluginNeedsFetch('vep_clinvar')) { + error("--${md5_param}: only checks a ClinVar the pipeline downloads, so it needs --vep_clinvar to be an http(s) or ftp URL. Drop --${md5_param} for a local or cloud-storage file.") + } + } + // The download task fetches both files, so a remote VCF cannot be paired with a local index + if (vepPluginNeedsFetch('vep_clinvar') && !isFetchableUrl(vepPluginIndex('vep_clinvar'))) { + error("--vep_clinvar_tbi: '${vepPluginIndex('vep_clinvar')}' is not an http(s) or ftp URL, but --vep_clinvar '${vepPluginResource('vep_clinvar')}' is downloaded, and its index is downloaded with it. Pass the index URL, or point both at local copies.") + } + if (vepPluginNeedsFetch('vep_clinvar') && !vepPluginMd5('vep_clinvar')) { + log.warn("--vep_clinvar: '${vepPluginResource('vep_clinvar')}' is downloaded without --vep_clinvar_md5, so its release is not verified: a host that re-publishes under the same name, like the rolling clinvar.vcf.gz, changes the annotation between runs.") + } } // @@ -494,8 +549,14 @@ def stageVepPluginFile(staged, data_param) { // // A resource needing prep is kept as its raw value rather than a file(), since neither the REVEL nor // the EVE host can be staged by Nextflow -- PREPARE_VEP_PLUGINS fetches those with WGET instead. +// A remote ClinVar is recorded with its index and their MD5s, and keeps its own basename. // def registerVepPlugin(staged, prepare, data_param) { + if (vepPluginNeedsFetch(data_param)) { + def url = vepPluginResource(data_param).toString() + prepare[data_param] = [ vcf: url, tbi: vepPluginIndex(data_param), md5: vepPluginMd5(data_param), tbi_md5: vepPluginIndexMd5(data_param) ] + return url.tokenize('/').last() + } if (vepPluginNeedsPrep(data_param)) { prepare[data_param] = vepPluginResource(data_param) return vepPluginPreparedName(data_param) diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test index d98daa90..8c90956b 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test @@ -413,6 +413,321 @@ nextflow_function { } } + // --- vepPluginNeedsFetch / vepPluginMd5: a remote ClinVar is downloaded once, against a pinned MD5 + + test("vepPluginNeedsFetch is true for the default remote ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == true } + ) + } + } + + test("vepPluginNeedsFetch is false for a local ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + vep_clinvar = '/data/mine/clinvar.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert !function.result } + ) + } + } + + test("vepPluginNeedsFetch is false for a remote resource other than ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_alphamissense' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert !function.result } + ) + } + } + + test("vepPluginMd5 takes the default MD5 when ClinVar is the default") { + + function "vepPluginMd5" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result ==~ /[0-9a-f]{32}/ } + ) + } + } + + test("vepPluginMd5 drops the default MD5 once ClinVar is overridden") { + + function "vepPluginMd5" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 'https://example.org/clinvar_20250101.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + // the default MD5 belongs to the default release, not to this one + { assert function.result == null } + ) + } + } + + test("vepPluginMd5 returns an explicitly supplied MD5") { + + function "vepPluginMd5" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 'https://example.org/clinvar_20250101.vcf.gz' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == '0123456789abcdef0123456789abcdef' } + ) + } + } + + test("resolveVepPlugins defers a remote ClinVar to the download task") { + + function "resolveVepPlugins" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/archive/clinvar_20250101.vcf.gz' + vep_clinvar_tbi = 'https://example.org/archive/clinvar_20250101.vcf.gz.tbi' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + { assert function.success }, + // the argument keeps the release's own basename, which the download task writes + { assert function.result.args.startsWith('--custom file=clinvar_20250101.vcf.gz,short_name=ClinVar') }, + { assert function.result.prepare['vep_clinvar'].vcf == 'https://example.org/archive/clinvar_20250101.vcf.gz' }, + { assert function.result.prepare['vep_clinvar'].tbi == 'https://example.org/archive/clinvar_20250101.vcf.gz.tbi' }, + { assert function.result.prepare['vep_clinvar'].md5 == '0123456789abcdef0123456789abcdef' }, + // never staged as a foreign file: every VEP task would re-check it on the host + { assert function.result.ready_files == [] } + ) + } + } + + test("vepPluginNeedsFetch is false for a cloud-storage ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 's3://my-bucket/clinvar_20250101.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + // wget cannot fetch s3://, so Nextflow keeps staging it as before + { assert !function.result } + ) + } + } + + test("vepPluginNeedsFetch is true for an ftp ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == true } + ) + } + } + + test("vepPluginIndexMd5 takes the default index MD5 on CHM13") { + + function "vepPluginIndexMd5" + + when { + params { + genome = 'CHM13' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result ==~ /[0-9a-f]{32}/ } + ) + } + } + + test("vepPluginIndexMd5 is null on GRCh38, where NCBI publishes none") { + + function "vepPluginIndexMd5" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == null } + ) + } + } + + test("vepPluginIndexMd5 drops the default once the index is overridden") { + + function "vepPluginIndexMd5" + + when { + params { + genome = 'CHM13' + vep_clinvar_tbi = 'https://example.org/clinvar_20240624_GCA_009914755.4.vcf.gz.tbi' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + // the default MD5 belongs to the default index, not to this one + { assert function.result == null } + ) + } + } + + test("resolveVepPlugins carries the index MD5 to the download task") { + + function "resolveVepPlugins" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/archive/clinvar_20250101.vcf.gz' + vep_clinvar_tbi = 'https://example.org/archive/clinvar_20250101.vcf.gz.tbi' + vep_clinvar_tbi_md5 = 'fedcba9876543210fedcba9876543210' + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.prepare['vep_clinvar'].tbi_md5 == 'fedcba9876543210fedcba9876543210' } + ) + } + } + // --- validateVepPluginParams: the error paths test("validateVepPluginParams passes when plugins are skipped") { @@ -579,4 +894,94 @@ nextflow_function { assert function.success } } + + test("validateVepPluginParams rejects an MD5 for a local ClinVar") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = '/data/mine/clinvar.vcf.gz' + vep_clinvar_tbi = '/data/mine/clinvar.vcf.gz.tbi' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + // only the download task checks it, so a local file would skip it silently + { assert function.failed }, + { assert function.stdout.any { line -> line.contains('vep_clinvar_md5') } } + ) + } + } + + test("validateVepPluginParams warns on a remote ClinVar without an MD5") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/clinvar.vcf.gz' + vep_clinvar_tbi = 'https://example.org/clinvar.vcf.gz.tbi' + } + } + + then { + assertAll( + { assert function.success }, + // log.warn reaches the log file, not the captured stdout + { assert path("${metaDir}/nextflow.log").text.contains('not verified') } + ) + } + } + + test("validateVepPluginParams rejects a remote ClinVar with a local index") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/clinvar_20250101.vcf.gz' + vep_clinvar_tbi = '/data/mine/clinvar_20250101.vcf.gz.tbi' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + // the download task fetches both, so wget would be handed a local path + { assert function.failed }, + { assert function.stdout.any { line -> line.contains('vep_clinvar_tbi') } } + ) + } + } + + test("validateVepPluginParams rejects an index MD5 for a local ClinVar") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = '/data/mine/clinvar.vcf.gz' + vep_clinvar_tbi = '/data/mine/clinvar.vcf.gz.tbi' + vep_clinvar_tbi_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + { assert function.failed }, + { assert function.stdout.any { line -> line.contains('vep_clinvar_tbi_md5') } } + ) + } + } }