From 07473ccd6f9b0204b3148af0691b88763a40a0dd Mon Sep 17 00:00:00 2001 From: Amber Verhasselt Date: Fri, 25 Sep 2026 11:06:24 +0200 Subject: [PATCH 1/4] Download ClinVar once in PREPARE_VEP_PLUGINS, pinned by MD5 A remote ClinVar was staged as a Nextflow foreign file, which every VEP task re-checks on its host with a HEAD request. On a multi-sample GRCh38 run NCBI answers that burst with HTTP 503, so SOMATIC_VEP failed with "Can't stage file ...clinvar_20260829.vcf.gz", and -resume could not recover because the failed check changed the staging cache key. The new VEPPLUGIN_CLINVAR step downloads the VCF and its index once per run, retrying 503s, and checks the VCF against --vep_clinvar_md5. The GRCh38 and CHM13 defaults carry the MD5 their hosts publish, so the pinned release cannot change silently; a user URL without an MD5 warns, and an MD5 with a local file is rejected. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 1 + conf/igenomes.config | 6 +- conf/modules.config | 9 +- docs/usage.md | 13 +- .../local/vepplugin/clinvar/environment.yml | 8 + modules/local/vepplugin/clinvar/main.nf | 55 +++++ modules/local/vepplugin/clinvar/meta.yml | 63 +++++ .../vepplugin/clinvar/tests/main.nf.test | 38 +++ .../vepplugin/clinvar/tests/main.nf.test.snap | 39 ++++ nextflow.config | 1 + nextflow_schema.json | 7 + subworkflows/local/prepare_vep_plugins.nf | 21 +- .../utils_nfcore_lrsomatic_pipeline/main.nf | 36 +++ .../tests/main.function.nf.test | 218 ++++++++++++++++++ 14 files changed, 508 insertions(+), 7 deletions(-) create mode 100644 modules/local/vepplugin/clinvar/environment.yml create mode 100644 modules/local/vepplugin/clinvar/main.nf create mode 100644 modules/local/vepplugin/clinvar/meta.yml create mode 100644 modules/local/vepplugin/clinvar/tests/main.nf.test create mode 100644 modules/local/vepplugin/clinvar/tests/main.nf.test.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index b80c26bb..c94a5c74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#PR](https://github.com/IntGenomicsLab/lrsomatic/pull/PR) - A remote ClinVar is now downloaded once per run by the new `VEPPLUGIN_CLINVAR` step in `PREPARE_VEP_PLUGINS`, instead of being staged by Nextflow as a foreign file. Every VEP task re-checked a foreign file on its host, and NCBI answered the burst from a multi-sample GRCh38 run with HTTP 503, so `SOMATIC_VEP` failed with `Can't stage file ...clinvar_20260829.vcf.gz`; `-resume` could not recover, since the failed check changed the staging cache key. The download is checked against the new `--vep_clinvar_md5`, set by default to the MD5 NCBI (GRCh38) and Ensembl (CHM13) publish, so the pinned release cannot change silently. The ClinVar sizes in `docs/usage.md` are also corrected (@AmberVerhasselt). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `CLAIRS` no longer runs with `--haplotagged_tumor_bam_provided_so_skip_intermediate_phasing_and_haplotagging`. Since somatic calling was moved ahead of `PHASING_HAPLOTYPING` (v1.1.0), ClairS has received the untagged minimap2 BAM, so the flag made it skip its own phasing and haplotagging and call every paired sample without haplotype information: the full-alignment model saw no `HP` tags and the haplotype filtering step had nothing to filter on, the same as `--disable_phasing`. ClairS now runs Clair3 on the normal and tumour BAMs and phases the tumour itself. **Paired somatic calls change** (fewer false positives expected), and `CLAIRS` takes longer and uses more work-directory space (@ljwharbers). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `docs/output.md` now lists the ClairS SNV output as `snvs.vcf.gz`, the name the pipeline publishes, instead of `snv.vcf.gz` (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). diff --git a/conf/igenomes.config b/conf/igenomes.config index 71ff9f59..8924def0 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -28,9 +28,11 @@ params.genomes = [ vep_alphamissense_tbi : "https://g-608c0c.273595.03c0.data.globus.org/VEP_plugins/AlphaMissense_hg38.tsv.gz.tbi", // A dated release rather than the rolling vcf_GRCh38/clinvar.vcf.gz, whose VCF and // separately-downloaded index both move weekly: two runs would otherwise annotate - // differently, and one straddling an update could pair a new VCF with an old index + // differently, and one straddling an update could pair a new VCF with an old index. + // Downloaded once per run by VEPPLUGIN_CLINVAR, which checks it against NCBI's published MD5 vep_clinvar : "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz", vep_clinvar_tbi : "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz.tbi", + vep_clinvar_md5 : "dffae274fd559ee764d1de44f36df2cf", // CADD is opt-in: the SNV table alone is 81 GB, and Nextflow's foreign-file cache is // per session, so every non-resumed run would re-stage it. Enable with --vep_cadd_snv // and --vep_cadd_indel (each with its _tbi); see docs/usage.md. @@ -60,6 +62,8 @@ params.genomes = [ vep_polyphen_sift_db : "https://ftp.ensembl.org/pub/release-115/variation/pangenomes/Human/homo_sapiens_pangenome_PolyPhen_SIFT_20240502.db", vep_clinvar : "https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/ensembl/variation/2022_10/vcf/2024_07/clinvar_20240624_GCA_009914755.4.vcf.gz", vep_clinvar_tbi : "https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/ensembl/variation/2022_10/vcf/2024_07/clinvar_20240624_GCA_009914755.4.vcf.gz.tbi", + // From the CHECKSUMS file Ensembl publishes alongside the VCF + vep_clinvar_md5 : "98bde67975c5f0b95230672f642a3c07", // SigProfilerMatrixGenerator CHM13-T2T payload (SigProfilerSuite/SigProfilerMatrixGenerator#250); // not yet hosted on the AlexandrovLab FTP, so it is fetched from the IntGenomicsLab Globus collection sigprofiler_genome : "CHM13-T2T", diff --git a/conf/modules.config b/conf/modules.config index bd0ef4f1..90cc1c56 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -675,9 +675,14 @@ process { time = { 24.h * task.attempt } } + // The wget container carries no CA bundle, as for WGET above; the pinned MD5 still checks the download + withName : '.*:VEPPLUGIN_CLINVAR' { + ext.args = { "--no-check-certificate" } + } + // Published so a later run can skip both the download and the reshaping by pointing - // --vep_revel / --vep_eve (and their _tbi) at these files - withName : '.*:VEPPLUGIN_(REVEL|EVE)' { + // --vep_revel / --vep_eve / --vep_clinvar (and their _tbi) at these files + withName : '.*:VEPPLUGIN_(REVEL|EVE|CLINVAR)' { publishDir = [ mode: params.publish_dir_mode, path: { "${params.outdir}/vep_plugins" } diff --git a/docs/usage.md b/docs/usage.md index c9fe4d26..5333398a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -223,6 +223,7 @@ opt-in. See [VEP plugins](#vep-plugins) for sizes, licence terms and per-assembl | `--vep_polyphen_sift_db` | Ensembl pangenome PolyPhen/SIFT SQLite database, for the `PolyPhen_SIFT` plugin. Needed on CHM13 only | | `--vep_clinvar` | ClinVar VCF, added as a VEP `--custom` annotation | | `--vep_clinvar_tbi` | Index for `--vep_clinvar`. Required whenever `--vep_clinvar` is set | +| `--vep_clinvar_md5` | Expected MD5 of a remote `--vep_clinvar`, checked after download. Dropped when `--vep_clinvar` is overridden | | `--vep_clinvar_fields` | Comma-separated ClinVar INFO fields to carry through. Default = `"CLNSIG,CLNREVSTAT,CLNDN"` | | `--vep_cadd_snv` | CADD SNV score file, for the `CADD` plugin. No default — 81 GB, so opt-in; prefer a local path. GRCh38 only | | `--vep_cadd_snv_tbi` | Index for `--vep_cadd_snv`. Required whenever `--vep_cadd_snv` is set | @@ -468,7 +469,7 @@ Plugins are applied to the germline and somatic VEP runs, not to the structural- | **SIFT** | already in the VEP cache, no file needed | `PolyPhen_SIFT` plugin | CHM13: 13 GB database | | **PolyPhen** | already in the VEP cache, no file needed | `PolyPhen_SIFT` plugin | as above, the same database | | **AlphaMissense** | `AlphaMissense` plugin | `AlphaMissenseProtein` plugin | 613 MB (GRCh38) / 1.1 GB (CHM13) | -| **ClinVar** | `--custom` annotation | `--custom`, CHM13-lifted VCF | 105 MB (GRCh38) / 190 MB (CHM13) | +| **ClinVar** | `--custom` annotation | `--custom`, CHM13-lifted VCF | 193 MB (GRCh38) / 99 MB (CHM13) | | **CADD** | `CADD` plugin, opt-in | not available — see below | none — `--vep_cadd_snv` enables it (81 GB + 1.2 GB) | | **REVEL** | `REVEL` plugin | not available — see below | 667 MB release zip | | **EVE** | `EVE` plugin, opt-in | not available — see below | none — `--vep_eve` enables it (9.6 GB) | @@ -527,6 +528,12 @@ Whether an index is required depends on the shape of what you supply: - **AlphaMissense, ClinVar and CADD** are used exactly as given, so their `_tbi` parameter is always required alongside them. Overriding a data file drops the default index: supply both, or neither. +- **A remote ClinVar** is downloaded once per run by `VEPPLUGIN_CLINVAR`, with its index, rather + than staged by every VEP task: NCBI answers the burst of requests a multi-sample run sends with + HTTP 503. The download is checked against `--vep_clinvar_md5`, so a release re-published under the + same name fails the run instead of changing the annotation. Both defaults carry the MD5 their host + publishes; with your own URL, pass its MD5 too, or the pipeline warns that the release is not + verified. The files are published to `/vep_plugins/` for reuse as local paths. - **REVEL and EVE** ship as zip archives. Pass a `.zip` and the pipeline unpacks and reshapes it; pass a prepared file and its index to use it directly. Remote zips are fetched with `wget` rather than staged by Nextflow, and EVE's 9.6 GB archive serves slowly, so expect hours. @@ -538,8 +545,8 @@ Whether an index is required depends on the shape of what you supply: | AlphaMissense (GRCh38) | `https://storage.googleapis.com/dm_alphamissense/AlphaMissense_hg38.tsv.gz`, with an index we host | 613 MB | no, the index is fetched ready | CC BY 4.0 | | AlphaMissense (protein) | a gene-symbol-keyed table we host, built from the AlphaMissense protein-space release | 1.1 GB | no, fetched ready — see `CITATIONS.md` | CC BY 4.0 | | Pangenome PolyPhen/SIFT | `https://ftp.ensembl.org/pub/release-115/variation/pangenomes/Human/homo_sapiens_pangenome_PolyPhen_SIFT_20240502.db` | 13 GB | no, it is an SQLite database | Ensembl / EMBL-EBI open | -| ClinVar (GRCh38) | `clinvar_20260829.vcf.gz` under `https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/` (+ `.tbi`) | 105 MB | no, `.tbi` is published | public domain | -| ClinVar (CHM13) | `clinvar_20240624_GCA_009914755.4.vcf.gz` under `https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/` | 190 MB | no, `.tbi` is published | public domain | +| ClinVar (GRCh38) | `clinvar_20260829.vcf.gz` under `https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/` (+ `.tbi`) | 193 MB | downloaded once, MD5-checked | public domain | +| ClinVar (CHM13) | `clinvar_20240624_GCA_009914755.4.vcf.gz` under `https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/` | 99 MB | downloaded once, MD5-checked | public domain | | CADD v1.7 SNVs (opt-in) | `https://krishna.gs.washington.edu/download/CADD/v1.7/GRCh38/whole_genome_SNVs.tsv.gz` (+ `.tbi`) | 81 GB | no, `.tbi` is published | free for non-commercial use | | CADD v1.7 indels (opt-in) | `https://krishna.gs.washington.edu/download/CADD/v1.7/GRCh38/gnomad.genomes.r4.0.indel.tsv.gz` (+ `.tbi`) | 1.2 GB | no, `.tbi` is published | free for non-commercial use | | REVEL v1.3 | `https://rothsj06.dmz.hpc.mssm.edu/revel-v1.3_all_chromosomes.zip` | 667 MB | unpacked, re-sorted on GRCh38, indexed | free for non-commercial use | diff --git a/modules/local/vepplugin/clinvar/environment.yml b/modules/local/vepplugin/clinvar/environment.yml new file mode 100644 index 00000000..daa07275 --- /dev/null +++ b/modules/local/vepplugin/clinvar/environment.yml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + # renovate: datasource=conda depName=conda-forge/wget + - conda-forge::wget=1.21.4 diff --git a/modules/local/vepplugin/clinvar/main.nf b/modules/local/vepplugin/clinvar/main.nf new file mode 100644 index 00000000..c4b9d1fb --- /dev/null +++ b/modules/local/vepplugin/clinvar/main.nf @@ -0,0 +1,55 @@ +process VEPPLUGIN_CLINVAR { + tag "${vcf_url.toString().tokenize('/').last()}" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container + ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/3b/3b54fa9135194c72a18d00db6b399c03248103f87e43ca75e4b50d61179994b3/data' + : 'community.wave.seqera.io/library/wget:1.21.4--8b0fcde81c17be5e'}" + + input: + tuple val(vcf_url), val(tbi_url), val(md5) + + output: + path "${vcf_name}{,.tbi}", emit: files + tuple val("${task.process}"), val('wget'), eval("wget --version | head -1 | cut -d ' ' -f 3"), topic: versions, emit: versions_wget + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + vcf_name = vcf_url.toString().tokenize('/').last() + def check = md5 ? "echo '${md5} ${vcf_name}' | md5sum -c -" : '' + """ + # NCBI answers bursts with 503, so retry those rather than fail on the first one + wget \\ + --no-verbose \\ + --tries=5 \\ + --waitretry=30 \\ + --retry-on-http-error=429,500,502,503,504 \\ + ${args} \\ + -O ${vcf_name} \\ + ${vcf_url} + + # Saved next to the VCF under the name VEP looks for, whatever the host calls it + wget \\ + --no-verbose \\ + --tries=5 \\ + --waitretry=30 \\ + --retry-on-http-error=429,500,502,503,504 \\ + ${args} \\ + -O ${vcf_name}.tbi \\ + ${tbi_url} + + # A pinned checksum keeps the release fixed: a host that re-publishes under the same name fails here + ${check} + """ + + stub: + vcf_name = vcf_url.toString().tokenize('/').last() + """ + echo "" | gzip > ${vcf_name} + touch ${vcf_name}.tbi + """ +} diff --git a/modules/local/vepplugin/clinvar/meta.yml b/modules/local/vepplugin/clinvar/meta.yml new file mode 100644 index 00000000..3ccffb78 --- /dev/null +++ b/modules/local/vepplugin/clinvar/meta.yml @@ -0,0 +1,63 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "vepplugin_clinvar" +description: Download a dated ClinVar VCF and its index once per run, checked against a pinned MD5, for the VEP --custom annotation +keywords: + - vep + - clinvar + - download + - annotation +tools: + - "wget": + description: "GNU Wget is a free software package for retrieving files using HTTP, HTTPS, FTP and FTPS" + homepage: "https://www.gnu.org/software/wget/" + documentation: "https://www.gnu.org/software/wget/manual/wget.html" + licence: ["GPL-3.0-or-later"] + identifier: "" + +input: + - - vcf_url: + type: string + description: URL of the ClinVar VCF; its basename becomes the output name + - tbi_url: + type: string + description: URL of the tabix index published alongside the VCF + - md5: + type: string + description: | + Expected MD5 of the VCF. When set, a download that does not match fails + the task. May be null, in which case the release is not verified. + +output: + files: + - "${vcf_name}{,.tbi}": + type: file + description: The ClinVar VCF and its tabix index + pattern: "*.vcf.gz{,.tbi}" + versions_wget: + - - ${task.process}: + type: string + description: The process the versions were collected from + - wget: + type: string + description: The tool name + - "wget --version | head -1 | cut -d ' ' -f 3": + type: string + description: The command used to generate the version of the tool + +topics: + versions: + - - ${task.process}: + type: string + description: The process the versions were collected from + - wget: + type: string + description: The tool name + - "wget --version | head -1 | cut -d ' ' -f 3": + type: string + description: The command used to generate the version of the tool + +authors: + - "@AmberVerhasselt" +maintainers: + - "@AmberVerhasselt" diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test b/modules/local/vepplugin/clinvar/tests/main.nf.test new file mode 100644 index 00000000..4f0806c2 --- /dev/null +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test @@ -0,0 +1,38 @@ +nextflow_process { + + name "Test Process VEPPLUGIN_CLINVAR" + script "../main.nf" + process "VEPPLUGIN_CLINVAR" + + tag "modules" + tag "modules_local" + tag "vepplugin_clinvar" + + // Stub only: a real run downloads the full ClinVar release + test("clinvar release - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz', + 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz.tbi', + 'dffae274fd559ee764d1de44f36df2cf' + ] + """ + } + } + + then { + assert process.success + assertAll( + // the VCF keeps its release name, and the index sits beside it under the name VEP looks for + { assert process.out.files[0].collect { f -> file(f).name }.sort() == [ 'clinvar_20260829.vcf.gz', 'clinvar_20260829.vcf.gz.tbi' ] }, + { assert snapshot(process.out).match() } + ) + } + + } +} diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test.snap b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap new file mode 100644 index 00000000..c7a226d1 --- /dev/null +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap @@ -0,0 +1,39 @@ +{ + "clinvar release - stub": { + "content": [ + { + "0": [ + [ + "clinvar_20260829.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "clinvar_20260829.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + "VEPPLUGIN_CLINVAR", + "wget", + "1.21.4" + ] + ], + "files": [ + [ + "clinvar_20260829.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "clinvar_20260829.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions_wget": [ + [ + "VEPPLUGIN_CLINVAR", + "wget", + "1.21.4" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2026-09-25T10:53:31.340624257" + } +} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 696d5161..399c9584 100644 --- a/nextflow.config +++ b/nextflow.config @@ -58,6 +58,7 @@ params { vep_polyphen_sift_db = null vep_clinvar = null vep_clinvar_tbi = null + vep_clinvar_md5 = null vep_clinvar_fields = "CLNSIG,CLNREVSTAT,CLNDN" vep_cadd_snv = null vep_cadd_snv_tbi = null diff --git a/nextflow_schema.json b/nextflow_schema.json index 713e2baa..1e6c4720 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -283,6 +283,13 @@ "fa_icon": "fas fa-file", "help_text": "Required whenever --vep_clinvar is set: ClinVar publishes a .tbi alongside every VCF, so the pipeline does not build one." }, + "vep_clinvar_md5": { + "type": "string", + "description": "Expected MD5 of the --vep_clinvar VCF, checked after the pipeline downloads it.", + "fa_icon": "fas fa-fingerprint", + "pattern": "^[0-9a-fA-F]{32}$", + "help_text": "A remote --vep_clinvar is downloaded once per run by VEPPLUGIN_CLINVAR, rather than staged by every VEP task, and a VCF that does not match this MD5 fails the run, so the release cannot change silently. The GRCh38 and CHM13 defaults carry the MD5 their hosts publish; overriding --vep_clinvar drops it. Only applies to a URL: a local --vep_clinvar is staged as given." + }, "vep_clinvar_fields": { "type": "string", "description": "Comma-separated ClinVar INFO fields to carry into the annotation.", diff --git a/subworkflows/local/prepare_vep_plugins.nf b/subworkflows/local/prepare_vep_plugins.nf index d445a7f6..9303d4b7 100644 --- a/subworkflows/local/prepare_vep_plugins.nf +++ b/subworkflows/local/prepare_vep_plugins.nf @@ -1,5 +1,6 @@ // -// Reshape the VEP plugin releases that ship as zip archives (REVEL and EVE) +// Reshape the VEP plugin releases that ship as zip archives (REVEL and EVE), and download a remote +// ClinVar once so the VEP tasks never stage it from its host themselves // include { UNZIP as UNZIP_REVEL } from '../../modules/nf-core/unzip/main.nf' @@ -8,6 +9,7 @@ include { WGET as WGET_REVEL } from '../../modules/nf-core/wget/main' include { WGET as WGET_EVE } from '../../modules/nf-core/wget/main' include { VEPPLUGIN_REVEL } from '../../modules/local/vepplugin/revel/main.nf' include { VEPPLUGIN_EVE } from '../../modules/local/vepplugin/eve/main.nf' +include { VEPPLUGIN_CLINVAR } from '../../modules/local/vepplugin/clinvar/main.nf' workflow PREPARE_VEP_PLUGINS { @@ -82,6 +84,23 @@ workflow PREPARE_VEP_PLUGINS { ch_versions = ch_versions.mix(UNZIP_EVE.out.versions) } + // + // MODULE: VEPPLUGIN_CLINVAR (label: process_single) + // Input: the ClinVar VCF and index URLs, and the MD5 pinning the release (or null) + // Output: .files -- the VCF and its index, under the VCF's own basename + // One download per run: a foreign file is re-checked on its host by every VEP task, and NCBI + // answers the burst a multi-sample run sends with 503 + // + if (prepare.containsKey('vep_clinvar')) { + def clinvar = prepare['vep_clinvar'] + + VEPPLUGIN_CLINVAR ( + channel.value([ clinvar.vcf, clinvar.tbi, clinvar.md5 ]) + ) + + staged << VEPPLUGIN_CLINVAR.out.files + } + // Value channel read by both VEP tasks; ifEmpty carries the no-plugins case, since collect() emits nothing then ch_extra_files = staged .inject(channel.empty()) { acc, ch -> acc.mix(ch) } diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index a64c405c..fcf50d45 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -408,6 +408,28 @@ def vepPluginNeedsPrep(data_param) { !value.toString().toLowerCase().endsWith('.gz') } +// +// Whether a resource is downloaded once by a prep task instead of staged as a foreign file +// +// Only a remote ClinVar is: a foreign file is re-checked on its host by every VEP task that takes +// it, and NCBI answers the burst a multi-sample run sends with 503, which fails the staging. +// +def vepPluginNeedsFetch(data_param) { + def value = vepPluginResource(data_param) + return value && + data_param == 'vep_clinvar' && + value.toString().contains('://') +} + +// +// The expected MD5 of a fetched resource. Overriding the data file drops the default MD5, +// which belongs to a different release +// +def vepPluginMd5(data_param) { + def md5_param = "${data_param}_md5".toString() + return params[md5_param] ?: (params[data_param] ? null : getGenomeAttribute(md5_param)) +} + // // The filename a prep task writes, referenced by the VEP argument since plugins stage into the task root // @@ -474,6 +496,14 @@ def validateVepPluginParams() { if (params.vep_custom && !(params.vep_args =~ /--custom file=/)) { error("--vep_custom: needs a matching '--custom file=...' entry in --vep_args, which is where the staged file is substituted in. Add one, e.g. --vep_args '${params.vep_args} --custom file=placeholder,short_name=MyTrack,format=vcf,type=exact,coords=0'.") } + + // The MD5 is checked by the download task, so a local ClinVar would silently skip it + if (params.vep_clinvar_md5 && !vepPluginNeedsFetch('vep_clinvar')) { + error("--vep_clinvar_md5: only checks a ClinVar the pipeline downloads, so it needs --vep_clinvar to be a URL. Drop --vep_clinvar_md5 for a local file.") + } + if (vepPluginNeedsFetch('vep_clinvar') && !vepPluginMd5('vep_clinvar')) { + log.warn("--vep_clinvar: '${vepPluginResource('vep_clinvar')}' is downloaded without --vep_clinvar_md5, so its release is not verified: a host that re-publishes under the same name, like the rolling clinvar.vcf.gz, changes the annotation between runs.") + } } // @@ -494,8 +524,14 @@ def stageVepPluginFile(staged, data_param) { // // A resource needing prep is kept as its raw value rather than a file(), since neither the REVEL nor // the EVE host can be staged by Nextflow -- PREPARE_VEP_PLUGINS fetches those with WGET instead. +// A remote ClinVar is recorded with its index and MD5, and keeps its own basename. // def registerVepPlugin(staged, prepare, data_param) { + if (vepPluginNeedsFetch(data_param)) { + def url = vepPluginResource(data_param).toString() + prepare[data_param] = [ vcf: url, tbi: vepPluginIndex(data_param), md5: vepPluginMd5(data_param) ] + return url.tokenize('/').last() + } if (vepPluginNeedsPrep(data_param)) { prepare[data_param] = vepPluginResource(data_param) return vepPluginPreparedName(data_param) diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test index d98daa90..2d302b88 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test @@ -413,6 +413,179 @@ nextflow_function { } } + // --- vepPluginNeedsFetch / vepPluginMd5: a remote ClinVar is downloaded once, against a pinned MD5 + + test("vepPluginNeedsFetch is true for the default remote ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == true } + ) + } + } + + test("vepPluginNeedsFetch is false for a local ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + vep_clinvar = '/data/mine/clinvar.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert !function.result } + ) + } + } + + test("vepPluginNeedsFetch is false for a remote resource other than ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_alphamissense' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert !function.result } + ) + } + } + + test("vepPluginMd5 takes the default MD5 when ClinVar is the default") { + + function "vepPluginMd5" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result ==~ /[0-9a-f]{32}/ } + ) + } + } + + test("vepPluginMd5 drops the default MD5 once ClinVar is overridden") { + + function "vepPluginMd5" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 'https://example.org/clinvar_20250101.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + // the default MD5 belongs to the default release, not to this one + { assert function.result == null } + ) + } + } + + test("vepPluginMd5 returns an explicitly supplied MD5") { + + function "vepPluginMd5" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 'https://example.org/clinvar_20250101.vcf.gz' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == '0123456789abcdef0123456789abcdef' } + ) + } + } + + test("resolveVepPlugins defers a remote ClinVar to the download task") { + + function "resolveVepPlugins" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/archive/clinvar_20250101.vcf.gz' + vep_clinvar_tbi = 'https://example.org/archive/clinvar_20250101.vcf.gz.tbi' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + { assert function.success }, + // the argument keeps the release's own basename, which the download task writes + { assert function.result.args.startsWith('--custom file=clinvar_20250101.vcf.gz,short_name=ClinVar') }, + { assert function.result.prepare['vep_clinvar'].vcf == 'https://example.org/archive/clinvar_20250101.vcf.gz' }, + { assert function.result.prepare['vep_clinvar'].tbi == 'https://example.org/archive/clinvar_20250101.vcf.gz.tbi' }, + { assert function.result.prepare['vep_clinvar'].md5 == '0123456789abcdef0123456789abcdef' }, + // never staged as a foreign file: every VEP task would re-check it on the host + { assert function.result.ready_files == [] } + ) + } + } + // --- validateVepPluginParams: the error paths test("validateVepPluginParams passes when plugins are skipped") { @@ -579,4 +752,49 @@ nextflow_function { assert function.success } } + + test("validateVepPluginParams rejects an MD5 for a local ClinVar") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = '/data/mine/clinvar.vcf.gz' + vep_clinvar_tbi = '/data/mine/clinvar.vcf.gz.tbi' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + // only the download task checks it, so a local file would skip it silently + { assert function.failed }, + { assert function.stdout.any { line -> line.contains('vep_clinvar_md5') } } + ) + } + } + + test("validateVepPluginParams warns on a remote ClinVar without an MD5") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/clinvar.vcf.gz' + vep_clinvar_tbi = 'https://example.org/clinvar.vcf.gz.tbi' + } + } + + then { + assertAll( + { assert function.success }, + // log.warn reaches the log file, not the captured stdout + { assert path("${metaDir}/nextflow.log").text.contains('not verified') } + ) + } + } } From 941567fa43990ce1531da7bbd62ddee072586c6b Mon Sep 17 00:00:00 2001 From: Amber Verhasselt Date: Fri, 25 Sep 2026 11:43:34 +0200 Subject: [PATCH 2/4] Report VEPPLUGIN_CLINVAR's version through versions.yml, not eval() Nextflow numbers eval() outputs across the whole pipeline and hashes that number, so one new eval() output shifts the cache key of every task that has one: a -resume of an existing run re-ran everything from METAEXTRACT on. A versions.yml path output leaves other tasks' hashes untouched. Co-Authored-By: Claude Opus 5.5 (1M context) --- modules/local/vepplugin/clinvar/main.nf | 14 ++++++++++- modules/local/vepplugin/clinvar/meta.yml | 25 +++---------------- .../vepplugin/clinvar/tests/main.nf.test.snap | 16 +++--------- subworkflows/local/prepare_vep_plugins.nf | 1 + 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/modules/local/vepplugin/clinvar/main.nf b/modules/local/vepplugin/clinvar/main.nf index c4b9d1fb..1a9290a1 100644 --- a/modules/local/vepplugin/clinvar/main.nf +++ b/modules/local/vepplugin/clinvar/main.nf @@ -12,7 +12,9 @@ process VEPPLUGIN_CLINVAR { output: path "${vcf_name}{,.tbi}", emit: files - tuple val("${task.process}"), val('wget'), eval("wget --version | head -1 | cut -d ' ' -f 3"), topic: versions, emit: versions_wget + // versions.yml rather than an eval() topic: eval outputs are numbered pipeline-wide, so adding + // one shifts the cache key of every other task that has one and breaks -resume of existing runs + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -44,6 +46,11 @@ process VEPPLUGIN_CLINVAR { # A pinned checksum keeps the release fixed: a host that re-publishes under the same name fails here ${check} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wget: \$(wget --version | head -1 | cut -d ' ' -f 3) + END_VERSIONS """ stub: @@ -51,5 +58,10 @@ process VEPPLUGIN_CLINVAR { """ echo "" | gzip > ${vcf_name} touch ${vcf_name}.tbi + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + wget: \$(wget --version | head -1 | cut -d ' ' -f 3) + END_VERSIONS """ } diff --git a/modules/local/vepplugin/clinvar/meta.yml b/modules/local/vepplugin/clinvar/meta.yml index 3ccffb78..dc6a6eb6 100644 --- a/modules/local/vepplugin/clinvar/meta.yml +++ b/modules/local/vepplugin/clinvar/meta.yml @@ -34,28 +34,11 @@ output: type: file description: The ClinVar VCF and its tabix index pattern: "*.vcf.gz{,.tbi}" - versions_wget: - - - ${task.process}: - type: string - description: The process the versions were collected from - - wget: - type: string - description: The tool name - - "wget --version | head -1 | cut -d ' ' -f 3": - type: string - description: The command used to generate the version of the tool - -topics: versions: - - - ${task.process}: - type: string - description: The process the versions were collected from - - wget: - type: string - description: The tool name - - "wget --version | head -1 | cut -d ' ' -f 3": - type: string - description: The command used to generate the version of the tool + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@AmberVerhasselt" diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test.snap b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap index c7a226d1..e31faa30 100644 --- a/modules/local/vepplugin/clinvar/tests/main.nf.test.snap +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap @@ -9,11 +9,7 @@ ] ], "1": [ - [ - "VEPPLUGIN_CLINVAR", - "wget", - "1.21.4" - ] + "versions.yml:md5,ca06b557f10c3af9a0e088e9a76b0580" ], "files": [ [ @@ -21,12 +17,8 @@ "clinvar_20260829.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "versions_wget": [ - [ - "VEPPLUGIN_CLINVAR", - "wget", - "1.21.4" - ] + "versions": [ + "versions.yml:md5,ca06b557f10c3af9a0e088e9a76b0580" ] } ], @@ -34,6 +26,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T10:53:31.340624257" + "timestamp": "2026-09-25T11:40:31.870446909" } } \ No newline at end of file diff --git a/subworkflows/local/prepare_vep_plugins.nf b/subworkflows/local/prepare_vep_plugins.nf index 9303d4b7..4803c543 100644 --- a/subworkflows/local/prepare_vep_plugins.nf +++ b/subworkflows/local/prepare_vep_plugins.nf @@ -99,6 +99,7 @@ workflow PREPARE_VEP_PLUGINS { ) staged << VEPPLUGIN_CLINVAR.out.files + ch_versions = ch_versions.mix(VEPPLUGIN_CLINVAR.out.versions) } // Value channel read by both VEP tasks; ifEmpty carries the no-plugins case, since collect() emits nothing then From 39628257c91208eab6f1903547b729618fc79411 Mon Sep 17 00:00:00 2001 From: Amber Verhasselt Date: Fri, 25 Sep 2026 13:38:35 +0200 Subject: [PATCH 3/4] Fill in PR number in CHANGELOG entry Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c94a5c74..c0e51743 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#PR](https://github.com/IntGenomicsLab/lrsomatic/pull/PR) - A remote ClinVar is now downloaded once per run by the new `VEPPLUGIN_CLINVAR` step in `PREPARE_VEP_PLUGINS`, instead of being staged by Nextflow as a foreign file. Every VEP task re-checked a foreign file on its host, and NCBI answered the burst from a multi-sample GRCh38 run with HTTP 503, so `SOMATIC_VEP` failed with `Can't stage file ...clinvar_20260829.vcf.gz`; `-resume` could not recover, since the failed check changed the staging cache key. The download is checked against the new `--vep_clinvar_md5`, set by default to the MD5 NCBI (GRCh38) and Ensembl (CHM13) publish, so the pinned release cannot change silently. The ClinVar sizes in `docs/usage.md` are also corrected (@AmberVerhasselt). +- [#206](https://github.com/IntGenomicsLab/lrsomatic/pull/206) - A remote ClinVar is now downloaded once per run by the new `VEPPLUGIN_CLINVAR` step in `PREPARE_VEP_PLUGINS`, instead of being staged by Nextflow as a foreign file. Every VEP task re-checked a foreign file on its host, and NCBI answered the burst from a multi-sample GRCh38 run with HTTP 503, so `SOMATIC_VEP` failed with `Can't stage file ...clinvar_20260829.vcf.gz`; `-resume` could not recover, since the failed check changed the staging cache key. The download is checked against the new `--vep_clinvar_md5`, set by default to the MD5 NCBI (GRCh38) and Ensembl (CHM13) publish, so the pinned release cannot change silently. The ClinVar sizes in `docs/usage.md` are also corrected (@AmberVerhasselt). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `CLAIRS` no longer runs with `--haplotagged_tumor_bam_provided_so_skip_intermediate_phasing_and_haplotagging`. Since somatic calling was moved ahead of `PHASING_HAPLOTYPING` (v1.1.0), ClairS has received the untagged minimap2 BAM, so the flag made it skip its own phasing and haplotagging and call every paired sample without haplotype information: the full-alignment model saw no `HP` tags and the haplotype filtering step had nothing to filter on, the same as `--disable_phasing`. ClairS now runs Clair3 on the normal and tumour BAMs and phases the tumour itself. **Paired somatic calls change** (fewer false positives expected), and `CLAIRS` takes longer and uses more work-directory space (@ljwharbers). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `docs/output.md` now lists the ClairS SNV output as `snvs.vcf.gz`, the name the pipeline publishes, instead of `snv.vcf.gz` (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). From a327baa57cd5ecaf3c79ca86f7ca8eb2e36d642b Mon Sep 17 00:00:00 2001 From: Amber Verhasselt Date: Fri, 25 Sep 2026 16:24:17 +0200 Subject: [PATCH 4/4] Address review on #206: fetch only http(s)/ftp, pin the CHM13 index - Don't publish VEPPLUGIN_CLINVAR's versions.yml to vep_plugins/: the publishDir override lacked the default saveAs. - Fetch ClinVar only for http(s)/ftp URLs; s3://, gs://, az:// and file:// are staged by Nextflow again, as before this PR. - Reject a downloaded ClinVar paired with a local --vep_clinvar_tbi, which wget could not fetch. - Add --vep_clinvar_tbi_md5, pinned by default to the MD5 Ensembl publishes for the CHM13 index; NCBI publishes none for GRCh38. - wget --tries=10 (about 45 s of linear backoff), tag the module test "small" so PR CI runs it, and document vep_plugins/ in output.md. - Correct the wording: only GERMLINE_VEP and SOMATIC_VEP staged ClinVar, never SV_VEP; note the one-off VEP re-run on resume. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- conf/igenomes.config | 4 +- conf/modules.config | 3 +- docs/output.md | 25 +++ docs/usage.md | 16 +- modules/local/vepplugin/clinvar/main.nf | 17 +- modules/local/vepplugin/clinvar/meta.yml | 5 + .../vepplugin/clinvar/tests/main.nf.test | 5 +- .../vepplugin/clinvar/tests/main.nf.test.snap | 2 +- nextflow.config | 1 + nextflow_schema.json | 9 +- subworkflows/local/prepare_vep_plugins.nf | 8 +- .../utils_nfcore_lrsomatic_pipeline/main.nf | 47 +++-- .../tests/main.function.nf.test | 187 ++++++++++++++++++ 14 files changed, 296 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0e51743..fb2c964c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#206](https://github.com/IntGenomicsLab/lrsomatic/pull/206) - A remote ClinVar is now downloaded once per run by the new `VEPPLUGIN_CLINVAR` step in `PREPARE_VEP_PLUGINS`, instead of being staged by Nextflow as a foreign file. Every VEP task re-checked a foreign file on its host, and NCBI answered the burst from a multi-sample GRCh38 run with HTTP 503, so `SOMATIC_VEP` failed with `Can't stage file ...clinvar_20260829.vcf.gz`; `-resume` could not recover, since the failed check changed the staging cache key. The download is checked against the new `--vep_clinvar_md5`, set by default to the MD5 NCBI (GRCh38) and Ensembl (CHM13) publish, so the pinned release cannot change silently. The ClinVar sizes in `docs/usage.md` are also corrected (@AmberVerhasselt). +- [#206](https://github.com/IntGenomicsLab/lrsomatic/pull/206) - A remote (http, https or ftp) ClinVar is now downloaded once per run by the new `VEPPLUGIN_CLINVAR` step in `PREPARE_VEP_PLUGINS`, instead of being staged by Nextflow as a foreign file; local and cloud-storage paths are staged as before. `GERMLINE_VEP` and `SOMATIC_VEP` re-checked the foreign file on its host for every sample, and NCBI answered the burst from a multi-sample GRCh38 run with HTTP 503, so `SOMATIC_VEP` failed with `Can't stage file ...clinvar_20260829.vcf.gz`; `-resume` could not recover, since the failed check changed the staging cache key. The download is checked against the new `--vep_clinvar_md5` and `--vep_clinvar_tbi_md5`, set by default to the checksums NCBI (GRCh38, VCF only) and Ensembl (CHM13, VCF and index) publish, so the pinned release cannot change silently. Resuming a run that already finished re-runs `GERMLINE_VEP` and `SOMATIC_VEP` once, since ClinVar now comes from a task rather than the stage cache. The ClinVar sizes in `docs/usage.md` are also corrected, and `docs/output.md` now documents `vep_plugins/` (@AmberVerhasselt). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `CLAIRS` no longer runs with `--haplotagged_tumor_bam_provided_so_skip_intermediate_phasing_and_haplotagging`. Since somatic calling was moved ahead of `PHASING_HAPLOTYPING` (v1.1.0), ClairS has received the untagged minimap2 BAM, so the flag made it skip its own phasing and haplotagging and call every paired sample without haplotype information: the full-alignment model saw no `HP` tags and the haplotype filtering step had nothing to filter on, the same as `--disable_phasing`. ClairS now runs Clair3 on the normal and tumour BAMs and phases the tumour itself. **Paired somatic calls change** (fewer false positives expected), and `CLAIRS` takes longer and uses more work-directory space (@ljwharbers). - [#203](https://github.com/IntGenomicsLab/lrsomatic/pull/203) - `docs/output.md` now lists the ClairS SNV output as `snvs.vcf.gz`, the name the pipeline publishes, instead of `snv.vcf.gz` (@ljwharbers). - [#196](https://github.com/IntGenomicsLab/lrsomatic/pull/196) - `LRSOMATICREPORT` now points `XDG_CACHE_HOME` at the task directory alongside `HOME` and `TMPDIR`. Singularity/Apptainer inherit the host environment, so on sites that set it outside the bind-mounted work tree the render died with `Read-only file system (os error 30): mkdir '<...>/.cache/quarto'` (@AmberVerhasselt, @ljwharbers). diff --git a/conf/igenomes.config b/conf/igenomes.config index 8924def0..430e65a2 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -33,6 +33,7 @@ params.genomes = [ vep_clinvar : "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz", vep_clinvar_tbi : "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz.tbi", vep_clinvar_md5 : "dffae274fd559ee764d1de44f36df2cf", + // No vep_clinvar_tbi_md5: NCBI publishes no checksum for its indexes // CADD is opt-in: the SNV table alone is 81 GB, and Nextflow's foreign-file cache is // per session, so every non-resumed run would re-stage it. Enable with --vep_cadd_snv // and --vep_cadd_indel (each with its _tbi); see docs/usage.md. @@ -62,8 +63,9 @@ params.genomes = [ vep_polyphen_sift_db : "https://ftp.ensembl.org/pub/release-115/variation/pangenomes/Human/homo_sapiens_pangenome_PolyPhen_SIFT_20240502.db", vep_clinvar : "https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/ensembl/variation/2022_10/vcf/2024_07/clinvar_20240624_GCA_009914755.4.vcf.gz", vep_clinvar_tbi : "https://ftp.ensembl.org/pub/rapid-release/species/Homo_sapiens/GCA_009914755.4/ensembl/variation/2022_10/vcf/2024_07/clinvar_20240624_GCA_009914755.4.vcf.gz.tbi", - // From the CHECKSUMS file Ensembl publishes alongside the VCF + // From the CHECKSUMS file Ensembl publishes alongside the VCF and its index vep_clinvar_md5 : "98bde67975c5f0b95230672f642a3c07", + vep_clinvar_tbi_md5 : "50b634a69c36e2479cc2c8f71a8eafd6", // SigProfilerMatrixGenerator CHM13-T2T payload (SigProfilerSuite/SigProfilerMatrixGenerator#250); // not yet hosted on the AlexandrovLab FTP, so it is fetched from the IntGenomicsLab Globus collection sigprofiler_genome : "CHM13-T2T", diff --git a/conf/modules.config b/conf/modules.config index 90cc1c56..54494857 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -685,7 +685,8 @@ process { withName : '.*:VEPPLUGIN_(REVEL|EVE|CLINVAR)' { publishDir = [ mode: params.publish_dir_mode, - path: { "${params.outdir}/vep_plugins" } + path: { "${params.outdir}/vep_plugins" }, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } diff --git a/docs/output.md b/docs/output.md index b392805f..54fb647c 100644 --- a/docs/output.md +++ b/docs/output.md @@ -497,6 +497,31 @@ Phased variant calls produced by Longphase. Present in all samples. +#### `vep_plugins` + +
+Output files + +``` +├── vep_plugins +│ ├── clinvar_20260829.vcf.gz +│ ├── clinvar_20260829.vcf.gz.tbi +│ ├── revel_grch38.tsv.gz +│ ├── revel_grch38.tsv.gz.tbi +│ ├── eve_merged.vcf.gz +│ ├── eve_merged.vcf.gz.tbi +``` + +| File | Description | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------- | +| `clinvar_.vcf.gz{,.tbi}` | ClinVar release and index as downloaded and MD5-checked by `VEPPLUGIN_CLINVAR`; only for a remote `--vep_clinvar` | +| `revel_grch38.tsv.gz{,.tbi}` | REVEL scores re-sorted on GRCh38 and indexed by `VEPPLUGIN_REVEL` | +| `eve_merged.vcf.gz{,.tbi}` | Per-protein EVE VCFs merged and indexed by `VEPPLUGIN_EVE`; only with the opt-in `--vep_eve` | + +Pass these to a later run as `--vep_clinvar`/`--vep_revel`/`--vep_eve` with their `_tbi` to skip the download and preparation. On CHM13 the ClinVar file is the CHM13-lifted Ensembl release instead. + +
+ #### Plugin fields in the `CSQ` annotation The germline and somatic VCFs carry these extra subfields inside VEP's `CSQ` INFO annotation, on diff --git a/docs/usage.md b/docs/usage.md index 5333398a..7d822ecb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -224,6 +224,7 @@ opt-in. See [VEP plugins](#vep-plugins) for sizes, licence terms and per-assembl | `--vep_clinvar` | ClinVar VCF, added as a VEP `--custom` annotation | | `--vep_clinvar_tbi` | Index for `--vep_clinvar`. Required whenever `--vep_clinvar` is set | | `--vep_clinvar_md5` | Expected MD5 of a remote `--vep_clinvar`, checked after download. Dropped when `--vep_clinvar` is overridden | +| `--vep_clinvar_tbi_md5` | Expected MD5 of the downloaded `--vep_clinvar_tbi`. Set on CHM13 only; dropped when either ClinVar file is overridden | | `--vep_clinvar_fields` | Comma-separated ClinVar INFO fields to carry through. Default = `"CLNSIG,CLNREVSTAT,CLNDN"` | | `--vep_cadd_snv` | CADD SNV score file, for the `CADD` plugin. No default — 81 GB, so opt-in; prefer a local path. GRCh38 only | | `--vep_cadd_snv_tbi` | Index for `--vep_cadd_snv`. Required whenever `--vep_cadd_snv` is set | @@ -528,12 +529,15 @@ Whether an index is required depends on the shape of what you supply: - **AlphaMissense, ClinVar and CADD** are used exactly as given, so their `_tbi` parameter is always required alongside them. Overriding a data file drops the default index: supply both, or neither. -- **A remote ClinVar** is downloaded once per run by `VEPPLUGIN_CLINVAR`, with its index, rather - than staged by every VEP task: NCBI answers the burst of requests a multi-sample run sends with - HTTP 503. The download is checked against `--vep_clinvar_md5`, so a release re-published under the - same name fails the run instead of changing the annotation. Both defaults carry the MD5 their host - publishes; with your own URL, pass its MD5 too, or the pipeline warns that the release is not - verified. The files are published to `/vep_plugins/` for reuse as local paths. +- **A remote ClinVar** (http, https or ftp) is downloaded once per run by `VEPPLUGIN_CLINVAR`, with + its index, rather than staged by `GERMLINE_VEP` and `SOMATIC_VEP` for every sample: NCBI answers + the burst of requests a multi-sample run sends with HTTP 503. The index must then be a URL too. The + download is checked against `--vep_clinvar_md5` (and `--vep_clinvar_tbi_md5` where set), so a + release re-published under the same name fails the run instead of changing the annotation. Both + defaults carry the MD5 their host publishes, and the CHM13 default also pins its index; NCBI + publishes no index checksum. With your own URL, pass its MD5 too, or the pipeline warns that the + release is not verified. Local and cloud-storage (`s3://`, `gs://`, `az://`) paths are staged as + given. The files are published to `/vep_plugins/` for reuse as local paths. - **REVEL and EVE** ship as zip archives. Pass a `.zip` and the pipeline unpacks and reshapes it; pass a prepared file and its index to use it directly. Remote zips are fetched with `wget` rather than staged by Nextflow, and EVE's 9.6 GB archive serves slowly, so expect hours. diff --git a/modules/local/vepplugin/clinvar/main.nf b/modules/local/vepplugin/clinvar/main.nf index 1a9290a1..ff2947cf 100644 --- a/modules/local/vepplugin/clinvar/main.nf +++ b/modules/local/vepplugin/clinvar/main.nf @@ -8,7 +8,7 @@ process VEPPLUGIN_CLINVAR { : 'community.wave.seqera.io/library/wget:1.21.4--8b0fcde81c17be5e'}" input: - tuple val(vcf_url), val(tbi_url), val(md5) + tuple val(vcf_url), val(tbi_url), val(md5), val(tbi_md5) output: path "${vcf_name}{,.tbi}", emit: files @@ -22,13 +22,14 @@ process VEPPLUGIN_CLINVAR { script: def args = task.ext.args ?: '' vcf_name = vcf_url.toString().tokenize('/').last() - def check = md5 ? "echo '${md5} ${vcf_name}' | md5sum -c -" : '' + def sums = [ md5 ? "${md5} ${vcf_name}" : null, tbi_md5 ? "${tbi_md5} ${vcf_name}.tbi" : null ].findAll() + def check = sums ? "printf '%s\\n' ${sums.collect { line -> "'${line}'" }.join(' ')} | md5sum -c -" : '' """ - # NCBI answers bursts with 503, so retry those rather than fail on the first one + # Retry a transient 503 from NCBI: a linear backoff of 1 s, 2 s, ... up to 10 s, about 45 s over 10 tries wget \\ --no-verbose \\ - --tries=5 \\ - --waitretry=30 \\ + --tries=10 \\ + --waitretry=10 \\ --retry-on-http-error=429,500,502,503,504 \\ ${args} \\ -O ${vcf_name} \\ @@ -37,14 +38,14 @@ process VEPPLUGIN_CLINVAR { # Saved next to the VCF under the name VEP looks for, whatever the host calls it wget \\ --no-verbose \\ - --tries=5 \\ - --waitretry=30 \\ + --tries=10 \\ + --waitretry=10 \\ --retry-on-http-error=429,500,502,503,504 \\ ${args} \\ -O ${vcf_name}.tbi \\ ${tbi_url} - # A pinned checksum keeps the release fixed: a host that re-publishes under the same name fails here + # Pinned checksums keep the release fixed: a host that re-publishes under the same name fails here ${check} cat <<-END_VERSIONS > versions.yml diff --git a/modules/local/vepplugin/clinvar/meta.yml b/modules/local/vepplugin/clinvar/meta.yml index dc6a6eb6..c35d7bd4 100644 --- a/modules/local/vepplugin/clinvar/meta.yml +++ b/modules/local/vepplugin/clinvar/meta.yml @@ -27,6 +27,11 @@ input: description: | Expected MD5 of the VCF. When set, a download that does not match fails the task. May be null, in which case the release is not verified. + - tbi_md5: + type: string + description: | + Expected MD5 of the index, checked the same way. May be null, as for + NCBI, which publishes no checksum for its indexes. output: files: diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test b/modules/local/vepplugin/clinvar/tests/main.nf.test index 4f0806c2..e1ed2a25 100644 --- a/modules/local/vepplugin/clinvar/tests/main.nf.test +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test @@ -4,6 +4,8 @@ nextflow_process { script "../main.nf" process "VEPPLUGIN_CLINVAR" + // "small" is what .github/workflows/nf-test.yml selects on for pull_request + tag "small" tag "modules" tag "modules_local" tag "vepplugin_clinvar" @@ -19,7 +21,8 @@ nextflow_process { input[0] = [ 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz', 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz.tbi', - 'dffae274fd559ee764d1de44f36df2cf' + 'dffae274fd559ee764d1de44f36df2cf', + null // NCBI publishes no checksum for its index ] """ } diff --git a/modules/local/vepplugin/clinvar/tests/main.nf.test.snap b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap index e31faa30..d1a484af 100644 --- a/modules/local/vepplugin/clinvar/tests/main.nf.test.snap +++ b/modules/local/vepplugin/clinvar/tests/main.nf.test.snap @@ -26,6 +26,6 @@ "nf-test": "0.9.3", "nextflow": "25.10.4" }, - "timestamp": "2026-09-25T11:40:31.870446909" + "timestamp": "2026-09-25T16:19:22.283693138" } } \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 399c9584..628b5ca3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -59,6 +59,7 @@ params { vep_clinvar = null vep_clinvar_tbi = null vep_clinvar_md5 = null + vep_clinvar_tbi_md5 = null vep_clinvar_fields = "CLNSIG,CLNREVSTAT,CLNDN" vep_cadd_snv = null vep_cadd_snv_tbi = null diff --git a/nextflow_schema.json b/nextflow_schema.json index 1e6c4720..47356879 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -288,7 +288,14 @@ "description": "Expected MD5 of the --vep_clinvar VCF, checked after the pipeline downloads it.", "fa_icon": "fas fa-fingerprint", "pattern": "^[0-9a-fA-F]{32}$", - "help_text": "A remote --vep_clinvar is downloaded once per run by VEPPLUGIN_CLINVAR, rather than staged by every VEP task, and a VCF that does not match this MD5 fails the run, so the release cannot change silently. The GRCh38 and CHM13 defaults carry the MD5 their hosts publish; overriding --vep_clinvar drops it. Only applies to a URL: a local --vep_clinvar is staged as given." + "help_text": "A remote --vep_clinvar (http, https or ftp) is downloaded once per run by VEPPLUGIN_CLINVAR, rather than staged by GERMLINE_VEP and SOMATIC_VEP for every sample, and a VCF that does not match this MD5 fails the run, so the release cannot change silently. The GRCh38 and CHM13 defaults carry the MD5 their hosts publish; overriding --vep_clinvar drops it. A local or cloud-storage --vep_clinvar is staged as given, and takes no MD5." + }, + "vep_clinvar_tbi_md5": { + "type": "string", + "description": "Expected MD5 of the --vep_clinvar_tbi index, checked after the pipeline downloads it.", + "fa_icon": "fas fa-fingerprint", + "pattern": "^[0-9a-fA-F]{32}$", + "help_text": "Checked alongside --vep_clinvar_md5. The CHM13 default carries the MD5 Ensembl publishes; GRCh38 has none, since NCBI publishes no checksum for its indexes. Overriding --vep_clinvar or --vep_clinvar_tbi drops the default." }, "vep_clinvar_fields": { "type": "string", diff --git a/subworkflows/local/prepare_vep_plugins.nf b/subworkflows/local/prepare_vep_plugins.nf index 4803c543..bfa31096 100644 --- a/subworkflows/local/prepare_vep_plugins.nf +++ b/subworkflows/local/prepare_vep_plugins.nf @@ -86,16 +86,16 @@ workflow PREPARE_VEP_PLUGINS { // // MODULE: VEPPLUGIN_CLINVAR (label: process_single) - // Input: the ClinVar VCF and index URLs, and the MD5 pinning the release (or null) + // Input: the ClinVar VCF and index URLs, and the MD5s pinning them (either may be null) // Output: .files -- the VCF and its index, under the VCF's own basename - // One download per run: a foreign file is re-checked on its host by every VEP task, and NCBI - // answers the burst a multi-sample run sends with 503 + // One download per run: a foreign file is re-checked on its host by GERMLINE_VEP and SOMATIC_VEP + // for every sample, and NCBI answers the burst a multi-sample run sends with 503 // if (prepare.containsKey('vep_clinvar')) { def clinvar = prepare['vep_clinvar'] VEPPLUGIN_CLINVAR ( - channel.value([ clinvar.vcf, clinvar.tbi, clinvar.md5 ]) + channel.value([ clinvar.vcf, clinvar.tbi, clinvar.md5, clinvar.tbi_md5 ]) ) staged << VEPPLUGIN_CLINVAR.out.files diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf index fcf50d45..bc8c7da8 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/main.nf @@ -408,17 +408,23 @@ def vepPluginNeedsPrep(data_param) { !value.toString().toLowerCase().endsWith('.gz') } +// +// Whether a value is a URL wget can fetch. Cloud and file:// URIs are left to Nextflow's own +// filesystem providers, which stage them without a request per task +// +def isFetchableUrl(value) { + return value && value.toString() ==~ /(?i)^(https?|ftp):\/\/.*/ +} + // // Whether a resource is downloaded once by a prep task instead of staged as a foreign file // -// Only a remote ClinVar is: a foreign file is re-checked on its host by every VEP task that takes -// it, and NCBI answers the burst a multi-sample run sends with 503, which fails the staging. +// Only a remote ClinVar is: a foreign file is re-checked on its host by GERMLINE_VEP and +// SOMATIC_VEP for every sample, and NCBI answers the burst a multi-sample run sends with 503, +// which fails the staging. // def vepPluginNeedsFetch(data_param) { - def value = vepPluginResource(data_param) - return value && - data_param == 'vep_clinvar' && - value.toString().contains('://') + return data_param == 'vep_clinvar' && isFetchableUrl(vepPluginResource(data_param)) } // @@ -430,6 +436,19 @@ def vepPluginMd5(data_param) { return params[md5_param] ?: (params[data_param] ? null : getGenomeAttribute(md5_param)) } +// +// The expected MD5 of a fetched resource's index. Overriding the data file or the index drops +// the default, which belongs to a different file +// +def vepPluginIndexMd5(data_param) { + def index_param = vepPluginIndexParams()[data_param] + if (!index_param) { + return null + } + def md5_param = "${index_param}_md5".toString() + return params[md5_param] ?: ((params[data_param] || params[index_param]) ? null : getGenomeAttribute(md5_param)) +} + // // The filename a prep task writes, referenced by the VEP argument since plugins stage into the task root // @@ -497,9 +516,15 @@ def validateVepPluginParams() { error("--vep_custom: needs a matching '--custom file=...' entry in --vep_args, which is where the staged file is substituted in. Add one, e.g. --vep_args '${params.vep_args} --custom file=placeholder,short_name=MyTrack,format=vcf,type=exact,coords=0'.") } - // The MD5 is checked by the download task, so a local ClinVar would silently skip it - if (params.vep_clinvar_md5 && !vepPluginNeedsFetch('vep_clinvar')) { - error("--vep_clinvar_md5: only checks a ClinVar the pipeline downloads, so it needs --vep_clinvar to be a URL. Drop --vep_clinvar_md5 for a local file.") + // The MD5s are checked by the download task, so a ClinVar that is staged instead would silently skip them + ['vep_clinvar_md5', 'vep_clinvar_tbi_md5'].each { md5_param -> + if (params[md5_param] && !vepPluginNeedsFetch('vep_clinvar')) { + error("--${md5_param}: only checks a ClinVar the pipeline downloads, so it needs --vep_clinvar to be an http(s) or ftp URL. Drop --${md5_param} for a local or cloud-storage file.") + } + } + // The download task fetches both files, so a remote VCF cannot be paired with a local index + if (vepPluginNeedsFetch('vep_clinvar') && !isFetchableUrl(vepPluginIndex('vep_clinvar'))) { + error("--vep_clinvar_tbi: '${vepPluginIndex('vep_clinvar')}' is not an http(s) or ftp URL, but --vep_clinvar '${vepPluginResource('vep_clinvar')}' is downloaded, and its index is downloaded with it. Pass the index URL, or point both at local copies.") } if (vepPluginNeedsFetch('vep_clinvar') && !vepPluginMd5('vep_clinvar')) { log.warn("--vep_clinvar: '${vepPluginResource('vep_clinvar')}' is downloaded without --vep_clinvar_md5, so its release is not verified: a host that re-publishes under the same name, like the rolling clinvar.vcf.gz, changes the annotation between runs.") @@ -524,12 +549,12 @@ def stageVepPluginFile(staged, data_param) { // // A resource needing prep is kept as its raw value rather than a file(), since neither the REVEL nor // the EVE host can be staged by Nextflow -- PREPARE_VEP_PLUGINS fetches those with WGET instead. -// A remote ClinVar is recorded with its index and MD5, and keeps its own basename. +// A remote ClinVar is recorded with its index and their MD5s, and keeps its own basename. // def registerVepPlugin(staged, prepare, data_param) { if (vepPluginNeedsFetch(data_param)) { def url = vepPluginResource(data_param).toString() - prepare[data_param] = [ vcf: url, tbi: vepPluginIndex(data_param), md5: vepPluginMd5(data_param) ] + prepare[data_param] = [ vcf: url, tbi: vepPluginIndex(data_param), md5: vepPluginMd5(data_param), tbi_md5: vepPluginIndexMd5(data_param) ] return url.tokenize('/').last() } if (vepPluginNeedsPrep(data_param)) { diff --git a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test index 2d302b88..8c90956b 100644 --- a/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test +++ b/subworkflows/local/utils_nfcore_lrsomatic_pipeline/tests/main.function.nf.test @@ -586,6 +586,148 @@ nextflow_function { } } + test("vepPluginNeedsFetch is false for a cloud-storage ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 's3://my-bucket/clinvar_20250101.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + // wget cannot fetch s3://, so Nextflow keeps staging it as before + { assert !function.result } + ) + } + } + + test("vepPluginNeedsFetch is true for an ftp ClinVar") { + + function "vepPluginNeedsFetch" + + when { + params { + genome = 'GRCh38' + vep_clinvar = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive_2.0/2026/clinvar_20260829.vcf.gz' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == true } + ) + } + } + + test("vepPluginIndexMd5 takes the default index MD5 on CHM13") { + + function "vepPluginIndexMd5" + + when { + params { + genome = 'CHM13' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result ==~ /[0-9a-f]{32}/ } + ) + } + } + + test("vepPluginIndexMd5 is null on GRCh38, where NCBI publishes none") { + + function "vepPluginIndexMd5" + + when { + params { + genome = 'GRCh38' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result == null } + ) + } + } + + test("vepPluginIndexMd5 drops the default once the index is overridden") { + + function "vepPluginIndexMd5" + + when { + params { + genome = 'CHM13' + vep_clinvar_tbi = 'https://example.org/clinvar_20240624_GCA_009914755.4.vcf.gz.tbi' + } + function { + """ + input[0] = 'vep_clinvar' + """ + } + } + + then { + assertAll( + { assert function.success }, + // the default MD5 belongs to the default index, not to this one + { assert function.result == null } + ) + } + } + + test("resolveVepPlugins carries the index MD5 to the download task") { + + function "resolveVepPlugins" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/archive/clinvar_20250101.vcf.gz' + vep_clinvar_tbi = 'https://example.org/archive/clinvar_20250101.vcf.gz.tbi' + vep_clinvar_tbi_md5 = 'fedcba9876543210fedcba9876543210' + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.prepare['vep_clinvar'].tbi_md5 == 'fedcba9876543210fedcba9876543210' } + ) + } + } + // --- validateVepPluginParams: the error paths test("validateVepPluginParams passes when plugins are skipped") { @@ -797,4 +939,49 @@ nextflow_function { ) } } + + test("validateVepPluginParams rejects a remote ClinVar with a local index") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = 'https://example.org/clinvar_20250101.vcf.gz' + vep_clinvar_tbi = '/data/mine/clinvar_20250101.vcf.gz.tbi' + vep_clinvar_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + // the download task fetches both, so wget would be handed a local path + { assert function.failed }, + { assert function.stdout.any { line -> line.contains('vep_clinvar_tbi') } } + ) + } + } + + test("validateVepPluginParams rejects an index MD5 for a local ClinVar") { + + function "validateVepPluginParams" + + when { + params { + genome = null + skip_vep_plugins = false + vep_clinvar = '/data/mine/clinvar.vcf.gz' + vep_clinvar_tbi = '/data/mine/clinvar.vcf.gz.tbi' + vep_clinvar_tbi_md5 = '0123456789abcdef0123456789abcdef' + } + } + + then { + assertAll( + { assert function.failed }, + { assert function.stdout.any { line -> line.contains('vep_clinvar_tbi_md5') } } + ) + } + } }