From 71859368c01e0aa6facd153de3efc01f7d70a0b2 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 27 Jul 2026 20:48:03 -0700 Subject: [PATCH] Package the KEGG module graphs and module descriptions I'm working to get the blimmp_nexflow code working, but it was failing to find any hits. This commit addresses that. In the manifest, `package_data` referenced `KEGG_Module_Graphs.zip`, a filename that no longer exists in the repo, and it didn't list `kegg_bacteria_modules.json`. Both files were therefore dropped by `pip install .`, so the published container had them in /app but not in site-packages, which is where module_detection.py looks. The consequence was silent. `modules_to_kos()` returned an empty dict, so the KO universe was empty, so the left join of observed annotations onto it discarded every hit. Runs exited 0 and wrote a structurally complete result in which the per-KO table had zero rows and all 340 modules scored 0.0 confidence. Verified against ghcr.io/traviswheelerlab/blimmp:0.1.2. Changes: - setup.py / MANIFEST.in: reference the real archive name and add the module descriptions. Globs instead of individual filenames so a future rename does not silently drop a file again. - module_detection.py: extract the graph archive via `ensure_module_graphs()`, which reuses an existing extraction, never deletes the source zip, and falls back to a writable cache dir (BLIMMP_CACHE_DIR) when the install tree is read-only. The previous code wrote into site-packages and removed the zip, which would have started failing under Singularity as soon as the zip was actually packaged. - module_detection.py: `validate_paths()` and an empty-KO-universe guard, so a mispackaged install aborts with a clear message instead of reporting every module absent. - Dockerfile: extract the graphs at build time, while the filesystem is still writable, and assert the data files made it into the installed package. - tests/smoke_test.py + CI: run BLIMMP on Examples/example.domtblout and assert a non-empty per-KO table, at least one non-zero module confidence, and populated module descriptions. Also runs against a read-only container. This commit deliberately leaves the version alone. Once it is merged to main, bump setup.py to 0.1.3 and tag v0.1.3 to trigger a corrected container build; setup.py has been stuck at 0.1.0 across the v0.1.1 and v0.1.2 tags. Verified on Python 3.10 (Examples/example.domtblout, sigma 1.0): before 0 per-KO rows, 0/340 modules with confidence, 0/340 with descriptions after 1980 per-KO rows, 219/340 with confidence, 323/340 with descriptions Implemented with help from Claude Opus 5 --- .github/workflows/test.yml | 38 +++++++ .gitignore | 8 ++ BLIMMP_Scripts/module_detection.py | 157 ++++++++++++++++++++++++----- Dockerfile | 18 ++++ MANIFEST.in | 13 ++- setup.py | 16 +-- tests/smoke_test.py | 116 +++++++++++++++++++++ 7 files changed, 325 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 tests/smoke_test.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a092b61 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,38 @@ +name: Test + +on: + push: + branches: ['**'] + pull_request: + +jobs: + smoke-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Build the same image the release workflow publishes, so the test covers + # the installed package rather than the source tree. A build that drops a + # data file fails here instead of silently producing all-zero results. + - name: Build image + uses: docker/build-push-action@v5 + with: + context: . + push: false + load: true + tags: blimmp:ci + + - name: BLIMMP --help + run: docker run --rm blimmp:ci BLIMMP -h + + - name: Smoke test + run: docker run --rm blimmp:ci python /app/tests/smoke_test.py + + # The published image runs read-only under Singularity on HPC. Catch any + # attempt to write into the install tree at analysis time. + - name: Smoke test on a read-only filesystem + run: | + docker run --rm --read-only --tmpfs /tmp \ + -e BLIMMP_CACHE_DIR=/tmp/blimmp-cache \ + blimmp:ci python /app/tests/smoke_test.py diff --git a/.gitignore b/.gitignore index 1f7513b..a2ce5e7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,11 @@ __pycache__/ *.pyc BLIMMP/Step1_BATH_KOFAMDB + +# Build artifacts from `pip install .` +build/ +dist/ +*.egg-info/ + +# Module graphs, extracted from the packaged zip on first use +BLIMMP_Scripts/Graph_Dependencies/KEGG_Graphs_Generated_March26/ diff --git a/BLIMMP_Scripts/module_detection.py b/BLIMMP_Scripts/module_detection.py index 708c2e3..e28cafa 100644 --- a/BLIMMP_Scripts/module_detection.py +++ b/BLIMMP_Scripts/module_detection.py @@ -5,10 +5,12 @@ import os import glob import re +import shutil import sys import math import ast import json +import tempfile import argparse import pandas as pd import numpy as np @@ -2041,6 +2043,14 @@ def run(self): # Build KO universe straight from my module JSONs ko_to_modules_str = File_Helpers.modules_to_kos(self.paths.module_json_dir) + if not ko_to_modules_str: + # Observed annotations are left-joined onto this universe below, so an + # empty universe silently discards every hit and scores all modules 0. + raise SystemExit( + f"[FATAL] No KEGG module graphs found in {self.paths.module_json_dir} " + f"(expected module_*_nodes.json). Refusing to continue: every module " + f"would be reported absent regardless of the input." + ) ko_universe = pd.DataFrame({'KO id': sorted(ko_to_modules_str.keys())}) @@ -2169,6 +2179,124 @@ def run(self): +GRAPH_ZIP_NAME = "KEGG_Graphs_Generated_March26.zip" +GRAPH_DIR_NAME = "KEGG_Graphs_Generated_March26" + + +def _graphs_present(directory) -> bool: + """True if `directory` holds at least one extracted module graph.""" + return bool(glob.glob(os.path.join(str(directory), "module_*_nodes.json"))) + + +def ensure_module_graphs(graph_dependencies_dir=None) -> Path: + """Return a directory holding the extracted KEGG module graphs. + + The graphs ship as a zip inside the installed package. They are extracted + once, on first use, and reused thereafter. + + The install tree is read-only in a container (Singularity mounts the image + read-only), so if extraction there fails we fall back to a user-writable + cache directory. Set BLIMMP_CACHE_DIR to control where that lands. + Building the container pre-extracts the graphs so this never runs at + analysis time. + + Raises SystemExit if the graphs can be neither found nor extracted -- + without them BLIMMP has an empty KO universe and every module scores zero. + """ + if graph_dependencies_dir is None: + graph_dependencies_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "Graph_Dependencies" + ) + graph_dependencies_dir = Path(graph_dependencies_dir) + + in_place = graph_dependencies_dir / GRAPH_DIR_NAME + if _graphs_present(in_place): + return in_place + + cache_root = Path( + os.environ.get("BLIMMP_CACHE_DIR") + or (Path(tempfile.gettempdir()) / f"blimmp-cache-{os.getuid()}") + ) + cached = cache_root / GRAPH_DIR_NAME + if _graphs_present(cached): + return cached + + zip_path = graph_dependencies_dir / GRAPH_ZIP_NAME + if not zip_path.is_file(): + raise SystemExit( + f"[FATAL] KEGG module graphs are missing from this BLIMMP installation.\n" + f" Expected extracted graphs in : {in_place}\n" + f" or the source archive at : {zip_path}\n" + f"This usually means the package was built without its data files " + f"(check `package_data` in setup.py). Reinstall BLIMMP, or use a " + f"container image built from a corrected release." + ) + + # Prefer the install tree; fall back to the cache if it is read-only. + for destination in (in_place, cached): + try: + destination.mkdir(parents=True, exist_ok=True) + _extract_module_graphs(zip_path, destination) + except OSError as exc: + logging.debug("Could not extract graphs to %s: %s", destination, exc) + continue + if _graphs_present(destination): + return destination + + raise SystemExit( + f"[FATAL] Failed to extract {GRAPH_ZIP_NAME}. Tried {in_place} and {cached}. " + f"Set BLIMMP_CACHE_DIR to a writable directory and retry." + ) + + +def _extract_module_graphs(zip_path, destination) -> None: + """Extract `zip_path` into `destination`, normalising the archive layout. + + The archive was built on macOS, so it carries __MACOSX metadata and wraps + everything in a redundant top-level folder. Both are flattened away. + The source zip is deliberately left in place so extraction stays repeatable. + """ + print(f"Extracting {os.path.basename(str(zip_path))} to {destination} ...") + destination = Path(destination) + with zipfile.ZipFile(str(zip_path), "r") as z: + z.extractall(str(destination)) + + macosx_path = destination / "__MACOSX" + if macosx_path.is_dir(): + shutil.rmtree(str(macosx_path)) + + nested = destination / destination.name + if nested.is_dir(): + for item in os.listdir(str(nested)): + shutil.move(str(nested / item), str(destination / item)) + nested.rmdir() + + +def validate_paths(paths: "Paths") -> None: + """Fail fast if a required data dependency is missing. + + Several of these were silently optional, which let a mispackaged build + produce a structurally complete but entirely zero-valued result. + """ + required = { + "KEGG module equations": paths.module_eq_json, + "KOfam KO list": paths.kofam_ko_list_path, + "module frequencies": paths.module_frequencies, + "module/KO reaction map": paths.module_reaction_dir, + "module descriptions": paths.module_descriptions_path, + "taxonomy frequency tables": paths.counts_dir, + "module neighbor data": paths.module_neighbor_dir, + } + missing = [f" {label}: {path}" for label, path in required.items() if not Path(path).exists()] + if missing: + raise SystemExit( + "[FATAL] BLIMMP is missing required data files:\n" + + "\n".join(missing) + + "\nThe installation is incomplete -- reinstall BLIMMP or use a " + "container image built from a corrected release." + ) + + def main(): p = argparse.ArgumentParser( description='BLIMMP: Bayesian Likelihood Inference of Metabolic Module Presence. ' @@ -2237,44 +2365,19 @@ def main(): GD = os.path.join(HERE, "Graph_Dependencies") DD = os.path.join(HERE, "Data_Dependencies") - ZIPS_TO_EXTRACT = {os.path.join(GD, "KEGG_Graphs_Generated_March26.zip") : os.path.join(GD, "KEGG_Graphs_Generated_March26")} - - for zip_path, extract_to in ZIPS_TO_EXTRACT.items(): - if os.path.isfile(zip_path): - os.makedirs(extract_to, exist_ok=True) - print(f"Extracting {os.path.basename(zip_path)}...") - with zipfile.ZipFile(zip_path, 'r') as z: - z.extractall(extract_to) - - macosx_path = os.path.join(extract_to, "__MACOSX") - if os.path.isdir(macosx_path): - import shutil - shutil.rmtree(macosx_path) - print(f"Removed MACOSX metadata folder") - - extract_name = os.path.basename(extract_to) - nested = os.path.join(extract_to, extract_name) - if os.path.isdir(nested): - import shutil - for item in os.listdir(nested): - shutil.move(os.path.join(nested, item), extract_to) - os.rmdir(nested) - print(f" Flattened double-nested folder: {extract_name}/{extract_name} to {extract_name}/") - os.remove(zip_path) - print(f"Done. {extract_to}") - paths = Paths( counts_dir = Path(DD)/ "ATB_Taxonomy_Frequency", onehop_dir = Path(GD)/ "ONE_HOP_NEIGHBOR_DATA", twohop_dir = Path(GD)/ "TWO_HOP_NEIGHBOR_DATA", module_neighbor_dir = Path(GD)/ "MODULE_ALL_NEIGHBOR_DATA", module_eq_json = Path(GD)/ "KEGG_Module_Equations_Jan26.json", - module_json_dir = Path(GD)/ "KEGG_Graphs_Generated_March26", + module_json_dir = ensure_module_graphs(GD), kofam_ko_list_path = Path(DD)/ "ko_list.txt", module_frequencies = Path(DD)/ "module_freq.txt", module_reaction_dir = Path(GD)/ "module_ko_reaction.json", module_descriptions_path = Path(DD)/ "kegg_bacteria_modules.json" ) + validate_paths(paths) BlimmpPipeline(cfg, paths).run() diff --git a/Dockerfile b/Dockerfile index dc0440b..f3ac893 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,3 +5,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -r WORKDIR /app COPY . . RUN pip install --no-cache-dir . + +# Extract the KEGG module graphs now, while the filesystem is still writable. +# Singularity mounts the image read-only at run time, so leaving this to first +# use would fail on HPC. Also fails the build if the data files did not make it +# into the installed package. +RUN python -c "\ +from BLIMMP_Scripts.module_detection import ensure_module_graphs; \ +print('module graphs:', ensure_module_graphs())" \ + && python -c "\ +import BLIMMP_Scripts, pathlib, sys; \ +root = pathlib.Path(BLIMMP_Scripts.__file__).parent; \ +required = ['Data_Dependencies/kegg_bacteria_modules.json', \ + 'Data_Dependencies/ko_list.txt', \ + 'Data_Dependencies/module_freq.txt', \ + 'Graph_Dependencies/KEGG_Module_Equations_Jan26.json', \ + 'Graph_Dependencies/module_ko_reaction.json']; \ +missing = [r for r in required if not (root / r).exists()]; \ +sys.exit('Missing packaged data files: ' + repr(missing)) if missing else print('packaged data OK')" diff --git a/MANIFEST.in b/MANIFEST.in index 956f2d0..0706c59 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ -include BLIMMP_Scripts/Graph_Dependencies/KEGG_Module_Graphs.zip -include BLIMMP_Scripts/Graph_Dependencies/KEGG_Module_Equations_Jan26.json -include BLIMMP_Scripts/Graph_Dependencies/module_ko_reaction.json -include BLIMMP_Scripts/Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/* -include BLIMMP_Scripts/Data_Dependencies/ATB_Taxonomy_Frequency/* -include BLIMMP_Scripts/Data_Dependencies/ko_list.txt -include BLIMMP_Scripts/Data_Dependencies/module_freq.txt +include BLIMMP_Scripts/Graph_Dependencies/KEGG_Graphs_Generated_March26.zip +include BLIMMP_Scripts/Graph_Dependencies/*.json +include BLIMMP_Scripts/Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*.json +include BLIMMP_Scripts/Data_Dependencies/*.txt +include BLIMMP_Scripts/Data_Dependencies/*.json +include BLIMMP_Scripts/Data_Dependencies/ATB_Taxonomy_Frequency/*.tsv diff --git a/setup.py b/setup.py index 71e6fb2..3fbda02 100644 --- a/setup.py +++ b/setup.py @@ -14,13 +14,15 @@ ], package_data={ "BLIMMP_Scripts": [ - "Graph_Dependencies/KEGG_Module_Graphs.zip", - "Graph_Dependencies/KEGG_Module_Equations_Jan26.json", - "Graph_Dependencies/module_ko_reaction.json", - "Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*", - "Data_Dependencies/ATB_Taxonomy_Frequency/*", - "Data_Dependencies/ko_list.txt", - "Data_Dependencies/module_freq.txt", + # Module graphs, shipped zipped and extracted on first use. + # module_detection.py hard-fails if this is missing, so a rename + # here can no longer silently produce all-zero results. + "Graph_Dependencies/KEGG_Graphs_Generated_March26.zip", + "Graph_Dependencies/*.json", + "Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*.json", + "Data_Dependencies/*.txt", + "Data_Dependencies/*.json", + "Data_Dependencies/ATB_Taxonomy_Frequency/*.tsv", ] }, include_package_data=True, diff --git a/tests/smoke_test.py b/tests/smoke_test.py new file mode 100644 index 0000000..fa0e007 --- /dev/null +++ b/tests/smoke_test.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Smoke test: run BLIMMP end-to-end and assert the results are non-trivial. + +This guards against mispackaged builds. BLIMMP builds its KO universe from data +files that ship inside the package; when those files are missing, every observed +annotation is dropped by a left join and the run still exits 0 while reporting +every module absent. The output looks structurally complete, so nothing short of +inspecting the values catches it. + +Usage: + python tests/smoke_test.py [--blimmp BLIMMP] [--input Examples/example.domtblout] +""" + +import argparse +import csv +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def fail(message): + print(f"FAIL: {message}", file=sys.stderr) + sys.exit(1) + + +def read_rows(path): + with open(path, newline="") as handle: + return list(csv.DictReader(handle)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--blimmp", default="BLIMMP", help="BLIMMP executable (default: BLIMMP)") + parser.add_argument( + "--input", + default=str(REPO_ROOT / "Examples" / "example.domtblout"), + help="domtblout file to run against", + ) + args = parser.parse_args() + + if not Path(args.input).is_file(): + fail(f"test input not found: {args.input}") + + with tempfile.TemporaryDirectory() as tmpdir: + prefix = str(Path(tmpdir) / "smoke") + cmd = [args.blimmp, args.input, "-f", "domtblout", "--sigma", "1.0", "--output", prefix] + print("Running:", " ".join(cmd), flush=True) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr, file=sys.stderr) + fail(f"BLIMMP exited {result.returncode}") + + # A mispackaged build warns instead of failing; treat that as fatal here. + combined = result.stdout + result.stderr + for marker in ("file not found", "Module descriptions file not found"): + if marker in combined: + print(combined) + fail(f"BLIMMP reported a missing data file (matched {marker!r})") + + dk_path = Path(f"{prefix}_BLIMMP_dk.csv") + modules_csv = Path(f"{prefix}_BLIMMP_module_probabilities.csv") + modules_json = Path(f"{prefix}_BLIMMP_modules.json") + for path in (dk_path, modules_csv, modules_json): + if not path.is_file(): + fail(f"expected output missing: {path.name}") + + # 1. The per-KO table must not be empty. This is the direct symptom of an + # empty KO universe -- the bug that produced all-zero results in v0.1.2. + dk_rows = read_rows(dk_path) + if not dk_rows: + fail("per-KO table (_BLIMMP_dk.csv) has no data rows -- KO universe was empty") + print(f" per-KO rows: {len(dk_rows)}") + + module_rows = read_rows(modules_csv) + if not module_rows: + fail("module table has no data rows") + + # 2. At least one module must carry non-zero confidence. The example input + # is a real genome's search results, so an all-zero result is a bug. + def as_float(value): + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + confident = [r for r in module_rows if as_float(r.get("module_confidence")) > 0] + if not confident: + fail( + f"all {len(module_rows)} modules scored 0.0 confidence -- " + "this is the signature of missing KEGG module graphs" + ) + print(f" modules with non-zero confidence: {len(confident)} / {len(module_rows)}") + + # 3. Module descriptions come from kegg_bacteria_modules.json, which was + # omitted from package_data in v0.1.2. + described = [r for r in module_rows if (r.get("module_description") or "").strip()] + if not described: + fail("no module has a description -- kegg_bacteria_modules.json was not packaged") + print(f" modules with descriptions: {len(described)} / {len(module_rows)}") + + with open(modules_json) as handle: + payload = json.load(handle) + if not payload: + fail("module JSON is empty") + print(f" modules in JSON: {len(payload)}") + + print("PASS: BLIMMP produced non-trivial results") + + +if __name__ == "__main__": + main()