Package the KEGG module graphs and module descriptions

.github/workflows/test.yml

@@ -0,0 +1,38 @@
+name: Test
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+
+jobs:
+  smoke-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Build the same image the release workflow publishes, so the test covers
+      # the installed package rather than the source tree. A build that drops a
+      # data file fails here instead of silently producing all-zero results.
+      - name: Build image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          load: true
+          tags: blimmp:ci
+
+      - name: BLIMMP --help
+        run: docker run --rm blimmp:ci BLIMMP -h
+
+      - name: Smoke test
+        run: docker run --rm blimmp:ci python /app/tests/smoke_test.py
+
+      # The published image runs read-only under Singularity on HPC. Catch any
+      # attempt to write into the install tree at analysis time.
+      - name: Smoke test on a read-only filesystem
+        run: |
+          docker run --rm --read-only --tmpfs /tmp \
+            -e BLIMMP_CACHE_DIR=/tmp/blimmp-cache \
+            blimmp:ci python /app/tests/smoke_test.py

.gitignore

@@ -6,3 +6,11 @@
 __pycache__/
 *.pyc
 BLIMMP/Step1_BATH_KOFAMDB
+
+# Build artifacts from `pip install .`
+build/
+dist/
+*.egg-info/
+
+# Module graphs, extracted from the packaged zip on first use
+BLIMMP_Scripts/Graph_Dependencies/KEGG_Graphs_Generated_March26/

BLIMMP_Scripts/module_detection.py

@@ -5,10 +5,12 @@
 import os
 import glob
 import re
+import shutil
 import sys
 import math
 import ast
 import json
+import tempfile
 import argparse
 import pandas as pd
 import numpy as np
@@ -2041,6 +2043,14 @@ def run(self):
 
         # Build KO universe straight from my module JSONs
         ko_to_modules_str = File_Helpers.modules_to_kos(self.paths.module_json_dir)
+        if not ko_to_modules_str:
+            # Observed annotations are left-joined onto this universe below, so an
+            # empty universe silently discards every hit and scores all modules 0.
+            raise SystemExit(
+                f"[FATAL] No KEGG module graphs found in {self.paths.module_json_dir} "
+                f"(expected module_*_nodes.json). Refusing to continue: every module "
+                f"would be reported absent regardless of the input."
+            )
         ko_universe = pd.DataFrame({'KO id': sorted(ko_to_modules_str.keys())})
 
 
@@ -2169,6 +2179,124 @@ def run(self):
 
 
 
+GRAPH_ZIP_NAME = "KEGG_Graphs_Generated_March26.zip"
+GRAPH_DIR_NAME = "KEGG_Graphs_Generated_March26"
+
+
+def _graphs_present(directory) -> bool:
+    """True if `directory` holds at least one extracted module graph."""
+    return bool(glob.glob(os.path.join(str(directory), "module_*_nodes.json")))
+
+
+def ensure_module_graphs(graph_dependencies_dir=None) -> Path:
+    """Return a directory holding the extracted KEGG module graphs.
+
+    The graphs ship as a zip inside the installed package.  They are extracted
+    once, on first use, and reused thereafter.
+
+    The install tree is read-only in a container (Singularity mounts the image
+    read-only), so if extraction there fails we fall back to a user-writable
+    cache directory.  Set BLIMMP_CACHE_DIR to control where that lands.
+    Building the container pre-extracts the graphs so this never runs at
+    analysis time.
+
+    Raises SystemExit if the graphs can be neither found nor extracted --
+    without them BLIMMP has an empty KO universe and every module scores zero.
+    """
+    if graph_dependencies_dir is None:
+        graph_dependencies_dir = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), "Graph_Dependencies"
+        )
+    graph_dependencies_dir = Path(graph_dependencies_dir)
+
+    in_place = graph_dependencies_dir / GRAPH_DIR_NAME
+    if _graphs_present(in_place):
+        return in_place
+
+    cache_root = Path(
+        os.environ.get("BLIMMP_CACHE_DIR")
+        or (Path(tempfile.gettempdir()) / f"blimmp-cache-{os.getuid()}")
+    )
+    cached = cache_root / GRAPH_DIR_NAME
+    if _graphs_present(cached):
+        return cached
+
+    zip_path = graph_dependencies_dir / GRAPH_ZIP_NAME
+    if not zip_path.is_file():
+        raise SystemExit(
+            f"[FATAL] KEGG module graphs are missing from this BLIMMP installation.\n"
+            f"  Expected extracted graphs in : {in_place}\n"
+            f"  or the source archive at     : {zip_path}\n"
+            f"This usually means the package was built without its data files "
+            f"(check `package_data` in setup.py). Reinstall BLIMMP, or use a "
+            f"container image built from a corrected release."
+        )
+
+    # Prefer the install tree; fall back to the cache if it is read-only.
+    for destination in (in_place, cached):
+        try:
+            destination.mkdir(parents=True, exist_ok=True)
+            _extract_module_graphs(zip_path, destination)
+        except OSError as exc:
+            logging.debug("Could not extract graphs to %s: %s", destination, exc)
+            continue
+        if _graphs_present(destination):
+            return destination
+
+    raise SystemExit(
+        f"[FATAL] Failed to extract {GRAPH_ZIP_NAME}. Tried {in_place} and {cached}. "
+        f"Set BLIMMP_CACHE_DIR to a writable directory and retry."
+    )
+
+
+def _extract_module_graphs(zip_path, destination) -> None:
+    """Extract `zip_path` into `destination`, normalising the archive layout.
+
+    The archive was built on macOS, so it carries __MACOSX metadata and wraps
+    everything in a redundant top-level folder.  Both are flattened away.
+    The source zip is deliberately left in place so extraction stays repeatable.
+    """
+    print(f"Extracting {os.path.basename(str(zip_path))} to {destination} ...")
+    destination = Path(destination)
+    with zipfile.ZipFile(str(zip_path), "r") as z:
+        z.extractall(str(destination))
+
+    macosx_path = destination / "__MACOSX"
+    if macosx_path.is_dir():
+        shutil.rmtree(str(macosx_path))
+
+    nested = destination / destination.name
+    if nested.is_dir():
+        for item in os.listdir(str(nested)):
+            shutil.move(str(nested / item), str(destination / item))
+        nested.rmdir()
+
+
+def validate_paths(paths: "Paths") -> None:
+    """Fail fast if a required data dependency is missing.
+
+    Several of these were silently optional, which let a mispackaged build
+    produce a structurally complete but entirely zero-valued result.
+    """
+    required = {
+        "KEGG module equations": paths.module_eq_json,
+        "KOfam KO list": paths.kofam_ko_list_path,
+        "module frequencies": paths.module_frequencies,
+        "module/KO reaction map": paths.module_reaction_dir,
+        "module descriptions": paths.module_descriptions_path,
+        "taxonomy frequency tables": paths.counts_dir,
+        "module neighbor data": paths.module_neighbor_dir,
+    }
+    missing = [f"  {label}: {path}" for label, path in required.items() if not Path(path).exists()]
+    if missing:
+        raise SystemExit(
+            "[FATAL] BLIMMP is missing required data files:\n"
+            + "\n".join(missing)
+            + "\nThe installation is incomplete -- reinstall BLIMMP or use a "
+              "container image built from a corrected release."
+        )
+
+
 def main():
     p = argparse.ArgumentParser(
     description='BLIMMP: Bayesian Likelihood Inference of Metabolic Module Presence. '
@@ -2237,44 +2365,19 @@ def main():
     GD   = os.path.join(HERE, "Graph_Dependencies")
     DD   = os.path.join(HERE, "Data_Dependencies")
 
-    ZIPS_TO_EXTRACT = {os.path.join(GD, "KEGG_Graphs_Generated_March26.zip") : os.path.join(GD, "KEGG_Graphs_Generated_March26")}
-
-    for zip_path, extract_to in ZIPS_TO_EXTRACT.items():
-        if os.path.isfile(zip_path):                   
-            os.makedirs(extract_to, exist_ok=True)
-            print(f"Extracting {os.path.basename(zip_path)}...")
-            with zipfile.ZipFile(zip_path, 'r') as z:
-                z.extractall(extract_to)
-
-                macosx_path = os.path.join(extract_to, "__MACOSX")
-                if os.path.isdir(macosx_path):
-                    import shutil
-                    shutil.rmtree(macosx_path)
-                    print(f"Removed MACOSX metadata folder")
-
-                extract_name = os.path.basename(extract_to)
-                nested = os.path.join(extract_to, extract_name)
-                if os.path.isdir(nested):
-                    import shutil
-                    for item in os.listdir(nested):
-                        shutil.move(os.path.join(nested, item), extract_to)
-                    os.rmdir(nested)
-                    print(f"  Flattened double-nested folder: {extract_name}/{extract_name} to {extract_name}/")
-            os.remove(zip_path)                           
-            print(f"Done. {extract_to}")
-
     paths = Paths(
         counts_dir = Path(DD)/ "ATB_Taxonomy_Frequency",
         onehop_dir = Path(GD)/ "ONE_HOP_NEIGHBOR_DATA",
         twohop_dir = Path(GD)/ "TWO_HOP_NEIGHBOR_DATA",
         module_neighbor_dir   = Path(GD)/ "MODULE_ALL_NEIGHBOR_DATA",
         module_eq_json = Path(GD)/ "KEGG_Module_Equations_Jan26.json",
-        module_json_dir = Path(GD)/ "KEGG_Graphs_Generated_March26",
+        module_json_dir = ensure_module_graphs(GD),
         kofam_ko_list_path = Path(DD)/ "ko_list.txt",
         module_frequencies = Path(DD)/ "module_freq.txt",
         module_reaction_dir = Path(GD)/ "module_ko_reaction.json",
         module_descriptions_path = Path(DD)/ "kegg_bacteria_modules.json"
     )
+    validate_paths(paths)
 
 
     BlimmpPipeline(cfg, paths).run()

Dockerfile

@@ -5,3 +5,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -r
 WORKDIR /app
 COPY . .
 RUN pip install --no-cache-dir .
+
+# Extract the KEGG module graphs now, while the filesystem is still writable.
+# Singularity mounts the image read-only at run time, so leaving this to first
+# use would fail on HPC. Also fails the build if the data files did not make it
+# into the installed package.
+RUN python -c "\
+from BLIMMP_Scripts.module_detection import ensure_module_graphs; \
+print('module graphs:', ensure_module_graphs())" \
+ && python -c "\
+import BLIMMP_Scripts, pathlib, sys; \
+root = pathlib.Path(BLIMMP_Scripts.__file__).parent; \
+required = ['Data_Dependencies/kegg_bacteria_modules.json', \
+            'Data_Dependencies/ko_list.txt', \
+            'Data_Dependencies/module_freq.txt', \
+            'Graph_Dependencies/KEGG_Module_Equations_Jan26.json', \
+            'Graph_Dependencies/module_ko_reaction.json']; \
+missing = [r for r in required if not (root / r).exists()]; \
+sys.exit('Missing packaged data files: ' + repr(missing)) if missing else print('packaged data OK')"

MANIFEST.in

@@ -1,7 +1,6 @@
-include BLIMMP_Scripts/Graph_Dependencies/KEGG_Module_Graphs.zip
-include BLIMMP_Scripts/Graph_Dependencies/KEGG_Module_Equations_Jan26.json
-include BLIMMP_Scripts/Graph_Dependencies/module_ko_reaction.json
-include BLIMMP_Scripts/Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*
-include BLIMMP_Scripts/Data_Dependencies/ATB_Taxonomy_Frequency/*
-include BLIMMP_Scripts/Data_Dependencies/ko_list.txt
-include BLIMMP_Scripts/Data_Dependencies/module_freq.txt
+include BLIMMP_Scripts/Graph_Dependencies/KEGG_Graphs_Generated_March26.zip
+include BLIMMP_Scripts/Graph_Dependencies/*.json
+include BLIMMP_Scripts/Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*.json
+include BLIMMP_Scripts/Data_Dependencies/*.txt
+include BLIMMP_Scripts/Data_Dependencies/*.json
+include BLIMMP_Scripts/Data_Dependencies/ATB_Taxonomy_Frequency/*.tsv

setup.py

@@ -14,13 +14,15 @@
     ],
     package_data={
         "BLIMMP_Scripts": [
-            "Graph_Dependencies/KEGG_Module_Graphs.zip",
-            "Graph_Dependencies/KEGG_Module_Equations_Jan26.json",
-            "Graph_Dependencies/module_ko_reaction.json",
-            "Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*",
-            "Data_Dependencies/ATB_Taxonomy_Frequency/*",
-            "Data_Dependencies/ko_list.txt",
-            "Data_Dependencies/module_freq.txt",
+            # Module graphs, shipped zipped and extracted on first use.
+            # module_detection.py hard-fails if this is missing, so a rename
+            # here can no longer silently produce all-zero results.
+            "Graph_Dependencies/KEGG_Graphs_Generated_March26.zip",
+            "Graph_Dependencies/*.json",
+            "Graph_Dependencies/MODULE_ALL_NEIGHBOR_DATA/*.json",
+            "Data_Dependencies/*.txt",
+            "Data_Dependencies/*.json",
+            "Data_Dependencies/ATB_Taxonomy_Frequency/*.tsv",
         ]
     },
     include_package_data=True,