feat(stack): add build-list line numbers, module dumps and version-change.tsv

.github/workflows/ci_test.yml

@@ -185,6 +185,7 @@ jobs:
             --test llms_txt \
             --test mcp_registry \
             --test mcp_stdio \
+            --test module_tree_dump_script \
             --test oom_profile \
             --test optimisation_criteria \
             --test pr_merge_order \

CHANGELOG.md

@@ -6,6 +6,19 @@ All notable changes to this project are documented here.
 
 ### Added
 
+- `stack provided` includes the one-based build-list line on every
+  `provided` and `unparsed` finding in its text report, as it already did for
+  hook, toolchain, family-lock, and licensed findings.
+- `scripts/dump-module-trees.sh` captures the live site, older EESSI, and
+  target EESSI `module -t avail` inventories in isolated, purged shells for
+  `stack gap`.
+  It sources the Lmod bash init when the shell has no `module` function.
+  It exits with the module name and the Lmod error when a purge or load fails.
+  It then writes no inventory for that tree.
+- `stack gap` writes `version-change.tsv`, one row per version in the site
+  and older-EESSI inventories. A module in the target is `moved` and names
+  its EESSI version; an excluded module is `site-only`; every other missing
+  route is `gone` until it lands.
 - `stack shard` splits one Jenkins build list into shards that can
   build at the same time. A line's set is its closure over
   `--easyconfigs` minus the modules in `--installed-modules-dir` Lmod

docs/orgmode/howto/build-list-provided.org

@@ -29,10 +29,10 @@ eb-stack stack provided \
 That command printed:
 
 #+begin_example
-provided GROMACS-2026.0-foss-2026.1.eb name=GROMACS list=tests/provided_fixtures/site.list
-provided FFTW-3.3.10-gompi-2026.1.eb name=FFTW list=tests/provided_fixtures/site.list
-provided fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/Python-3.12.3-foss-2025b.eb name=Python list=tests/provided_fixtures/site.list
-unparsed not-a-version.eb list=tests/provided_fixtures/site.list
+provided GROMACS-2026.0-foss-2026.1.eb name=GROMACS list=tests/provided_fixtures/site.list line=3
+provided FFTW-3.3.10-gompi-2026.1.eb name=FFTW list=tests/provided_fixtures/site.list line=4
+provided fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/Python-3.12.3-foss-2025b.eb name=Python list=tests/provided_fixtures/site.list line=5
+unparsed not-a-version.eb list=tests/provided_fixtures/site.list line=7
 toolchain fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/Python-3.12.3-foss-2025b.eb name=Python list=tests/provided_fixtures/site.list line=5 uses=foss-2025b; a site-layer line on EESSI 2026.06 must use a 2026.1 toolchain or GCCcore-15.2.0 below it (GCCcore-15.2.0, GCC-15.2.0, gfbf-2026.1, gompi-2026.1, foss-2026.1)
 hooks FFTW-3.3.10-gompi-2026.1.eb name=FFTW list=tests/provided_fixtures/site.list line=4 sets=--hooks=/sw/hooks/site.py; which replaces EESSI's hooks
 hits=3 unparsed=1 allowed=0 provided=3 toolchain=1 hooks=1 family=0 licensed=0

docs/orgmode/howto/module-gap.org

@@ -20,6 +20,62 @@ A toolchain whose version is the generation counts as on it, so
 generation in ~data/eessi-generations.toml~, such as
 ~llvm-compilers-21.1.8~.
 
+* Capture the live inventories
+
+The helper uses the shell's ~module~ function.
+A non-interactive shell often has none, so the helper then sources ~$LMOD_PKG/init/bash~, or ~$MODULESHOME/init/bash~ when the first is missing.
+It exits 2 when neither file exists.
+Its four arguments are the output directory, the site meta-module, the
+older EESSI module, and the target EESSI module. Use ~-~ for the site
+argument when the base module path is the live site tree.
+
+#+begin_src bash
+scripts/dump-module-trees.sh \
+  /tmp/module-dumps - EESSI/2025.06 EESSI/2026.06
+#+end_src
+
+Each inventory is captured in a separately purged subshell, so loading
+one EESSI version cannot add paths to the next dump. The files are
+~/tmp/module-dumps/site.txt~, ~eessi-older.txt~, and
+~eessi-target.txt~. Pass them as ~--site~, ~--older~, and
+~--installed~, respectively. They are the raw ~module -t avail~
+output; ~stack gap~ ignores path headers and default markers.
+
+The purge and load output for each tree goes to a ~.module.log~ file next to its inventory, such as ~eessi-target.module.log~.
+If the purge or the load fails, or ~module is-loaded~ does not report the module afterwards, the helper prints the module name and that log and exits 1.
+It writes no inventory for that tree, so ~stack gap~ never reads a purged base tree as an older or target inventory.
+Before the first dump, the helper deletes all three inventories from any earlier run, so a failed run leaves none of them behind.
+
+* Read the open-pull-request inventories
+
+The two ~fetch-prs.sh~ examples produce the JSON accepted by
+~--open-layer~ and ~--open-easyconfigs~. Run each script in the clone
+named in its comment:
+
+#+begin_src bash
+EB_STACK=$PWD
+(cd "$EB_EESSI_SOFTWARE_LAYER" && \
+  "$EB_STACK/examples/eessi-pr-overlap/fetch-prs.sh" \
+    EESSI/software-layer 2026.06 /tmp/open-layer)
+(cd "$EB_EASYCONFIGS_DEVELOP" && \
+  "$EB_STACK/examples/easyconfigs-pr-overlap/fetch-prs.sh" \
+    easybuilders/easybuild-easyconfigs /tmp/open-easyconfigs)
+
+layer_args=()
+for file in /tmp/open-layer/prs/*.json; do
+  layer_args+=(--open-layer "$file")
+done
+easyconfig_args=()
+for file in /tmp/open-easyconfigs/prs/*.json; do
+  easyconfig_args+=(--open-easyconfigs "$file")
+done
+#+end_src
+
+Add ~"${layer_args[@]}"~ and ~"${easyconfig_args[@]}"~ to the
+~stack gap~ command. The fetch scripts only read pull requests. A
+closed pull request disappears from their rewritten ~prs/~ directory
+on the next run.
+
 * Run the fixture
 
 The nine names are in ~tests/gap_routes/site.txt~. The easyconfig tree
@@ -74,6 +130,14 @@ Each route also has its own file: ~in-stack.tsv~, ~site-only.tsv~,
 and ~bump.tsv~. An ~open-layer~ match is the easystack line the pull
 request adds, not a dependency that line would build.
 
+~version-change.tsv~ is the user migration table. It keeps the module
+versions exactly as the live dumps spell them, including a toolchain
+suffix. An ~in-stack~ name is ~moved~ and names the target EESSI
+version. An ~excluded~ name is ~site-only~. Every other route is
+~gone~ from the opened stack until that route lands. A name with two
+old versions has two rows; multiple target versions are comma-separated.
+The columns are ~name~, ~old_version~, ~eessi_version~, and ~status~.
+
 ~eb_stack_gap~ writes the same files. ~open_layer~ and
 ~open_easyconfigs~ are arrays of the same pull-request JSON
 ~pr-overlap --pr~ reads.

docs/orgmode/reference/cli.org

@@ -677,7 +677,10 @@ version counts. The command writes ~gap.tsv~, one TSV per
 route (~site-only.tsv~ for ~excluded~, ~bump.tsv~ for ~no-recipe~, and
 ~in-stack.tsv~, ~filtered.tsv~, ~open-layer.tsv~, ~recipe.tsv~,
 ~open-easyconfigs.tsv~), ~roots.txt~ for ~stack batch --roots-file~ (the
-~recipe~ rows only), and ~bump-sources.txt~ for ~package bump --latest~.
+~recipe~ rows only), ~bump-sources.txt~ for ~package bump --latest~, and
+~version-change.tsv~. The last file has one row per old module version:
+~moved~ names the target EESSI version, ~site-only~ is an ~excluded~
+name, and every other missing route is ~gone~ until it lands.
 
 ~stack provided~ reads each ~--build-list~ (a Jenkins list or one
 easyconfig path per line; ~#~ comments are skipped) and fails when a
@@ -694,7 +697,7 @@ path, not left as site layer. ~--licensed~ and ~--licensed-file~ warn
 for group-restricted names; ~data/site-licensed.txt~ is that list for
 VASP, MATLAB, ANSYS, ORCA, and Gurobi. With neither flag there is no
 warning. ~--strict~ makes a remaining warning exit 1. Stdout lists each
-hit as ~provided <file> name=<Name> list=<path>~ and ends with
+hit as ~provided <file> name=<Name> list=<path> line=<N>~ and ends with
 ~hits= unparsed= allowed= provided= toolchain= hooks= family= licensed=~.
 ~--format~ is ~text~ by default, ~github~ for workflow commands, or
 ~json~. ~--json~ is the same as ~--format json~. See [[file:../howto/build-list-provided.org][fail a build list that names software the baseline provides]]

llms-full.txt

@@ -4051,10 +4051,10 @@ eb-stack stack provided \
 That command printed:
 
 ```
-provided GROMACS-2026.0-foss-2026.1.eb name=GROMACS list=tests/provided_fixtures/site.list
-provided FFTW-3.3.10-gompi-2026.1.eb name=FFTW list=tests/provided_fixtures/site.list
-provided fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/Python-3.12.3-foss-2025b.eb name=Python list=tests/provided_fixtures/site.list
-unparsed not-a-version.eb list=tests/provided_fixtures/site.list
+provided GROMACS-2026.0-foss-2026.1.eb name=GROMACS list=tests/provided_fixtures/site.list line=3
+provided FFTW-3.3.10-gompi-2026.1.eb name=FFTW list=tests/provided_fixtures/site.list line=4
+provided fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/Python-3.12.3-foss-2025b.eb name=Python list=tests/provided_fixtures/site.list line=5
+unparsed not-a-version.eb list=tests/provided_fixtures/site.list line=7
 toolchain fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/Python-3.12.3-foss-2025b.eb name=Python list=tests/provided_fixtures/site.list line=5 uses=foss-2025b; a site-layer line on EESSI 2026.06 must use a 2026.1 toolchain or GCCcore-15.2.0 below it (GCCcore-15.2.0, GCC-15.2.0, gfbf-2026.1, gompi-2026.1, foss-2026.1)
 hooks FFTW-3.3.10-gompi-2026.1.eb name=FFTW list=tests/provided_fixtures/site.list line=4 sets=--hooks=/sw/hooks/site.py; which replaces EESSI's hooks
 hits=3 unparsed=1 allowed=0 provided=3 toolchain=1 hooks=1 family=0 licensed=0
@@ -4167,6 +4167,62 @@ A toolchain whose version is the generation counts as on it, so
 generation in `data/eessi-generations.toml`, such as
 `llvm-compilers-21.1.8`.
 
+# Capture the live inventories
+
+The helper uses the shell's `module` function.
+A non-interactive shell often has none, so the helper then sources `$LMOD_PKG/init/bash`, or `$MODULESHOME/init/bash` when the first is missing.
+It exits 2 when neither file exists.
+Its four arguments are the output directory, the site meta-module, the
+older EESSI module, and the target EESSI module. Use `-` for the site
+argument when the base module path is the live site tree.
+
+```bash
+scripts/dump-module-trees.sh \
+  /tmp/module-dumps - EESSI/2025.06 EESSI/2026.06
+```
+
+Each inventory is captured in a separately purged subshell, so loading
+one EESSI version cannot add paths to the next dump. The files are
+`/tmp/module-dumps/site.txt`, `eessi-older.txt`, and
+`eessi-target.txt`. Pass them as `--site`, `--older`, and
+`--installed`, respectively. They are the raw `module -t avail`
+output; `stack gap` ignores path headers and default markers.
+
+The purge and load output for each tree goes to a `.module.log` file next to its inventory, such as `eessi-target.module.log`.
+If the purge or the load fails, or `module is-loaded` does not report the module afterwards, the helper prints the module name and that log and exits 1.
+It writes no inventory for that tree, so `stack gap` never reads a purged base tree as an older or target inventory.
+Before the first dump, the helper deletes all three inventories from any earlier run, so a failed run leaves none of them behind.
+
+# Read the open-pull-request inventories
+
+The two `fetch-prs.sh` examples produce the JSON accepted by
+`--open-layer` and `--open-easyconfigs`. Run each script in the clone
+named in its comment:
+
+```bash
+EB_STACK=$PWD
+(cd "$EB_EESSI_SOFTWARE_LAYER" && \
+  "$EB_STACK/examples/eessi-pr-overlap/fetch-prs.sh" \
+    EESSI/software-layer 2026.06 /tmp/open-layer)
+(cd "$EB_EASYCONFIGS_DEVELOP" && \
+  "$EB_STACK/examples/easyconfigs-pr-overlap/fetch-prs.sh" \
+    easybuilders/easybuild-easyconfigs /tmp/open-easyconfigs)
+
+layer_args=()
+for file in /tmp/open-layer/prs/*.json; do
+  layer_args+=(--open-layer "$file")
+done
+easyconfig_args=()
+for file in /tmp/open-easyconfigs/prs/*.json; do
+  easyconfig_args+=(--open-easyconfigs "$file")
+done
+```
+
+Add `"${layer_args[@]}"` and `"${easyconfig_args[@]}"` to the
+`stack gap` command. The fetch scripts only read pull requests. A
+closed pull request disappears from their rewritten `prs/` directory
+on the next run.
+
 # Run the fixture
 
 The nine names are in `tests/gap_routes/site.txt`. The easyconfig tree
@@ -4221,6 +4277,14 @@ Each route also has its own file: `in-stack.tsv`, `site-only.tsv`,
 and `bump.tsv`. An `open-layer` match is the easystack line the pull
 request adds, not a dependency that line would build.
 
+`version-change.tsv` is the user migration table. It keeps the module
+versions exactly as the live dumps spell them, including a toolchain
+suffix. An `in-stack` name is `moved` and names the target EESSI
+version. An `excluded` name is `site-only`. Every other route is
+`gone` from the opened stack until that route lands. A name with two
+old versions has two rows; multiple target versions are comma-separated.
+The columns are `name`, `old_version`, `eessi_version`, and `status`.
+
 `eb_stack_gap` writes the same files. `open_layer` and
 `open_easyconfigs` are arrays of the same pull-request JSON
 `pr-overlap --pr` reads.
@@ -5821,7 +5885,10 @@ version counts. The command writes `gap.tsv`, one TSV per
 route (`site-only.tsv` for `excluded`, `bump.tsv` for `no-recipe`, and
 `in-stack.tsv`, `filtered.tsv`, `open-layer.tsv`, `recipe.tsv`,
 `open-easyconfigs.tsv`), `roots.txt` for `stack batch --roots-file` (the
-`recipe` rows only), and `bump-sources.txt` for `package bump --latest`.
+`recipe` rows only), `bump-sources.txt` for `package bump --latest`, and
+`version-change.tsv`. The last file has one row per old module version:
+`moved` names the target EESSI version, `site-only` is an `excluded`
+name, and every other missing route is `gone` until it lands.
 
 `stack provided` reads each `--build-list` (a Jenkins list or one
 easyconfig path per line; `#` comments are skipped) and fails when a
@@ -5838,7 +5905,7 @@ path, not left as site layer. `--licensed` and `--licensed-file` warn
 for group-restricted names; `data/site-licensed.txt` is that list for
 VASP, MATLAB, ANSYS, ORCA, and Gurobi. With neither flag there is no
 warning. `--strict` makes a remaining warning exit 1. Stdout lists each
-hit as `provided <file> name=<Name> list=<path>` and ends with
+hit as `provided <file> name=<Name> list=<path> line=<N>` and ends with
 `hits= unparsed= allowed= provided= toolchain= hooks= family= licensed=`.
 `--format` is `text` by default, `github` for workflow commands, or
 `json`. `--json` is the same as `--format json`. See [fail a build list that names software the baseline provides](https://eb-stack.rgoswami.me/../howto/build-list-provided.html.md)

scripts/dump-module-trees.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Capture the live module inventories that `eb-stack stack gap` compares.
+#
+# Usage:
+#   dump-module-trees.sh OUT_DIR SITE_MODULE OLDER_EESSI_MODULE TARGET_EESSI_MODULE
+#
+# Pass - for SITE_MODULE to capture the base site tree without loading a
+# meta-module. When the shell has no `module` function, the helper sources
+# $LMOD_PKG/init/bash, then $MODULESHOME/init/bash. Each tree's purge and
+# load output goes to OUT_DIR/<name>.module.log. A failed purge or load, or a
+# meta-module that is not loaded afterwards, exits without writing that tree.
+set -euo pipefail
+
+readonly exit_failure=1
+readonly exit_usage=2
+
+if [[ $# -ne 4 ]]; then
+  echo "usage: $0 OUT_DIR SITE_MODULE OLDER_EESSI_MODULE TARGET_EESSI_MODULE" >&2
+  exit "$exit_usage"
+fi
+
+readonly out_dir=$1
+readonly site_module=$2
+readonly older_module=$3
+readonly target_module=$4
+
+module_is_function() {
+  [[ $(type -t module) == function ]]
+}
+
+# A non-interactive bash does not read the profile that defines `module`.
+initialize_module() {
+  local init
+  module_is_function && return 0
+  for init in "${LMOD_PKG:+$LMOD_PKG/init/bash}" \
+    "${MODULESHOME:+$MODULESHOME/init/bash}"; do
+    [[ -n $init && -e $init ]] || continue
+    # Lmod init is not nounset-safe, and a site's init may run a bare command
+    # that fails. Check for the function afterwards instead.
+    set +eu
+    # shellcheck disable=SC1090
+    source "$init"
+    set -eu
+    module_is_function && return 0
+  done
+  echo "dump-module-trees: the module command is not initialized," \
+    "and neither \$LMOD_PKG/init/bash nor \$MODULESHOME/init/bash exists" >&2
+  exit "$exit_usage"
+}
+
+initialize_module
+mkdir -p "$out_dir"
+
+fail_tree() {
+  local label=$1
+  local message=$2
+  local log=$3
+  echo "dump-module-trees: ${label} tree: ${message}" >&2
+  cat "$log" >&2
+  exit "$exit_failure"
+}
+
+dump_tree() {
+  local label=$1
+  local selector=$2
+  local output=$3
+  local temporary="${output}.tmp"
+  local log="${output%.txt}.module.log"
+
+  current_temporary=$temporary
+  (
+    module --force purge >"$log" 2>&1 ||
+      fail_tree "$label" "module --force purge failed" "$log"
+    if [[ $selector != - ]]; then
+      module load "$selector" >>"$log" 2>&1 ||
+        fail_tree "$label" "module load ${selector} failed" "$log"
+      module is-loaded "$selector" >>"$log" 2>&1 ||
+        fail_tree "$label" "${selector} is not loaded after module load" "$log"
+    fi
+    module -t avail >"$temporary" 2>&1
+  )
+  mv "$temporary" "$output"
+  current_temporary=
+  printf '%s\t%s\n' "$label" "$output"
+}
+
+# The inventory being written, removed on exit if it was never moved into place.
+current_temporary=
+trap '[[ -z $current_temporary ]] || rm -f -- "$current_temporary"' EXIT
+readonly site_output="$out_dir/site.txt"
+readonly older_output="$out_dir/eessi-older.txt"
+readonly target_output="$out_dir/eessi-target.txt"
+
+# A failed run must not leave an earlier run's inventory behind, including the
+# trees after the one that failed.
+rm -f -- "$site_output" "$older_output" "$target_output"
+dump_tree site "$site_module" "$site_output"
+dump_tree older "$older_module" "$older_output"
+dump_tree target "$target_module" "$target_output"