From 99bd1e01aa124d325ed01541e7905a09e8031bc2 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 09:25:13 +0000 Subject: [PATCH 1/2] feat(stack): warn when a foss solve marks an LLVM-family package as site layer A foss universe cannot see lfoss, lompi, lfbf, or llvm-compilers. The baseline drops an easystack entry on one of those toolchains. The stack diff then classifies the foss build of the same name as added. The solve prints that on standard error and still exits 0. Keep one lock per toolchain family, or pass `--baseline-also-toolchain` for that family. --- .github/workflows/ci_test.yml | 1 + CHANGELOG.md | 5 + docs/orgmode/howto/solve-lock.org | 41 +++++ docs/orgmode/reference/cli.org | 37 ++--- docs/orgmode/reference/glossary.org | 16 +- llms-full.txt | 94 +++++++---- src/easystack.rs | 76 +++++++++ src/generation_lint.rs | 244 ++++++++++++++++++++++++++++ src/lib.rs | 43 +++++ src/provided.rs | 16 +- tests/family_site_layer.rs | 76 +++++++++ 11 files changed, 580 insertions(+), 69 deletions(-) create mode 100644 tests/family_site_layer.rs diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index e400acdf..f248f17d 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -172,6 +172,7 @@ jobs: --test easyblock_layers \ --test easyblock_owed \ --test extension_provides \ + --test family_site_layer \ --test fetch_prs_script \ --test fix_cost \ --test from_commit \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bf1cb4d..316dd10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ All notable changes to this project are documented here. and older-EESSI inventories. A module in the target is `moved` and names its EESSI version; an excluded module is `site-only`; every other missing route is `gone` until it lands. +- `stack solve` warns on standard error when a foss solve's baseline easystack names an LLVM-family recipe (`lfoss`, `lompi`, `lfbf`, or `llvm-compilers`) and the stack diff marks that package added. + The foss universe cannot see that family, so the added row is the site layer by mistake. + The solve still exits 0. + Keep one lock per toolchain family, or pass `--baseline-also-toolchain` for that family. + `llvm-compilers-20.1.8` and `llvm-compilers-21.1.8` are the versions `data/eessi-generations.toml` lists for generations 2025b and 2026.1. - `stack shard` splits one Jenkins build list into shards that can build at the same time. A line's set is its closure over `--easyconfigs` minus the modules in `--installed-modules-dir` Lmod diff --git a/docs/orgmode/howto/solve-lock.org b/docs/orgmode/howto/solve-lock.org index 5cde31da..b3fecdbb 100644 --- a/docs/orgmode/howto/solve-lock.org +++ b/docs/orgmode/howto/solve-lock.org @@ -81,5 +81,46 @@ eb-stack stack order \ requirement. The hash file is one ~ ~ line per build. See [[file:../reference/cli.org][the CLI reference]]. +* One lock per toolchain family + +A foss solve only sees the foss hierarchy. +The LLVM family is ~lfoss~, ~lompi~, ~lfbf~, and ~llvm-compilers~. +OpenMPI under ~llvm-compilers-21.1.8~ is in that family. +~data/eessi-generations.toml~ lists ~llvm-compilers-20.1.8~ for generation 2025b and ~llvm-compilers-21.1.8~ for generation 2026.1, because an LLVM release is not a generation name. + +An easystack entry such as ~GROMACS-2025.0-lfoss-2025b.eb~ is absent from a foss baseline. +The foss solve then selects the foss build of the same name, and the stack diff marks that row added. +Added is the site layer. +Keep one lock per toolchain family: solve ~lfoss~ on its own. +Passing ~--baseline-also-toolchain lfoss/2025b~ keeps that family's hierarchy in the foss baseline, and the warning stays quiet for ~lfoss~. + +#+begin_src bash +eb-stack stack solve \ + --easyconfigs fixtures/gromacs_2025_to_next/easyconfigs \ + --toolchain foss/2025b \ + --root GROMACS \ + --baseline-easystack /tmp/lfoss.yml \ + --lock-out /tmp/foss.lock.json \ + --stack-diff-out /tmp/foss.diff.md +#+end_src + +~/tmp/lfoss.yml~ has one entry: + +#+begin_src yaml +easyconfigs: + - GROMACS-2025.0-lfoss-2025b.eb +#+end_src + +The solve still exits 0. +Standard error names the unresolved entry. +It also prints a warning that the added GROMACS row is the LLVM family: + +#+begin_src text +warning: GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS (fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b. +#+end_src + +The path in that warning is the foss easyconfig the solve selected. +A package the foss closure only version-bumps stays off this warning. + See [[file:../reference/policy.org][the policy reference]] and [[file:../reference/formats.org][the format reference]]. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 741a5eab..f90b6d63 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -579,26 +579,23 @@ requirement; a builddependency is dashed. A ~.svg~ or ~.png~ path is rendered with the Graphviz ~dot~ program. Any other suffix, including ~.dot~, is the DOT source. The MCP field ~dot_out~ writes the same file. -The baseline is either a previous generation (~--baseline-easyconfigs~, by -default the first ~--easyconfigs~ tree) or what a set of easystacks installs -(~--baseline-easystack~, repeatable). An easystack baseline is every entry -plus the closure its recipes pin, read from the ~--easyconfigs~ trees and kept -to the policy toolchain's hierarchy. An entry pinned with ~from-commit~ is read -from that commit of ~--baseline-commits-from~ when given, and matched by -filename otherwise. Entries that name no recipe are listed on standard error. -~--baseline-also-toolchain~ keeps another family's hierarchy in the baseline, -such as ~lfoss/2026.1~ beside a ~foss/2026.1~ solve. ~--filter-deps~ drops each named dependency before the solve, together -with the subtree under it, using EasyBuild's matching: a bare name, -~name=version~, or ~name=[low:high]~. ~[~ and ~]~ include a limit, the -opposite bracket excludes it, and an empty side is open. The last spec for -a name wins. Pass EESSI-extend's ~EASYBUILD_FILTER_DEPS~ list when solving -against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat -layer already provides. -Against an EESSI version's easystacks, ~unchanged~ in the stack diff means the -software layer already provides that module, and -~--build-list-excludes-baseline~ drops those modules (same name, version, -toolchain and versionsuffix) from the build list, leaving what the site layer -builds on top in dependency order. +The baseline is either a previous generation (~--baseline-easyconfigs~, by default the first ~--easyconfigs~ tree) or what a set of easystacks installs (~--baseline-easystack~, repeatable). +An easystack baseline is every entry plus the closure its recipes pin, read from the ~--easyconfigs~ trees and kept to the policy toolchain's hierarchy. +An entry pinned with ~from-commit~ is read from that commit of ~--baseline-commits-from~ when given, and matched by filename otherwise. +Entries that name no recipe are listed on standard error. +~--baseline-also-toolchain~ keeps another family's hierarchy in the baseline, such as ~lfoss/2026.1~ beside a ~foss/2026.1~ solve. +A foss solve cannot see the LLVM family (~lfoss~, ~lompi~, ~lfbf~, ~llvm-compilers~). +A baseline easystack can name one of those toolchains. +If the stack diff then marks that package added, the solve warns on standard error and still exits 0. +The added row is the site layer because the foss universe dropped the LLVM recipe. +Keep one lock per toolchain family, or pass ~--baseline-also-toolchain~ for that family so its hierarchy stays in the baseline. +See [[file:../howto/solve-lock.org][one lock per toolchain family]]. +~--filter-deps~ drops each named dependency before the solve, together with the subtree under it, using EasyBuild's matching: a bare name, ~name=version~, or ~name=[low:high]~. +~[~ and ~]~ include a limit. +The opposite bracket excludes it, and an empty side is open. +The last spec for a name wins. +Pass EESSI-extend's ~EASYBUILD_FILTER_DEPS~ list when solving against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat layer already provides. +Against an EESSI version's easystacks, ~unchanged~ in the stack diff means the software layer already provides that module, and ~--build-list-excludes-baseline~ drops those modules (same name, version, toolchain and versionsuffix) from the build list, leaving what the site layer builds on top in dependency order. A Jenkins build list is one easyconfig filename per line with optional ~eb~ flags, ~#~ for comments. ~--roots-from-build-list~ takes the package name of diff --git a/docs/orgmode/reference/glossary.org b/docs/orgmode/reference/glossary.org index 1a3f0caa..7f32a2e5 100644 --- a/docs/orgmode/reference/glossary.org +++ b/docs/orgmode/reference/glossary.org @@ -29,14 +29,14 @@ Flags and exit status stay in the [[file:cli.org][command-line reference]]. ~--baseline-easyconfigs~ and the first ~--easyconfigs~ tree is the baseline. See [[file:../howto/emit-reports.org][emit a build list and stack diff]]. -- toolchain family :: The toolchain name without its version (~foss~, - ~GCCcore~, ~gfbf~). A bump retargets an explicit dependency's family - onto the new generation. ~--baseline-also-toolchain~ keeps another - family's hierarchy in the baseline. This manual does not use the phrase - "family lock": a lock is one solved selection, and a family is only the - toolchain name. On an architecture, family is the text before the first - slash (~x86_64~, ~aarch64~). See - [[file:../howto/arch-unsupported.org][refuse a plan an architecture cannot build]]. +- toolchain family :: The toolchain name without its version (~foss~, ~GCCcore~, ~gfbf~). + A bump retargets an explicit dependency's family onto the new generation. + ~--baseline-also-toolchain~ keeps another family's hierarchy in the baseline. + Keep one lock per toolchain family. + A foss solve that marks an ~lfoss~, ~lompi~, ~lfbf~, or ~llvm-compilers~ package added warns on standard error; see [[file:../howto/solve-lock.org][one lock per toolchain family]]. + This manual does not use the phrase "family lock": a lock is one solved selection, and a family is only the toolchain name. + On an architecture, family is the text before the first slash (~x86_64~, ~aarch64~). + See [[file:../howto/arch-unsupported.org][refuse a plan an architecture cannot build]]. - lock :: The Resolvo result for one selection. A package bundle has ~locks/.lock.json~ per profile. ~stack solve~ writes diff --git a/llms-full.txt b/llms-full.txt index cfe70331..c57f1fe0 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1832,6 +1832,47 @@ eb-stack stack order \ requirement. The hash file is one ` ` line per build. See [the CLI reference](https://eb-stack.rgoswami.me/../reference/cli.html.md). +# One lock per toolchain family + +A foss solve only sees the foss hierarchy. +The LLVM family is `lfoss`, `lompi`, `lfbf`, and `llvm-compilers`. +OpenMPI under `llvm-compilers-21.1.8` is in that family. +`data/eessi-generations.toml` lists `llvm-compilers-20.1.8` for generation 2025b and `llvm-compilers-21.1.8` for generation 2026.1, because an LLVM release is not a generation name. + +An easystack entry such as `GROMACS-2025.0-lfoss-2025b.eb` is absent from a foss baseline. +The foss solve then selects the foss build of the same name, and the stack diff marks that row added. +Added is the site layer. +Keep one lock per toolchain family: solve `lfoss` on its own. +Passing `--baseline-also-toolchain lfoss/2025b` keeps that family's hierarchy in the foss baseline, and the warning stays quiet for `lfoss`. + +```bash +eb-stack stack solve \ + --easyconfigs fixtures/gromacs_2025_to_next/easyconfigs \ + --toolchain foss/2025b \ + --root GROMACS \ + --baseline-easystack /tmp/lfoss.yml \ + --lock-out /tmp/foss.lock.json \ + --stack-diff-out /tmp/foss.diff.md +``` + +`/tmp/lfoss.yml` has one entry: + +```yaml +easyconfigs: + - GROMACS-2025.0-lfoss-2025b.eb +``` + +The solve still exits 0. +Standard error names the unresolved entry. +It also prints a warning that the added GROMACS row is the LLVM family: + +```text +warning: GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS (fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b. +``` + +The path in that warning is the foss easyconfig the solve selected. +A package the foss closure only version-bumps stays off this warning. + See [the policy reference](https://eb-stack.rgoswami.me/../reference/policy.html.md) and [the format reference](https://eb-stack.rgoswami.me/../reference/formats.html.md). @@ -5124,14 +5165,14 @@ Flags and exit status stay in the [command-line reference](https://eb-stack.rgos `--baseline-easyconfigs` and the first `--easyconfigs` tree is the baseline. See [emit a build list and stack diff](https://eb-stack.rgoswami.me/../howto/emit-reports.html.md). -- toolchain family :: The toolchain name without its version (`foss`, - `GCCcore`, `gfbf`). A bump retargets an explicit dependency's family - onto the new generation. `--baseline-also-toolchain` keeps another - family's hierarchy in the baseline. This manual does not use the phrase - "family lock": a lock is one solved selection, and a family is only the - toolchain name. On an architecture, family is the text before the first - slash (`x86_64`, `aarch64`). See - [refuse a plan an architecture cannot build](https://eb-stack.rgoswami.me/../howto/arch-unsupported.html.md). +- toolchain family :: The toolchain name without its version (`foss`, `GCCcore`, `gfbf`). + A bump retargets an explicit dependency's family onto the new generation. + `--baseline-also-toolchain` keeps another family's hierarchy in the baseline. + Keep one lock per toolchain family. + A foss solve that marks an `lfoss`, `lompi`, `lfbf`, or `llvm-compilers` package added warns on standard error; see [one lock per toolchain family](https://eb-stack.rgoswami.me/../howto/solve-lock.html.md). + This manual does not use the phrase "family lock": a lock is one solved selection, and a family is only the toolchain name. + On an architecture, family is the text before the first slash (`x86_64`, `aarch64`). + See [refuse a plan an architecture cannot build](https://eb-stack.rgoswami.me/../howto/arch-unsupported.html.md). - lock :: The Resolvo result for one selection. A package bundle has `locks/.lock.json` per profile. `stack solve` writes @@ -5787,26 +5828,23 @@ requirement; a builddependency is dashed. A `.svg` or `.png` path is rendered with the Graphviz `dot` program. Any other suffix, including `.dot`, is the DOT source. The MCP field `dot_out` writes the same file. -The baseline is either a previous generation (`--baseline-easyconfigs`, by -default the first `--easyconfigs` tree) or what a set of easystacks installs -(`--baseline-easystack`, repeatable). An easystack baseline is every entry -plus the closure its recipes pin, read from the `--easyconfigs` trees and kept -to the policy toolchain's hierarchy. An entry pinned with `from-commit` is read -from that commit of `--baseline-commits-from` when given, and matched by -filename otherwise. Entries that name no recipe are listed on standard error. -`--baseline-also-toolchain` keeps another family's hierarchy in the baseline, -such as `lfoss/2026.1` beside a `foss/2026.1` solve. `--filter-deps` drops each named dependency before the solve, together -with the subtree under it, using EasyBuild's matching: a bare name, -`name=version`, or `name=[low:high]`. `[` and `]` include a limit, the -opposite bracket excludes it, and an empty side is open. The last spec for -a name wins. Pass EESSI-extend's `EASYBUILD_FILTER_DEPS` list when solving -against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat -layer already provides. -Against an EESSI version's easystacks, `unchanged` in the stack diff means the -software layer already provides that module, and -`--build-list-excludes-baseline` drops those modules (same name, version, -toolchain and versionsuffix) from the build list, leaving what the site layer -builds on top in dependency order. +The baseline is either a previous generation (`--baseline-easyconfigs`, by default the first `--easyconfigs` tree) or what a set of easystacks installs (`--baseline-easystack`, repeatable). +An easystack baseline is every entry plus the closure its recipes pin, read from the `--easyconfigs` trees and kept to the policy toolchain's hierarchy. +An entry pinned with `from-commit` is read from that commit of `--baseline-commits-from` when given, and matched by filename otherwise. +Entries that name no recipe are listed on standard error. +`--baseline-also-toolchain` keeps another family's hierarchy in the baseline, such as `lfoss/2026.1` beside a `foss/2026.1` solve. +A foss solve cannot see the LLVM family (`lfoss`, `lompi`, `lfbf`, `llvm-compilers`). +A baseline easystack can name one of those toolchains. +If the stack diff then marks that package added, the solve warns on standard error and still exits 0. +The added row is the site layer because the foss universe dropped the LLVM recipe. +Keep one lock per toolchain family, or pass `--baseline-also-toolchain` for that family so its hierarchy stays in the baseline. +See [one lock per toolchain family](https://eb-stack.rgoswami.me/../howto/solve-lock.html.md). +`--filter-deps` drops each named dependency before the solve, together with the subtree under it, using EasyBuild's matching: a bare name, `name=version`, or `name=[low:high]`. +`[` and `]` include a limit. +The opposite bracket excludes it, and an empty side is open. +The last spec for a name wins. +Pass EESSI-extend's `EASYBUILD_FILTER_DEPS` list when solving against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat layer already provides. +Against an EESSI version's easystacks, `unchanged` in the stack diff means the software layer already provides that module, and `--build-list-excludes-baseline` drops those modules (same name, version, toolchain and versionsuffix) from the build list, leaving what the site layer builds on top in dependency order. A Jenkins build list is one easyconfig filename per line with optional `eb` flags, `#` for comments. `--roots-from-build-list` takes the package name of diff --git a/src/easystack.rs b/src/easystack.rs index 2a04fe72..fe671ef2 100644 --- a/src/easystack.rs +++ b/src/easystack.rs @@ -215,6 +215,57 @@ pub struct EessiLayout { pub members: Vec, } +/// Toolchain names a foss solve cannot see. +/// +/// The LLVM family is `lfoss`, `lompi`, `lfbf`, and `llvm-compilers`. +/// OpenMPI under `llvm-compilers-21.1.8` is in that family. A foss solve's +/// universe is the foss hierarchy, so a package that exists only on one of +/// these toolchains is absent from the foss lock and a later diff marks it +/// site layer. `data/eessi-generations.toml` lists `llvm-compilers-20.1.8` +/// (generation 2025b) and `llvm-compilers-21.1.8` (generation 2026.1) by +/// hand, because an LLVM release is not a generation name. +pub const LLVM_FAMILY: &[&str] = &["lfoss", "lompi", "lfbf", "llvm-compilers"]; + +/// The LLVM family `label` names, or `None` when it is some other toolchain. +/// +/// `label` is a toolchain name (`lfoss`) or a `name-version` label whose +/// version starts with a digit (`llvm-compilers-21.1.8`, `lfoss-2026.1`). +/// The longest family name wins, so `llvm-compilers` is not cut at a shorter +/// prefix. +pub fn llvm_family_name(label: &str) -> Option<&'static str> { + LLVM_FAMILY + .iter() + .copied() + .filter(|family| llvm_family_version_rest(label, family).is_some()) + .max_by_key(|family| family.len()) +} + +/// The version after `family` in `label`, when `label` is that family. +/// +/// `llvm-compilers-21.1.8` and `llvm-compilers` yield `Some("21.1.8")` and +/// `Some("")`. A label that is only the family name has an empty version. +pub fn llvm_family_version(label: &str, family: &str) -> Option { + let rest = llvm_family_version_rest(label, family)?; + Some(rest.to_string()) +} + +/// `Some` when `label` is `family` or `family-` with a digit version. +/// The returned string is the version, empty when `label` is the bare name. +fn llvm_family_version_rest<'a>(label: &'a str, family: &str) -> Option<&'a str> { + if label.eq_ignore_ascii_case(family) { + return Some(""); + } + if label.len() <= family.len() || !label[..family.len()].eq_ignore_ascii_case(family) { + return None; + } + let rest = label[family.len()..].strip_prefix('-')?; + if rest.starts_with(|c: char| c.is_ascii_digit()) { + Some(rest) + } else { + None + } +} + /// Toolchain names whose presence in a filename means it is not a system /// recipe, followed by the toolchain version up to the next dash. static TOOLCHAIN_IN_FILENAME: LazyLock = LazyLock::new(|| { @@ -408,6 +459,31 @@ mod tests { assert_eq!(split("foss-2025b.eb"), None); } + #[test] + fn llvm_family_name_takes_the_longest_family_and_a_digit_version() { + assert_eq!(llvm_family_name("lfoss"), Some("lfoss")); + assert_eq!(llvm_family_name("LFOSS-2026.1"), Some("lfoss")); + assert_eq!(llvm_family_name("lompi-2025b"), Some("lompi")); + assert_eq!(llvm_family_name("lfbf"), Some("lfbf")); + assert_eq!( + llvm_family_name("llvm-compilers-21.1.8"), + Some("llvm-compilers") + ); + assert_eq!( + llvm_family_version("llvm-compilers-21.1.8", "llvm-compilers").as_deref(), + Some("21.1.8") + ); + assert_eq!( + llvm_family_version("lfoss-2025b", "lfoss").as_deref(), + Some("2025b") + ); + assert_eq!(llvm_family_version("lfoss", "lfoss").as_deref(), Some("")); + assert_eq!(llvm_family_name("foss-2025b"), None); + assert_eq!(llvm_family_name("GCCcore-15.2.0"), None); + assert_eq!(llvm_family_name("lfoss-v2025b"), None); + assert_eq!(llvm_family_version("foss-2025b", "lfoss"), None); + } + #[test] fn eessi_easystacks_split_by_file_and_keep_options() { let mut options = EasystackOptions::new(); diff --git a/src/generation_lint.rs b/src/generation_lint.rs index d5c0a78c..2e5ed100 100644 --- a/src/generation_lint.rs +++ b/src/generation_lint.rs @@ -633,6 +633,156 @@ pub fn run_generation_lint(request: &GenerationLintRequest) -> Result Vec { + if crate::easystack::llvm_family_name(&policy.name).is_some() { + return Vec::new(); + } + let suppressed: BTreeSet<&str> = also + .iter() + .filter_map(|tc| crate::easystack::llvm_family_name(&tc.name)) + .collect(); + let mut added_by_name: BTreeMap)> = BTreeMap::new(); + for (name, path) in added { + let entry = added_by_name + .entry(name.to_ascii_lowercase()) + .or_insert_with(|| (name.clone(), BTreeSet::new())); + if !path.is_empty() { + entry.1.insert(path.clone()); + } + } + + struct Group { + name: String, + family: &'static str, + versions: BTreeSet, + files: BTreeSet, + from_easystack: bool, + solved: BTreeSet, + } + + let mut groups: BTreeMap<(String, &'static str), Group> = BTreeMap::new(); + for file in easystack_files { + let Some(split) = filename_toolchain(file) else { + continue; + }; + let Some(family) = crate::easystack::llvm_family_name(&split.toolchain) else { + continue; + }; + if suppressed.contains(family) { + continue; + } + let Some((display, solved)) = added_by_name.get(&split.name.to_ascii_lowercase()) else { + continue; + }; + let key = (split.name.to_ascii_lowercase(), family); + let group = groups.entry(key).or_insert_with(|| Group { + name: display.clone(), + family, + versions: BTreeSet::new(), + files: BTreeSet::new(), + from_easystack: true, + solved: solved.clone(), + }); + let base = file.rsplit(['/', '\\']).next().unwrap_or(file.as_str()); + group.files.insert(base.to_string()); + if let Some(version) = crate::easystack::llvm_family_version(&split.toolchain, family) { + if !version.is_empty() { + group.versions.insert(version); + } + } + } + for (name, path) in added { + if path.is_empty() { + continue; + } + let Some(split) = filename_toolchain(path) else { + continue; + }; + let Some(family) = crate::easystack::llvm_family_name(&split.toolchain) else { + continue; + }; + if suppressed.contains(family) { + continue; + } + let key = (name.to_ascii_lowercase(), family); + if groups.contains_key(&key) { + continue; + } + let group = groups.entry(key).or_insert_with(|| Group { + name: name.clone(), + family, + versions: BTreeSet::new(), + files: BTreeSet::new(), + from_easystack: false, + solved: BTreeSet::new(), + }); + group.solved.insert(path.clone()); + if let Some(version) = crate::easystack::llvm_family_version(&split.toolchain, family) { + if !version.is_empty() { + group.versions.insert(version); + } + } + } + + groups + .into_values() + .map(|group| { + let version = if group.versions.len() == 1 { + group.versions.iter().next().expect("one version").clone() + } else { + "".to_string() + }; + let flag = format!("{family}/{version}", family = group.family); + if group.from_easystack { + let files = group.files.into_iter().collect::>().join(", "); + let solved = if group.solved.is_empty() { + group.name.clone() + } else { + format!( + "{name} ({paths})", + name = group.name, + paths = group.solved.into_iter().collect::>().join(", ") + ) + }; + format!( + "{name} ({files}) is the LLVM family ({family}). A {policy} solve cannot reach it, so the site layer includes {solved}. Keep one lock per toolchain family: solve {family} on its own, or pass --baseline-also-toolchain {flag}.", + name = group.name, + family = group.family, + policy = policy.name, + ) + } else { + let solved = group.solved.into_iter().collect::>().join(", "); + format!( + "{name} ({solved}) is the LLVM family ({family}). This {policy} solve marked it site layer. Keep one lock per toolchain family: solve {family} on its own, or pass --baseline-also-toolchain {flag}.", + name = group.name, + family = group.family, + policy = policy.name, + ) + } + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -893,4 +1043,98 @@ mod tests { Some("2025b") ); } + + fn foss() -> Toolchain { + Toolchain { + name: "foss".into(), + version: "2025b".into(), + } + } + + #[test] + fn a_foss_solve_warns_when_an_lfoss_easystack_entry_is_site_layer() { + let added = [( + "GROMACS".into(), + "fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb" + .into(), + )]; + let files = ["GROMACS-2025.0-lfoss-2025b.eb".to_string()]; + let warnings = llvm_site_layer_warnings(&foss(), &[], &files, &added); + assert_eq!( + warnings, + vec![ + "GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS (fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b.".to_string() + ] + ); + } + + #[test] + fn an_also_toolchain_or_an_llvm_policy_suppresses_the_site_layer_warning() { + let added = [( + "OpenMPI".into(), + "easyconfigs/llvm-compilers-21.1.8/OpenMPI-5.0.7-llvm-compilers-21.1.8.eb".into(), + )]; + let files = ["OpenMPI-5.0.7-llvm-compilers-21.1.8.eb".to_string()]; + let also = [Toolchain { + name: "llvm-compilers".into(), + version: "21.1.8".into(), + }]; + assert!(llvm_site_layer_warnings(&foss(), &also, &files, &added).is_empty()); + let lfoss = Toolchain { + name: "lfoss".into(), + version: "2026.1".into(), + }; + assert!(llvm_site_layer_warnings(&lfoss, &[], &files, &added).is_empty()); + } + + #[test] + fn a_version_bump_and_a_foss_easystack_entry_are_not_site_layer() { + let files = [ + "GROMACS-2024.4-foss-2025b.eb".to_string(), + "OpenBLAS-0.3.32-lfoss-2025b.eb".to_string(), + ]; + let added = [( + "FFTW".into(), + "easyconfigs/foss-2025b/FFTW-3.3.10-foss-2025b.eb".into(), + )]; + assert!(llvm_site_layer_warnings(&foss(), &[], &files, &added).is_empty()); + } + + #[test] + fn a_solved_llvm_easyconfig_with_no_easystack_entry_is_site_layer() { + let added = [( + "OpenBLAS".into(), + "easyconfigs/lfbf-2026.1/OpenBLAS-0.3.32-lfbf-2026.1.eb".into(), + )]; + let warnings = llvm_site_layer_warnings(&foss(), &[], &[], &added); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert_eq!( + warnings[0], + "OpenBLAS (easyconfigs/lfbf-2026.1/OpenBLAS-0.3.32-lfbf-2026.1.eb) is the LLVM family (lfbf). This foss solve marked it site layer. Keep one lock per toolchain family: solve lfbf on its own, or pass --baseline-also-toolchain lfbf/2026.1." + ); + } + + #[test] + fn two_versions_of_one_llvm_family_leave_the_flag_version_unnamed() { + let added = [( + "OpenBLAS".into(), + "easyconfigs/foss-2025b/OpenBLAS-0.3.32-foss-2025b.eb".into(), + )]; + let files = [ + "OpenBLAS-0.3.32-lfoss-2026.1.eb".to_string(), + "OpenBLAS-0.3.30-lfoss-2025b.eb".to_string(), + ]; + let warnings = llvm_site_layer_warnings(&foss(), &[], &files, &added); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!( + warnings[0].contains( + "OpenBLAS (OpenBLAS-0.3.30-lfoss-2025b.eb, OpenBLAS-0.3.32-lfoss-2026.1.eb) is the LLVM family (lfoss)." + ), + "{warnings:?}" + ); + assert!( + warnings[0].contains("--baseline-also-toolchain lfoss/"), + "{warnings:?}" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 2b375509..ca5609d6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -878,10 +878,53 @@ pub fn solve_with_baseline_source( anyhow::anyhow!("{text}{hint}{constraints}") })?; validate_lock_deps(&lock, &universe.candidates).map_err(|e| anyhow::anyhow!(e))?; + if let Some(base) = baseline.as_ref() { + let (easystack_files, also) = llvm_baseline_inputs(baseline_source)?; + let added: Vec<(String, String)> = classify_stack_diff(base, &lock) + .into_iter() + .filter(|change| change.kind == PackageChangeKind::Added) + .map(|change| { + ( + change.name, + change.solved_easyconfig_path.unwrap_or_default(), + ) + }) + .collect(); + for warning in generation_lint::llvm_site_layer_warnings( + &policy.toolchain, + &also, + &easystack_files, + &added, + ) { + eprintln!("warning: {warning}"); + } + } write_lock_sbom_and_extras(&lock, baseline.as_ref(), lock_out, sbom_out, extra)?; Ok(lock) } +/// Easyconfig filenames a baseline easystack names, and the extra families +/// the solve was asked to keep. A tree baseline has neither. +fn llvm_baseline_inputs(source: BaselineSource<'_>) -> Result<(Vec, Vec)> { + let BaselineSource::Easystacks { + files, + also_toolchains, + .. + } = source + else { + return Ok((Vec::new(), Vec::new())); + }; + let mut names = Vec::new(); + for path in files { + let text = + std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + let entries = + parse_easystack_entries(&text).with_context(|| format!("parse {}", path.display()))?; + names.extend(entries.into_iter().map(|entry| entry.file)); + } + Ok((names, also_toolchains.to_vec())) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/provided.rs b/src/provided.rs index 1ea83b83..cbea19a4 100644 --- a/src/provided.rs +++ b/src/provided.rs @@ -15,7 +15,7 @@ use crate::cli_parse::usage; use crate::domain::{StackLock, Toolchain}; -use crate::easystack::filename_toolchain; +use crate::easystack::{filename_toolchain, llvm_family_name}; use crate::gap::{names_from_listing, software_name}; use crate::hierarchy::known_hierarchy; use crate::jenkins::parse_build_list; @@ -25,9 +25,6 @@ use serde::Serialize; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -/// Toolchain names of the LLVM family. A foss solve cannot see them. -const LLVM_FAMILY: &[&str] = &["lfoss", "lompi", "lfbf", "llvm-compilers"]; - /// How `stack provided` prints. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProvidedFormat { @@ -558,19 +555,12 @@ fn toolchain_detail(file: &str, families: &[FamilyLock], allowed: &[String]) -> } fn llvm_suffix(toolchain_name: &str, families: &[FamilyLock]) -> String { - if !LLVM_FAMILY - .iter() - .any(|family| family.eq_ignore_ascii_case(toolchain_name)) - { + if llvm_family_name(toolchain_name).is_none() { return String::new(); } let locks: Vec<&FamilyLock> = families .iter() - .filter(|family| { - LLVM_FAMILY - .iter() - .any(|name| name.eq_ignore_ascii_case(&family.toolchain_name)) - }) + .filter(|family| llvm_family_name(&family.toolchain_name).is_some()) .collect(); if locks.is_empty() { return format!(" {toolchain_name} is the LLVM family, not a site-layer toolchain"); diff --git a/tests/family_site_layer.rs b/tests/family_site_layer.rs new file mode 100644 index 00000000..eef1715c --- /dev/null +++ b/tests/family_site_layer.rs @@ -0,0 +1,76 @@ +//! A foss solve warns when an LLVM-family easystack entry looks like site layer. + +use std::path::PathBuf; +use std::process::Command; + +fn easyconfigs() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/gromacs_2025_to_next/easyconfigs") +} + +fn solve(stack: &str) -> std::process::Output { + let tmp = tempfile::tempdir().unwrap(); + let easystack = tmp.path().join("baseline.yml"); + std::fs::write(&easystack, stack).unwrap(); + let lock = tmp.path().join("stack.lock.json"); + let diff = tmp.path().join("stack.diff.md"); + let output = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args(["stack", "solve", "--easyconfigs"]) + .arg(easyconfigs()) + .args(["--toolchain", "foss/2025b", "--root", "GROMACS"]) + .arg("--baseline-easystack") + .arg(&easystack) + .arg("--lock-out") + .arg(&lock) + .arg("--stack-diff-out") + .arg(&diff) + .output() + .expect("run eb-stack"); + // Keep the temp dir alive until the command has finished reading it. + drop(tmp); + output +} + +#[test] +fn a_foss_solve_warns_that_an_lfoss_easystack_entry_is_not_site_layer() { + let output = solve("easyconfigs:\n - GROMACS-2025.0-lfoss-2025b.eb\n"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "solve failed: {stderr}\n{}", + String::from_utf8_lossy(&output.stdout) + ); + let solved = easyconfigs().join("foss-2025b/GROMACS-2025.0-foss-2025b.eb"); + let expected = format!( + "warning: GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS ({}). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b.", + solved.display() + ); + assert!( + stderr.contains(&expected), + "stderr missing the LLVM warning:\n{stderr}\nexpected:\n{expected}" + ); + assert!( + stderr.contains("GROMACS-2025.0-lfoss-2025b.eb"), + "the unresolved easystack entry is still reported: {stderr}" + ); +} + +#[test] +fn a_version_bumped_lfoss_name_is_not_reported_as_site_layer() { + let output = solve( + "easyconfigs:\n - GROMACS-2024.4-foss-2025b.eb\n - OpenBLAS-0.3.32-lfoss-2025b.eb\n", + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "solve failed: {stderr}\n{}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + stderr.contains("OpenBLAS-0.3.32-lfoss-2025b.eb"), + "an entry with no recipe is reported: {stderr}" + ); + assert!( + !stderr.contains("LLVM family"), + "a version bump is not site layer: {stderr}" + ); +} From f081fe6c6179de18cadd621e96ca387683a2615b Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 09:46:04 +0000 Subject: [PATCH 2/2] feat(stack): record reach classes and write a backport list The `component-reach` (schema 2) and `rebuild-set` (schema 5) outputs record cross-user, network-facing, local-only, not-reachable, or unclassified. The `stack sbom --backport-list` option writes the modules whose class justifies a backport, using Open Source Vulnerabilities (OSV) ranges and the software bill of materials (SBOM) reverse closure. Modules classed local-only, not-reachable, or unclassified stay in the recorded section. --- .github/workflows/ci_test.yml | 1 + .gitignore | 1 + CHANGELOG.md | 16 + .../orgmode/explanation/sbom-blast-radius.org | 50 +- docs/orgmode/howto/component-reach.org | 13 +- docs/orgmode/howto/rebuild-set.org | 6 + docs/orgmode/howto/solve-lock.org | 9 + docs/orgmode/reference/cli.org | 64 ++- docs/orgmode/reference/formats.org | 47 +- docs/orgmode/reference/glossary.org | 16 +- llms-full.txt | 205 +++++--- skills/mcp/SKILL.md | 10 +- src/backport.rs | 459 ++++++++++++++++++ src/component_reach.rs | 22 +- src/findings.rs | 23 + src/fix_cost.rs | 1 + src/lib.rs | 1 + src/main.rs | 56 ++- src/mcp.rs | 41 +- src/rebuild_set.rs | 30 +- src/site_commands.rs | 9 +- .../eb/App-1.0-GCCcore-15.2.0.eb | 4 + .../eb/Mystery-1.0-GCCcore-15.2.0.eb | 3 + .../eb/WinOnly-1.0-GCCcore-15.2.0.eb | 3 + .../eb/cURL-8.17.0-GCCcore-15.2.0.eb | 3 + .../eb/git-2.52.0-GCCcore-15.2.0.eb | 4 + .../eb/notebook-7.4.0-GCCcore-15.2.0.eb | 3 + .../eb/zlib-1.3.1-GCCcore-15.2.0.eb | 3 + .../osv/GHSA-test-curl-old.json | 7 + .../backport_fixtures/osv/GHSA-test-curl.json | 7 + .../osv/GHSA-test-missing.json | 7 + .../osv/GHSA-test-mystery.json | 7 + .../osv/GHSA-test-notebook.json | 7 + .../osv/GHSA-test-unparsed.json | 7 + .../backport_fixtures/osv/GHSA-test-win.json | 7 + .../backport_fixtures/osv/GHSA-test-zlib.json | 7 + tests/backport_fixtures/osv/queries.json | 12 + tests/backport_fixtures/reach.txt | 3 + tests/backport_fixtures/stack.lock.json | 79 +++ tests/backport_list.rs | 245 ++++++++++ tests/component_reach.rs | 3 +- tests/rebuild_set.rs | 7 +- 42 files changed, 1379 insertions(+), 129 deletions(-) create mode 100644 src/backport.rs create mode 100644 tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb create mode 100644 tests/backport_fixtures/osv/GHSA-test-curl-old.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-curl.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-missing.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-mystery.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-notebook.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-unparsed.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-win.json create mode 100644 tests/backport_fixtures/osv/GHSA-test-zlib.json create mode 100644 tests/backport_fixtures/osv/queries.json create mode 100644 tests/backport_fixtures/reach.txt create mode 100644 tests/backport_fixtures/stack.lock.json create mode 100644 tests/backport_list.rs diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index f248f17d..2a844467 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -160,6 +160,7 @@ jobs: cargo test --locked \ --test arch_known_failures \ --test arch_unsupported \ + --test backport_list \ --test baseline_easystack \ --test batch_versions \ --test bot_classify \ diff --git a/.gitignore b/.gitignore index f346a0a8..747abc12 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ docs/source/images/ # Local solve outputs stack.lock.json +!tests/backport_fixtures/stack.lock.json stack.cdx.json build.list stack.diff.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 316dd10b..07b7aabb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,22 @@ All notable changes to this project are documented here. The solve still exits 0. Keep one lock per toolchain family, or pass `--baseline-also-toolchain` for that family. `llvm-compilers-20.1.8` and `llvm-compilers-21.1.8` are the versions `data/eessi-generations.toml` lists for generations 2025b and 2026.1. +- `component-reach` and `rebuild-set` record a reach class: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified` when no built-in name and no `--reachability` file names the package. + `component-reach` uses the built-in names only. + `cross-user` and `network-facing` justify a backport. + Component-reach schema is 2. + Rebuild-set schema is 5. + The component-reach summary prints `reach=`. + `rebuild-set.md` ends with the class of each patched easyconfig. + `--reachability` on `rebuild-set` applies to that map as well as to `backport.md`. +- `stack sbom --backport-list` writes that list for a stack lock. + `--cve` resolves an advisory through Open Source Vulnerabilities (OSV). + `--osv-cache` reads recorded responses and, without `--cve`, every id in `queries.json`. + `--reachability` is the same `name class` file as `rebuild-set`. + Blast radius is the module plus every locked module that depends on it through a runtime or build edge. + `local-only`, `not-reachable`, and `unclassified` are recorded and left off the list. + A `.md` path also writes the sibling `.json`. + `eb_stack_sbom` takes the same optional arguments and reports `listed`, `recorded`, and `stack` when the list is requested. - `stack shard` splits one Jenkins build list into shards that can build at the same time. A line's set is its closure over `--easyconfigs` minus the modules in `--installed-modules-dir` Lmod diff --git a/docs/orgmode/explanation/sbom-blast-radius.org b/docs/orgmode/explanation/sbom-blast-radius.org index 9b09baa5..3efd40d8 100644 --- a/docs/orgmode/explanation/sbom-blast-radius.org +++ b/docs/orgmode/explanation/sbom-blast-radius.org @@ -804,12 +804,40 @@ sed -n '27,29p' /tmp/sbom-demo/rebuild/rebuild-set.md | `git-2.52.0-GCCcore-15.2.0.eb` | `git/2.52.0-GCCcore-15.2.0` | changed | must-rebuild | patched easyconfig | #+end_src +#+begin_src bash +grep 'Reach class' -A 2 /tmp/sbom-demo/rebuild/rebuild-set.md +#+end_src + +#+begin_src text +Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` is not a judgment. + +- `git-2.52.0-GCCcore-15.2.0.eb`: network-facing +#+end_src + The report has one row: the changed git easyconfig, class must-rebuild. -The easystack entries do not depend on that git module, so the walk does -not add a second row. The report also notes each ~from-commit~ pin whose -tree was not passed and says the release tree was used for that pin. -~--check-static~ was not passed, and the report says so. Classes follow -the easyconfig graph only. +The easystack entries do not depend on that git module, so the walk does not add a second row. +The report also notes each ~from-commit~ pin whose tree was not passed and says the release tree was used for that pin. +~--check-static~ was not passed, and the report says so. +Classes follow the easyconfig graph only. git is ~network-facing~, so this patched easyconfig would justify a backport. + +A stack lock has the same classes. +~stack sbom --backport-list~ reads recorded Open Source Vulnerabilities (OSV) responses and the software bill of materials (SBOM) reverse closure. +~cross-user~ and ~network-facing~ are the list. +The fixture lock has cURL (network-facing, blast radius 2 because git depends on it) and notebook (cross-user). zlib is ~local-only~ in the site file, so it is recorded and left off the list. + +#+begin_src bash +eb-stack stack sbom \ + --lock tests/backport_fixtures/stack.lock.json \ + --out /tmp/sbom-demo/backport.cdx.json \ + --backport-list /tmp/sbom-demo/backport.md \ + --osv-cache tests/backport_fixtures/osv \ + --reachability tests/backport_fixtures/reach.txt +#+end_src + +#+begin_src text +sbom=/tmp/sbom-demo/backport.cdx.json components=7 recipes_read=7 recipes_unreadable=0 with_artifact_facts=0 +backport=2 recorded=3 stack=7 out=/tmp/sbom-demo/backport.md +#+end_src * A component named by a CVE @@ -832,7 +860,7 @@ eb-stack component-reach \ #+end_src #+begin_src text -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl #+end_src #+begin_src bash @@ -840,7 +868,7 @@ sed -n '5,16p' /tmp/sbom-demo/component-curl/component-reach.md #+end_src #+begin_src text -2 carriers. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. +2 carriers. Reach: `network-facing`. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. | Easyconfig | Module | Role | Version | Installed | |---|---|---|---|---| @@ -1023,10 +1051,10 @@ eb_easyblock_reach eb_repro_summary #+end_src -That is 50 tools. ~eb_stack_sbom~ writes the CycloneDX file ~stack sbom~ -writes. Its ~tools/list~ description reads: "Emit CycloneDX from an existing -stack lock, reading dependency edges and source checksums from the -easyconfigs the lock names." ~eb_stack_sbom~ reads the Cbc lock: +That is 50 tools. +~eb_stack_sbom~ writes the CycloneDX file ~stack sbom~ writes. +Its ~tools/list~ description reads: "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names. With backport_list, also write the modules whose reach class justifies a backport." +~eb_stack_sbom~ reads the Cbc lock: #+begin_src bash printf '%s\n' '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"eb_stack_sbom","arguments":{"lock":"/tmp/sbom-demo/stack.lock.json","out":"/tmp/sbom-demo/mcp-stack.cdx.json"}}}' \ diff --git a/docs/orgmode/howto/component-reach.org b/docs/orgmode/howto/component-reach.org index df9e3856..d8ca861c 100644 --- a/docs/orgmode/howto/component-reach.org +++ b/docs/orgmode/howto/component-reach.org @@ -25,7 +25,7 @@ eb-stack component-reach \ The command prints: #+begin_example -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach #+end_example cURL 8.17.0 is the module. git 2.52.0 depends on that cURL and is the @@ -50,12 +50,15 @@ eb-stack component-reach \ The command prints: #+begin_example -component=curl version=>=8,<9 generation=2026.1 carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full #+end_example -The changed files are ~cURL-8.17.0-GCCcore-15.2.0.eb~ and -~cargo-c-0.10.23-GCCcore-15.2.0.eb~. The must-rebuild row is -~cURL/8.17.0-GCCcore-15.2.0~, reason ~patched easyconfig~. +The changed files are ~cURL-8.17.0-GCCcore-15.2.0.eb~ and ~cargo-c-0.10.23-GCCcore-15.2.0.eb~. +The must-rebuild row is ~cURL/8.17.0-GCCcore-15.2.0~, reason ~patched easyconfig~. +The summary's ~reach=~ is the class of the component name: ~cross-user~, ~network-facing~, ~local-only~, ~not-reachable~, or ~unclassified~ when the built-in names do not include it. +~component-reach~ reads no site file. +~cross-user~ and ~network-facing~ justify a backport. +cURL is ~network-facing~. ~--easystack~ and ~--installed~ narrow the walk to a deployed prefix. ~--filter-deps~ drops dependency tuples before the closure walk. When diff --git a/docs/orgmode/howto/rebuild-set.org b/docs/orgmode/howto/rebuild-set.org index ec5db2f6..2078740d 100644 --- a/docs/orgmode/howto/rebuild-set.org +++ b/docs/orgmode/howto/rebuild-set.org @@ -29,6 +29,12 @@ The command prints: changed=2 rows=8 must-rebuild=2 review=2 no-rebuild=4 missing=1 easystacks=2 skipped_easyconfigs=0 out=/tmp/rebuild-out #+end_example +~rebuild-set.md~ ends with the reach class of each changed easyconfig. +~cross-user~ and ~network-facing~ justify a backport. +~local-only~ and ~not-reachable~ do not. +~unclassified~ means neither the built-in names nor a ~--reachability~ file names the package. +zlib and ~Absent-1.0.eb~ on this fixture are ~unclassified~. + * Run it on a mounted stack From a clone of ~EESSI/software-layer~, with the 2026.06 stack mounted. diff --git a/docs/orgmode/howto/solve-lock.org b/docs/orgmode/howto/solve-lock.org index b3fecdbb..681cd7c9 100644 --- a/docs/orgmode/howto/solve-lock.org +++ b/docs/orgmode/howto/solve-lock.org @@ -62,6 +62,15 @@ that name is also in the lock. A path that cannot be read contributes nothing for that component. ~--sbom-out~ during ~stack solve~ writes the same document while the candidate graph is still in memory. +~--backport-list~ writes the locked modules whose reach class justifies a backport. +Pass ~--cve~ or ~--osv-cache~. +~--osv-cache~ reads recorded Open Source Vulnerabilities (OSV) responses and does not query the network when ~--cve~ is omitted. +~--reachability~ adds or replaces a class. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +~cross-user~ and ~network-facing~ are the list. +~local-only~, ~not-reachable~, and ~unclassified~ are recorded and left off it. +See [[file:../reference/cli.org][the CLI reference]]. + * Order every pinned build ~stack solve~ picks one version of each name. ~stack order~ keeps every diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index f90b6d63..6f4e425d 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -564,6 +564,21 @@ is the component property ~eb_stack:buildDependsOn~. ~stack solve --sbom-out~ writes that same document, checksums included. The MCP tool ~eb_stack_sbom~ writes it too. An unreadable path contributes no edge. +~--backport-list~ writes a markdown list of locked modules whose reach class justifies a backport. +A path ending in ~.md~ also writes the sibling ~.json~. +~--cve~ is repeatable and resolves that advisory through Open Source Vulnerabilities (OSV). +~--osv-cache~ is a directory of recorded responses: ~queries.json~ maps each request URL to a body file. +Without ~--cve~, every id in that file is resolved, and the command does not query the network. +~--reachability~ is the same ~name class~ file as ~rebuild-set~. +The classes are ~cross-user~, ~network-facing~, ~local-only~, and ~not-reachable~. +A name neither the built-in table nor that file mentions is ~unclassified~. +Only ~cross-user~ and ~network-facing~ are on the list. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +The command writes the CycloneDX file, then prints ~backport=~, ~recorded=~, ~stack=~, and ~out=~. +~--cve~, ~--osv-cache~, and ~--reachability~ need ~--backport-list~. +~--backport-list~ needs ~--cve~ or ~--osv-cache~. +~eb_stack_sbom~ takes the same arguments as ~backport_list~, ~cve~, ~osv_cache~, and ~reachability~. + ~stack solve~ is the multi-root EasyBuild stack solver. Package-profile stack preferences use the TOML schema documented with the package workflow; whole stack solving uses the JSON ~Policy~ schema. The policy file is @@ -1453,10 +1468,12 @@ review. The scan sees dynamic symbols only, so a hidden or inlined static link is a false no-rebuild, and a short name shared with another library is a false must-rebuild. -The output directory contains ~rebuild-set.json~, ~rebuild-set.md~, and -~rebuilds/-eb--cve-patches.yml~. The easystack contains -class ~must-rebuild~ only, including a dependent that ~--check-static~ -promoted. A second run replaces the ~rebuilds/*.yml~ files. +The output directory contains ~rebuild-set.json~, ~rebuild-set.md~, and ~rebuilds/-eb--cve-patches.yml~. +The easystack contains class ~must-rebuild~ only, including a dependent that ~--check-static~ promoted. +A second run replaces the ~rebuilds/*.yml~ files. +~rebuild-set.json~ and ~rebuild-set.md~ record the reach class of each patched easyconfig, including a filename listed under ~missing~. +~--reachability~ replaces or adds names for that map as well as for ~backport.md~. +~cross-user~ and ~network-facing~ justify a backport. * ~component-reach~ @@ -1479,26 +1496,25 @@ that carries a component, then feeds the recipes that contain it into the rebuild closure. See [[file:../howto/component-reach.org][which modules carry a component]]. -Pass ~--component~ and ~--version~, or ~--purl~, or ~--cve~. ~--version~ -is an exact version, ~*~, or a requirement such as ~>=8,<9~. With -~--purl~ it replaces the version in the URL. The purl type selects the -carriers. ~generic~ and ~github~ match the easyconfig name. ~pypi~, -~cargo~, ~npm~, ~golang~, and ~cran~ match an extension or a bundled -component of that language, plus a crate for ~cargo~ and a wheel for -~pypi~. Another type exits 2: the error says that purl type has no -carrier model. ~pkg:cargo/curl~ does not match the cURL recipe. -~--cve~ resolves the id through Open Source Vulnerabilities (OSV), follows GitHub Security Advisory (GHSA), RUSTSEC, PYSEC, and -GO aliases when the Common Vulnerabilities and Exposures (CVE) record has only a git range, and keeps named -version tags such as ~libXpm-3.5.17~. A GIT range is mapped onto -release versions: the affected entry's ~versions~ list when it has one, -and otherwise tags from the upstream repository (~git ls-remote --tags~, -then ~git merge-base --is-ancestor~ on a blobless fetch). That lookup is -cached and is skipped when it cannot reach the network, and the range -then stays under Unresolved ranges. An event that does not parse is -reported there too. A package whose ranges were all unparsed counts as -review. Several affected packages are written as one document with -~kind~ ~cve~. The summary line includes ~changed~, ~must-rebuild~, -~review~, and ~no-rebuild~. +Pass ~--component~ and ~--version~, or ~--purl~, or ~--cve~. +~--version~ is an exact version, ~*~, or a requirement such as ~>=8,<9~. +With ~--purl~ it replaces the version in the URL. +The purl type selects the carriers. +~generic~ and ~github~ match the easyconfig name. +~pypi~, ~cargo~, ~npm~, ~golang~, and ~cran~ match an extension or a bundled component of that language, plus a crate for ~cargo~ and a wheel for ~pypi~. +Another type exits 2: the error says that purl type has no carrier model. +~pkg:cargo/curl~ does not match the cURL recipe. +~--cve~ resolves the id through Open Source Vulnerabilities (OSV). +When the Common Vulnerabilities and Exposures (CVE) record has only a git range, it follows GitHub Security Advisory (GHSA), ~RUSTSEC~, ~PYSEC~, and ~GO~ aliases. +It keeps named version tags such as ~libXpm-3.5.17~. +A ~GIT~ range is mapped onto release versions: the affected entry's ~versions~ list when it has one, and otherwise tags from the upstream repository (~git ls-remote --tags~, then ~git merge-base --is-ancestor~ on a blobless fetch). +That lookup is cached and is skipped when it cannot reach the network, and the range then stays under Unresolved ranges. +An event that does not parse is reported there too. +A package whose ranges were all unparsed counts as review. +An advisory that affects more than one package is written as one document with ~kind~ ~cve~. +The summary line includes ~reach~, ~changed~, ~must-rebuild~, ~review~, and ~no-rebuild~. +~reach~ is the class of the component name: ~cross-user~, ~network-facing~, ~local-only~, ~not-reachable~, or ~unclassified~. +A ~--cve~ summary has no single class. Names match ignoring ASCII letter case, on a whole token. A recipe is in the generation when its toolchain version equals ~--generation~, or when the diff --git a/docs/orgmode/reference/formats.org b/docs/orgmode/reference/formats.org index 3b27f936..9c49c4ac 100644 --- a/docs/orgmode/reference/formats.org +++ b/docs/orgmode/reference/formats.org @@ -534,13 +534,14 @@ filename carried a version, the file is ~unknown~. | Field | Meaning | |- -| ~schema_version~ | ~4~ | +| ~schema_version~ | ~5~ | | ~generated_at~ | RFC 3339 timestamp of the run | | ~installed_source~ | Label of the installed list | | ~prefix~ | Installed prefix passed to ~--prefix~. Omitted when the flag was not passed | | ~symbols_checked~ | ~true~ when ~--check-static~ ran | | ~easystacks~ | Easystack paths that were read, sorted | | ~changed~ | Patched easyconfig filenames, sorted | +| ~reach~ | Reach class of each filename in ~changed~, including missing ones. ~cross-user~ and ~network-facing~ justify a backport. ~local-only~ and ~not-reachable~ do not. ~unclassified~ means neither the built-in names nor ~--reachability~ names the package | | ~missing~ | Changed filenames that are not in the deployed graph | | ~unresolved~ | Requirements no easyconfig satisfies, and easystack entries the robot path does not have | | ~notes~ | A missing easyblock class, a ~from-commit~ whose tree was not given, or an install directory that was not found. Empty when there is nothing to say | @@ -554,8 +555,8 @@ when an easystack filename carried one. On a review row, ~changed~ is the patched filenames joined by ~, ~. After ~--check-static~, a hit keeps only the filenames whose exports were defined. -~rebuild-set.md~ is the same rows as a table, the counts line, the missing -filenames, and each note as ~- note:~. +~rebuild-set.md~ is the same rows as a table, the counts line, the missing filenames, and each note as ~- note:~. +It ends with the reach class of each patched easyconfig. ** ~findings.json~, ~findings.md~, ~findings.csv~ @@ -674,7 +675,7 @@ One run writes ~component-reach.toml~, ~component-reach.json~, and | Field | Meaning | |- -| ~schema_version~ | ~1~ | +| ~schema_version~ | ~2~ | | ~generated_at~ | RFC 3339 timestamp of the run | | ~component~ | Name that was asked for | | ~version_req~ | Exact version, ~*~, or a requirement. Empty when the match is only ~versions~ | @@ -683,6 +684,7 @@ One run writes ~component-reach.toml~, ~component-reach.json~, and | ~purl~ | Package URL, when the query was a purl or a resolved CVE | | ~ecosystem~ | Ecosystem from the purl or the Open Source Vulnerabilities (OSV) record | | ~generation~ | Toolchain generation | +| ~reach~ | Class of the component name: ~cross-user~, ~network-facing~, ~local-only~, ~not-reachable~, or ~unclassified~ | | ~carriers~ | Easyconfigs and modules that carry it | | ~changed~ | Filenames that contain it, fed to the closure | | ~closure_counts~ | Row counts, keys ~no-rebuild~, ~review~, ~must-rebuild~ | @@ -700,15 +702,40 @@ compat-layer and installed-only rows), ~module~, and ~installed~ when a list was passed. A ~--cve~ run writes the same three filenames with a different document. -~kind~ is ~cve~. ~cve~ is the id. ~resolved~ is the packages OSV named -(~ecosystem~, ~name~, ~purl~, ~version_reqs~, ~versions~). ~results~ is -one component-reach document per package. ~notes~ contains alias fetches -that failed. Schema version stays ~1~. +~kind~ is ~cve~. +~cve~ is the id. +~resolved~ is the packages the Open Source Vulnerabilities (OSV) record named (~ecosystem~, ~name~, ~purl~, ~version_reqs~, ~versions~). +~results~ is one component-reach document per package. +~notes~ contains alias fetches that failed. +Schema version stays ~2~. +Each package document carries its own ~reach~. A closure row has the rebuild-set fields ~changed~, ~dependent~, ~dependent_easyconfig~, ~edge_kind~, ~class~, ~reason~, and ~eb_version~ when an easystack filename carried one. With no easystack the synthetic stack is ~generation-.yml~ and ~eb_version~ is absent. -~component-reach.md~ is the carrier table, then the closure table, the -counts line, and each note as ~- note:~. +~component-reach.md~ is the carrier table, then the closure table, the counts line, and each note as ~- note:~. +The carriers line names the reach class. + +* Backport list (~eb-stack stack sbom --backport-list~) + +A path ending in ~.md~ writes that markdown and a sibling ~.json~. + +| Field | Meaning | +|- +| ~schema_version~ | ~1~ | +| ~lock~ | Lock path, as it was passed | +| ~generation~ | Toolchain label of the lock | +| ~advisories~ | Advisory ids that were resolved, sorted | +| ~stack_modules~ | Packages in the lock | +| ~listed~ | Rows whose reach class justifies a backport | +| ~recorded~ | Rows left off the list | +| ~rows~ | One entry per locked module an advisory matched. On-list rows come first | +| ~notes~ | An advisory names a package the lock lacks, a locked version is outside the range, or a range did not parse | + +Each row has ~module~ (~Name/version-toolchain~, plus the versionsuffix when the lock has one), ~name~, ~version~, ~reach~, ~advisory~, ~fixed_in~ (upper bounds from ~<~ clauses; a ~<=~ bound is last-affected and is not a fix), ~blast_radius~, ~blast~ (those module labels, sorted), and ~on_list~. +Blast radius counts the module and every locked module that depends on it through a runtime or build edge. + +The markdown lists each module in a backport section or a recorded section. +The recorded sentence is: these modules stay off the backport list, and a backport is cross-user or network-facing. diff --git a/docs/orgmode/reference/glossary.org b/docs/orgmode/reference/glossary.org index 7f32a2e5..cdad0ab5 100644 --- a/docs/orgmode/reference/glossary.org +++ b/docs/orgmode/reference/glossary.org @@ -43,13 +43,15 @@ Flags and exit status stay in the [[file:cli.org][command-line reference]]. ~stack.lock.json~ for a whole stack. A ~preferred~ pin may fall back inside the solver. A ~locked~ pin may not. -- reachability :: Four checks, not one word. Host reachability is whether - a target's SSH, rsync, or staging path answers; a failure is a - ~transport~ finding. ~component-reach~ lists modules in one generation - that carry a named component. ~easyblock-reach~ lists easyconfigs that - use a changed class. ~stack arch --probe-sources~ reports ~unreachable~ - when no probed architecture family can fetch a source, and - ~arch-missing~ when another family can. +- reachability :: Four checks, not one word. + Host reachability is whether a target's SSH, rsync, or staging path answers; a failure is a ~transport~ finding. + ~component-reach~ lists modules in one generation that carry a named component. + ~easyblock-reach~ lists easyconfigs that use a changed class. + ~stack arch --probe-sources~ reports ~unreachable~ when no probed architecture family can fetch a source, and ~arch-missing~ when another family can. + A security finding's class is ~cross-user~, ~network-facing~, ~local-only~, or ~not-reachable~. + ~component-reach~ and ~rebuild-set~ record that class. + ~unclassified~ means neither the built-in names nor a ~--reachability~ file names the package. + Only ~cross-user~ and ~network-facing~ justify a backport. - Resolvo :: The conflict-driven clause learning (CDCL) Boolean satisfiability (SAT) solver diff --git a/llms-full.txt b/llms-full.txt index c57f1fe0..b4a7afcc 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1813,6 +1813,15 @@ that name is also in the lock. A path that cannot be read contributes nothing for that component. `--sbom-out` during `stack solve` writes the same document while the candidate graph is still in memory. +`--backport-list` writes the locked modules whose reach class justifies a backport. +Pass `--cve` or `--osv-cache`. +`--osv-cache` reads recorded Open Source Vulnerabilities (OSV) responses and does not query the network when `--cve` is omitted. +`--reachability` adds or replaces a class. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +`cross-user` and `network-facing` are the list. +`local-only`, `not-reachable`, and `unclassified` are recorded and left off it. +See [the CLI reference](https://eb-stack.rgoswami.me/../reference/cli.html.md). + # Order every pinned build `stack solve` picks one version of each name. `stack order` keeps every @@ -3067,6 +3076,12 @@ The command prints: changed=2 rows=8 must-rebuild=2 review=2 no-rebuild=4 missing=1 easystacks=2 skipped_easyconfigs=0 out=/tmp/rebuild-out ``` +`rebuild-set.md` ends with the reach class of each changed easyconfig. +`cross-user` and `network-facing` justify a backport. +`local-only` and `not-reachable` do not. +`unclassified` means neither the built-in names nor a `--reachability` file names the package. +zlib and `Absent-1.0.eb` on this fixture are `unclassified`. + # Run it on a mounted stack From a clone of `EESSI/software-layer`, with the 2026.06 stack mounted. @@ -3295,7 +3310,7 @@ eb-stack component-reach \ The command prints: ``` -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach ``` cURL 8.17.0 is the module. git 2.52.0 depends on that cURL and is the @@ -3320,12 +3335,15 @@ eb-stack component-reach \ The command prints: ``` -component=curl version=>=8,<9 generation=2026.1 carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full ``` -The changed files are `cURL-8.17.0-GCCcore-15.2.0.eb` and -`cargo-c-0.10.23-GCCcore-15.2.0.eb`. The must-rebuild row is -`cURL/8.17.0-GCCcore-15.2.0`, reason `patched easyconfig`. +The changed files are `cURL-8.17.0-GCCcore-15.2.0.eb` and `cargo-c-0.10.23-GCCcore-15.2.0.eb`. +The must-rebuild row is `cURL/8.17.0-GCCcore-15.2.0`, reason `patched easyconfig`. +The summary's `reach=` is the class of the component name: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified` when the built-in names do not include it. +`component-reach` reads no site file. +`cross-user` and `network-facing` justify a backport. +cURL is `network-facing`. `--easystack` and `--installed` narrow the walk to a deployed prefix. `--filter-deps` drops dependency tuples before the closure walk. When @@ -5179,13 +5197,15 @@ Flags and exit status stay in the [command-line reference](https://eb-stack.rgos `stack.lock.json` for a whole stack. A `preferred` pin may fall back inside the solver. A `locked` pin may not. -- reachability :: Four checks, not one word. Host reachability is whether - a target's SSH, rsync, or staging path answers; a failure is a - `transport` finding. `component-reach` lists modules in one generation - that carry a named component. `easyblock-reach` lists easyconfigs that - use a changed class. `stack arch --probe-sources` reports `unreachable` - when no probed architecture family can fetch a source, and - `arch-missing` when another family can. +- reachability :: Four checks, not one word. + Host reachability is whether a target's SSH, rsync, or staging path answers; a failure is a `transport` finding. + `component-reach` lists modules in one generation that carry a named component. + `easyblock-reach` lists easyconfigs that use a changed class. + `stack arch --probe-sources` reports `unreachable` when no probed architecture family can fetch a source, and `arch-missing` when another family can. + A security finding's class is `cross-user`, `network-facing`, `local-only`, or `not-reachable`. + `component-reach` and `rebuild-set` record that class. + `unclassified` means neither the built-in names nor a `--reachability` file names the package. + Only `cross-user` and `network-facing` justify a backport. - Resolvo :: The conflict-driven clause learning (CDCL) Boolean satisfiability (SAT) solver @@ -5813,6 +5833,21 @@ is the component property `eb_stack:buildDependsOn`. `stack solve --sbom-out` writes that same document, checksums included. The MCP tool `eb_stack_sbom` writes it too. An unreadable path contributes no edge. +`--backport-list` writes a markdown list of locked modules whose reach class justifies a backport. +A path ending in `.md` also writes the sibling `.json`. +`--cve` is repeatable and resolves that advisory through Open Source Vulnerabilities (OSV). +`--osv-cache` is a directory of recorded responses: `queries.json` maps each request URL to a body file. +Without `--cve`, every id in that file is resolved, and the command does not query the network. +`--reachability` is the same `name class` file as `rebuild-set`. +The classes are `cross-user`, `network-facing`, `local-only`, and `not-reachable`. +A name neither the built-in table nor that file mentions is `unclassified`. +Only `cross-user` and `network-facing` are on the list. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +The command writes the CycloneDX file, then prints `backport=`, `recorded=`, `stack=`, and `out=`. +`--cve`, `--osv-cache`, and `--reachability` need `--backport-list`. +`--backport-list` needs `--cve` or `--osv-cache`. +`eb_stack_sbom` takes the same arguments as `backport_list`, `cve`, `osv_cache`, and `reachability`. + `stack solve` is the multi-root EasyBuild stack solver. Package-profile stack preferences use the TOML schema documented with the package workflow; whole stack solving uses the JSON `Policy` schema. The policy file is @@ -6702,10 +6737,12 @@ review. The scan sees dynamic symbols only, so a hidden or inlined static link is a false no-rebuild, and a short name shared with another library is a false must-rebuild. -The output directory contains `rebuild-set.json`, `rebuild-set.md`, and -`rebuilds/-eb--cve-patches.yml`. The easystack contains -class `must-rebuild` only, including a dependent that `--check-static` -promoted. A second run replaces the `rebuilds/*.yml` files. +The output directory contains `rebuild-set.json`, `rebuild-set.md`, and `rebuilds/-eb--cve-patches.yml`. +The easystack contains class `must-rebuild` only, including a dependent that `--check-static` promoted. +A second run replaces the `rebuilds/*.yml` files. +`rebuild-set.json` and `rebuild-set.md` record the reach class of each patched easyconfig, including a filename listed under `missing`. +`--reachability` replaces or adds names for that map as well as for `backport.md`. +`cross-user` and `network-facing` justify a backport. # `component-reach` @@ -6728,26 +6765,25 @@ that carries a component, then feeds the recipes that contain it into the rebuild closure. See [which modules carry a component](https://eb-stack.rgoswami.me/../howto/component-reach.html.md). -Pass `--component` and `--version`, or `--purl`, or `--cve`. `--version` -is an exact version, `*`, or a requirement such as `>=8,<9`. With -`--purl` it replaces the version in the URL. The purl type selects the -carriers. `generic` and `github` match the easyconfig name. `pypi`, -`cargo`, `npm`, `golang`, and `cran` match an extension or a bundled -component of that language, plus a crate for `cargo` and a wheel for -`pypi`. Another type exits 2: the error says that purl type has no -carrier model. `pkg:cargo/curl` does not match the cURL recipe. -`--cve` resolves the id through Open Source Vulnerabilities (OSV), follows GitHub Security Advisory (GHSA), RUSTSEC, PYSEC, and -GO aliases when the Common Vulnerabilities and Exposures (CVE) record has only a git range, and keeps named -version tags such as `libXpm-3.5.17`. A GIT range is mapped onto -release versions: the affected entry's `versions` list when it has one, -and otherwise tags from the upstream repository (`git ls-remote --tags`, -then `git merge-base --is-ancestor` on a blobless fetch). That lookup is -cached and is skipped when it cannot reach the network, and the range -then stays under Unresolved ranges. An event that does not parse is -reported there too. A package whose ranges were all unparsed counts as -review. Several affected packages are written as one document with -`kind` `cve`. The summary line includes `changed`, `must-rebuild`, -`review`, and `no-rebuild`. +Pass `--component` and `--version`, or `--purl`, or `--cve`. +`--version` is an exact version, `*`, or a requirement such as `>=8,<9`. +With `--purl` it replaces the version in the URL. +The purl type selects the carriers. +`generic` and `github` match the easyconfig name. +`pypi`, `cargo`, `npm`, `golang`, and `cran` match an extension or a bundled component of that language, plus a crate for `cargo` and a wheel for `pypi`. +Another type exits 2: the error says that purl type has no carrier model. +`pkg:cargo/curl` does not match the cURL recipe. +`--cve` resolves the id through Open Source Vulnerabilities (OSV). +When the Common Vulnerabilities and Exposures (CVE) record has only a git range, it follows GitHub Security Advisory (GHSA), `RUSTSEC`, `PYSEC`, and `GO` aliases. +It keeps named version tags such as `libXpm-3.5.17`. +A `GIT` range is mapped onto release versions: the affected entry's `versions` list when it has one, and otherwise tags from the upstream repository (`git ls-remote --tags`, then `git merge-base --is-ancestor` on a blobless fetch). +That lookup is cached and is skipped when it cannot reach the network, and the range then stays under Unresolved ranges. +An event that does not parse is reported there too. +A package whose ranges were all unparsed counts as review. +An advisory that affects more than one package is written as one document with `kind` `cve`. +The summary line includes `reach`, `changed`, `must-rebuild`, `review`, and `no-rebuild`. +`reach` is the class of the component name: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified`. +A `--cve` summary has no single class. Names match ignoring ASCII letter case, on a whole token. A recipe is in the generation when its toolchain version equals `--generation`, or when the @@ -8620,13 +8656,14 @@ filename carried a version, the file is `unknown`. | Field | Meaning | | --- | --- | -| `schema_version` | `4` | +| `schema_version` | `5` | | `generated_at` | RFC 3339 timestamp of the run | | `installed_source` | Label of the installed list | | `prefix` | Installed prefix passed to `--prefix`. Omitted when the flag was not passed | | `symbols_checked` | `true` when `--check-static` ran | | `easystacks` | Easystack paths that were read, sorted | | `changed` | Patched easyconfig filenames, sorted | +| `reach` | Reach class of each filename in `changed`, including missing ones. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` means neither the built-in names nor `--reachability` names the package | | `missing` | Changed filenames that are not in the deployed graph | | `unresolved` | Requirements no easyconfig satisfies, and easystack entries the robot path does not have | | `notes` | A missing easyblock class, a `from-commit` whose tree was not given, or an install directory that was not found. Empty when there is nothing to say | @@ -8640,8 +8677,8 @@ when an easystack filename carried one. On a review row, `changed` is the patched filenames joined by `, `. After `--check-static`, a hit keeps only the filenames whose exports were defined. -`rebuild-set.md` is the same rows as a table, the counts line, the missing -filenames, and each note as `- note:`. +`rebuild-set.md` is the same rows as a table, the counts line, the missing filenames, and each note as `- note:`. +It ends with the reach class of each patched easyconfig. ## `findings.json`, `findings.md`, `findings.csv` @@ -8760,7 +8797,7 @@ One run writes `component-reach.toml`, `component-reach.json`, and | Field | Meaning | | --- | --- | -| `schema_version` | `1` | +| `schema_version` | `2` | | `generated_at` | RFC 3339 timestamp of the run | | `component` | Name that was asked for | | `version_req` | Exact version, `*`, or a requirement. Empty when the match is only `versions` | @@ -8769,6 +8806,7 @@ One run writes `component-reach.toml`, `component-reach.json`, and | `purl` | Package URL, when the query was a purl or a resolved CVE | | `ecosystem` | Ecosystem from the purl or the Open Source Vulnerabilities (OSV) record | | `generation` | Toolchain generation | +| `reach` | Class of the component name: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified` | | `carriers` | Easyconfigs and modules that carry it | | `changed` | Filenames that contain it, fed to the closure | | `closure_counts` | Row counts, keys `no-rebuild`, `review`, `must-rebuild` | @@ -8786,18 +8824,43 @@ compat-layer and installed-only rows), `module`, and `installed` when a list was passed. A `--cve` run writes the same three filenames with a different document. -`kind` is `cve`. `cve` is the id. `resolved` is the packages OSV named -(`ecosystem`, `name`, `purl`, `version_reqs`, `versions`). `results` is -one component-reach document per package. `notes` contains alias fetches -that failed. Schema version stays `1`. +`kind` is `cve`. +`cve` is the id. +`resolved` is the packages the Open Source Vulnerabilities (OSV) record named (`ecosystem`, `name`, `purl`, `version_reqs`, `versions`). +`results` is one component-reach document per package. +`notes` contains alias fetches that failed. +Schema version stays `2`. +Each package document carries its own `reach`. A closure row has the rebuild-set fields `changed`, `dependent`, `dependent_easyconfig`, `edge_kind`, `class`, `reason`, and `eb_version` when an easystack filename carried one. With no easystack the synthetic stack is `generation-.yml` and `eb_version` is absent. -`component-reach.md` is the carrier table, then the closure table, the -counts line, and each note as `- note:`. +`component-reach.md` is the carrier table, then the closure table, the counts line, and each note as `- note:`. +The carriers line names the reach class. + +# Backport list (`eb-stack stack sbom --backport-list`) + +A path ending in `.md` writes that markdown and a sibling `.json`. + +| Field | Meaning | +| --- | --- | +| `schema_version` | `1` | +| `lock` | Lock path, as it was passed | +| `generation` | Toolchain label of the lock | +| `advisories` | Advisory ids that were resolved, sorted | +| `stack_modules` | Packages in the lock | +| `listed` | Rows whose reach class justifies a backport | +| `recorded` | Rows left off the list | +| `rows` | One entry per locked module an advisory matched. On-list rows come first | +| `notes` | An advisory names a package the lock lacks, a locked version is outside the range, or a range did not parse | + +Each row has `module` (`Name/version-toolchain`, plus the versionsuffix when the lock has one), `name`, `version`, `reach`, `advisory`, `fixed_in` (upper bounds from `<` clauses; a `<=` bound is last-affected and is not a fix), `blast_radius`, `blast` (those module labels, sorted), and `on_list`. +Blast radius counts the module and every locked module that depends on it through a runtime or build edge. + +The markdown lists each module in a backport section or a recorded section. +The recorded sentence is: these modules stay off the backport list, and a backport is cross-user or network-facing. # From recipe to verified package @@ -10800,12 +10863,40 @@ sed -n '27,29p' /tmp/sbom-demo/rebuild/rebuild-set.md | `git-2.52.0-GCCcore-15.2.0.eb` | `git/2.52.0-GCCcore-15.2.0` | changed | must-rebuild | patched easyconfig | ``` +```bash +grep 'Reach class' -A 2 /tmp/sbom-demo/rebuild/rebuild-set.md +``` + +```text +Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` is not a judgment. + +- `git-2.52.0-GCCcore-15.2.0.eb`: network-facing +``` + The report has one row: the changed git easyconfig, class must-rebuild. -The easystack entries do not depend on that git module, so the walk does -not add a second row. The report also notes each `from-commit` pin whose -tree was not passed and says the release tree was used for that pin. -`--check-static` was not passed, and the report says so. Classes follow -the easyconfig graph only. +The easystack entries do not depend on that git module, so the walk does not add a second row. +The report also notes each `from-commit` pin whose tree was not passed and says the release tree was used for that pin. +`--check-static` was not passed, and the report says so. +Classes follow the easyconfig graph only. git is `network-facing`, so this patched easyconfig would justify a backport. + +A stack lock has the same classes. +`stack sbom --backport-list` reads recorded Open Source Vulnerabilities (OSV) responses and the software bill of materials (SBOM) reverse closure. +`cross-user` and `network-facing` are the list. +The fixture lock has cURL (network-facing, blast radius 2 because git depends on it) and notebook (cross-user). zlib is `local-only` in the site file, so it is recorded and left off the list. + +```bash +eb-stack stack sbom \ + --lock tests/backport_fixtures/stack.lock.json \ + --out /tmp/sbom-demo/backport.cdx.json \ + --backport-list /tmp/sbom-demo/backport.md \ + --osv-cache tests/backport_fixtures/osv \ + --reachability tests/backport_fixtures/reach.txt +``` + +```text +sbom=/tmp/sbom-demo/backport.cdx.json components=7 recipes_read=7 recipes_unreadable=0 with_artifact_facts=0 +backport=2 recorded=3 stack=7 out=/tmp/sbom-demo/backport.md +``` # A component named by a CVE @@ -10828,7 +10919,7 @@ eb-stack component-reach \ ``` ```text -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl ``` ```bash @@ -10836,7 +10927,7 @@ sed -n '5,16p' /tmp/sbom-demo/component-curl/component-reach.md ``` ```text -2 carriers. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. +2 carriers. Reach: `network-facing`. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. | Easyconfig | Module | Role | Version | Installed | |---|---|---|---|---| @@ -11019,10 +11110,10 @@ eb_easyblock_reach eb_repro_summary ``` -That is 50 tools. `eb_stack_sbom` writes the CycloneDX file `stack sbom` -writes. Its `tools/list` description reads: "Emit CycloneDX from an existing -stack lock, reading dependency edges and source checksums from the -easyconfigs the lock names." `eb_stack_sbom` reads the Cbc lock: +That is 50 tools. +`eb_stack_sbom` writes the CycloneDX file `stack sbom` writes. +Its `tools/list` description reads: "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names. With backport_list, also write the modules whose reach class justifies a backport." +`eb_stack_sbom` reads the Cbc lock: ```bash printf '%s\n' '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"eb_stack_sbom","arguments":{"lock":"/tmp/sbom-demo/stack.lock.json","out":"/tmp/sbom-demo/mcp-stack.cdx.json"}}}' \ diff --git a/skills/mcp/SKILL.md b/skills/mcp/SKILL.md index 6e50fbda..57829367 100644 --- a/skills/mcp/SKILL.md +++ b/skills/mcp/SKILL.md @@ -70,7 +70,15 @@ proc.wait(timeout=5) PY ``` -`tools/list` returns the catalog. This page does not keep a count. Read the names from that response. The names that mirror the commands above are `eb_package_inspect`, `eb_package_plan`, `eb_package_bump`, `eb_stack_sbom`, and `eb_pr_overlap`. The input schema requires `toolchain` as `name/version` for inspect, plan, and bump. Inspect and plan also accept `toolchain_version` alone and default the name to `foss`. Bump does not: it requires `toolchain`, or both `toolchain_name` and `toolchain_version`. `eb_stack_sbom` takes `lock` and optional `out`. `eb_pr_overlap` takes `pr` as an array of paths and `easyconfigs` as an array of directories. +`tools/list` returns the catalog. +This page does not keep a count. +Read the names from that response. +The names that mirror the commands above are `eb_package_inspect`, `eb_package_plan`, `eb_package_bump`, `eb_stack_sbom`, and `eb_pr_overlap`. +Each package tool's input schema requires `toolchain` as `name/version`. +Inspect and plan also accept `toolchain_version` alone and default the name to `foss`. +Bump does not: it requires `toolchain`, or both `toolchain_name` and `toolchain_version`. +`eb_stack_sbom` takes `lock` and optional `out`, `backport_list`, `cve`, `osv_cache`, and `reachability`. +`eb_pr_overlap` takes `pr` as an array of paths and `easyconfigs` as an array of directories. `eb_build_test` runs only when `EB_STACK_ALLOW_BUILD=1`. diff --git a/src/backport.rs b/src/backport.rs new file mode 100644 index 00000000..cf6f2202 --- /dev/null +++ b/src/backport.rs @@ -0,0 +1,459 @@ +//! Backport list for one stack lock. +//! +//! A locked module is on the list when an advisory matches its version and +//! its reach class is `cross-user` or `network-facing`. `local-only`, +//! `not-reachable`, and `unclassified` are recorded and left off the list. +//! Blast radius is the SBOM closure: the module and every locked module that +//! depends on it through a runtime or build edge. + +use crate::cli_parse::usage; +use crate::domain::{LockPackage, StackLock}; +use crate::findings::{ + justifies_backport, reach_key, ReachTable, REACH_CROSS_USER, REACH_LOCAL, REACH_NETWORK, + REACH_NOT_REACHABLE, +}; +use crate::osv::{load_recorded_osv, resolve_vulnerability, ResolvedComponent}; +use crate::registry::{RegistryClient, UreqClient}; +use crate::sbom::{closure_edges_for_lock, package_identity}; +use crate::version::matches_req; +use anyhow::{anyhow, bail, Context, Result}; +use serde::Serialize; +use std::collections::{BTreeSet, HashMap, VecDeque}; +use std::path::Path; + +/// Schema of [`BackportDocument`]. +pub const BACKPORT_LIST_SCHEMA_VERSION: u32 = 1; + +/// What `stack sbom --backport-list` reads. +pub struct BackportRequest<'a> { + /// Stack lock. + pub lock: &'a StackLock, + /// Path the lock was read from. Relative recipe paths resolve beside it. + pub lock_path: &'a Path, + /// Advisory ids. Empty means every id in [`Self::osv_cache`]. + pub cves: &'a [String], + /// Recorded OSV responses. Absent means a live fetch of [`Self::cves`]. + pub osv_cache: Option<&'a Path>, + /// Site file of `name class` lines. Absent uses the built-in names. + pub reachability: Option<&'a Path>, +} + +/// One locked module an advisory matched. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackportRow { + /// `Name/version-toolchain`, plus the versionsuffix when the lock has one. + pub module: String, + /// Package name. + pub name: String, + /// Locked version. + pub version: String, + /// Reach class. + pub reach: String, + /// Advisory id. + pub advisory: String, + /// Upper bounds from a less-than clause. A less-than-or-equal bound is last-affected, not a fix. + pub fixed_in: Vec, + /// The module plus every locked module that depends on it. + pub blast_radius: usize, + /// Those module labels, sorted. + pub blast: Vec, + /// `true` when [`justifies_backport`] is true for [`Self::reach`]. + pub on_list: bool, +} + +/// The backport list for one lock. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackportDocument { + /// [`BACKPORT_LIST_SCHEMA_VERSION`]. + pub schema_version: u32, + /// Lock path, as it was passed. + pub lock: String, + /// Toolchain label of the lock. + pub generation: String, + /// Advisory ids that were resolved, sorted. + pub advisories: Vec, + /// Packages in the lock. + pub stack_modules: usize, + /// Rows with [`BackportRow::on_list`]. + pub listed: usize, + /// Rows left off the list. + pub recorded: usize, + /// Matching modules. On-list rows come first. + pub rows: Vec, + /// Advisories that name a package the lock lacks, a version outside the + /// range, or a range that did not parse. + pub notes: Vec, +} + +/// Resolve the advisories and classify every matching locked module. +pub fn backport_list(request: &BackportRequest<'_>) -> Result { + let ids = advisory_ids(request.cves, request.osv_cache)?; + let resolutions = resolve_ids(&ids, request.osv_cache)?; + let reach = load_reach(request.reachability)?; + let blasts = blast_sets(request.lock, request.lock_path); + let mut rows = Vec::new(); + let mut notes = BTreeSet::new(); + for resolution in &resolutions { + for component in &resolution.components { + classify_component( + request.lock, + &resolution.id, + component, + &reach, + &blasts, + &mut rows, + &mut notes, + ); + } + for item in &resolution.unresolved { + notes.insert(format!("{}: {item}", resolution.id)); + } + } + rows.sort_by(|left, right| { + right + .on_list + .cmp(&left.on_list) + .then(rank(&left.reach).cmp(&rank(&right.reach))) + .then(right.blast_radius.cmp(&left.blast_radius)) + .then(left.module.cmp(&right.module)) + .then(left.advisory.cmp(&right.advisory)) + }); + let listed = rows.iter().filter(|row| row.on_list).count(); + Ok(BackportDocument { + schema_version: BACKPORT_LIST_SCHEMA_VERSION, + lock: request.lock_path.display().to_string(), + generation: generation_of(request.lock), + advisories: ids, + stack_modules: request.lock.packages.len(), + listed, + recorded: rows.len() - listed, + rows, + notes: notes.into_iter().collect(), + }) +} + +/// Markdown for [`BackportDocument`]. +pub fn render_markdown(document: &BackportDocument) -> String { + let mut out = String::from("# Backport list\n\n"); + out.push_str(&format!( + "Lock `{}`. Generation `{}`. {} modules.\n\n", + document.lock, document.generation, document.stack_modules + )); + out.push_str( + "A module is on this list when its reach class is `cross-user` or \ + `network-facing`. Those two classes justify a backport. Blast radius \ + is the SBOM closure: the module and every locked module that depends \ + on it through a runtime or build edge.\n\n", + ); + if !document.advisories.is_empty() { + let ids = document + .advisories + .iter() + .map(|id| format!("`{id}`")) + .collect::>() + .join(", "); + out.push_str(&format!("Advisories: {ids}.\n\n")); + } + out.push_str("## Backport\n\n"); + write_rows( + &mut out, + document.rows.iter().filter(|row| row.on_list), + "No module on this lock is `cross-user` or `network-facing`.\n", + ); + out.push_str("\n## Recorded\n\n"); + out.push_str( + "These modules stay off the backport list. A backport is cross-user or network-facing.\n\n", + ); + write_rows( + &mut out, + document.rows.iter().filter(|row| !row.on_list), + "No module was recorded off the list.\n", + ); + if !document.notes.is_empty() { + out.push_str("\n## Notes\n\n"); + for note in &document.notes { + out.push_str(&format!("- {note}\n")); + } + } + out +} + +/// Write the markdown. A path ending in `.md` also gets a sibling `.json`. +pub fn write_backport(path: &Path, document: &BackportDocument) -> Result<()> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create {}", parent.display()))?; + } + } + std::fs::write(path, render_markdown(document)) + .with_context(|| format!("write {}", path.display()))?; + if path.extension().and_then(|ext| ext.to_str()) == Some("md") { + crate::write_json_pretty(&path.with_extension("json"), document)?; + } + Ok(()) +} + +fn write_rows<'a>(out: &mut String, rows: impl Iterator, empty: &str) { + let rows: Vec<&BackportRow> = rows.collect(); + if rows.is_empty() { + out.push_str(empty); + return; + } + out.push_str("| Module | Reach | Advisory | Fixed in | Blast radius |\n"); + out.push_str("|---|---|---|---|---|\n"); + for row in rows { + let fixed = if row.fixed_in.is_empty() { + "-".to_string() + } else { + row.fixed_in.join(", ") + }; + out.push_str(&format!( + "| `{}` | {} | {} | {} | {} |\n", + row.module, row.reach, row.advisory, fixed, row.blast_radius + )); + } +} + +fn classify_component( + lock: &StackLock, + id: &str, + component: &ResolvedComponent, + reach: &ReachTable, + blasts: &HashMap>, + rows: &mut Vec, + notes: &mut BTreeSet, +) { + let named: Vec<&LockPackage> = lock + .packages + .iter() + .filter(|package| reach_key(&package.name) == reach_key(&component.name)) + .collect(); + if named.is_empty() { + notes.insert(format!( + "{id} names {}, which this lock does not have", + component.name + )); + return; + } + if component.version_reqs.is_empty() && component.versions.is_empty() { + notes.insert(format!( + "{id} names {}, but its ranges were not parsed", + component.name + )); + return; + } + let hits: Vec<&LockPackage> = named + .iter() + .copied() + .filter(|package| version_matches(package, component)) + .collect(); + if hits.is_empty() { + let present = named + .iter() + .map(|package| module_label(package)) + .collect::>() + .join(", "); + let verb = if named.len() == 1 { "is" } else { "are" }; + notes.insert(format!( + "{present} {verb} not in range for {id} ({})", + range_detail(component) + )); + return; + } + let fixed_in = fixed_bounds(&component.version_reqs); + for package in hits { + let identity = package_identity(package); + let blast = blasts.get(&identity).cloned().unwrap_or_default(); + let class = reach.class_of(&package.name, &[&component.name, &package.name]); + rows.push(BackportRow { + module: module_label(package), + name: package.name.clone(), + version: package.version.clone(), + reach: class.clone(), + advisory: id.to_string(), + fixed_in: fixed_in.clone(), + blast_radius: blast.len().max(1), + blast: blast.into_iter().collect(), + on_list: justifies_backport(&class), + }); + } +} + +fn version_matches(package: &LockPackage, component: &ResolvedComponent) -> bool { + component + .versions + .iter() + .any(|version| version == &package.version) + || component + .version_reqs + .iter() + .any(|req| matches_req(&package.version, req)) +} + +fn range_detail(component: &ResolvedComponent) -> String { + let mut parts = component.version_reqs.clone(); + for version in &component.versions { + parts.push(format!("version {version}")); + } + parts.join(" or ") +} + +/// Upper bounds introduced by a less-than clause. Less-than-or-equal is last-affected, not a fix. +fn fixed_bounds(reqs: &[String]) -> Vec { + let mut out = Vec::new(); + for req in reqs { + for clause in req.split(',') { + let Some(bound) = clause.trim().strip_prefix('<') else { + continue; + }; + if bound.is_empty() || bound.starts_with('=') { + continue; + } + out.push(bound.to_string()); + } + } + out.sort(); + out.dedup(); + out +} + +fn rank(class: &str) -> u8 { + match class { + REACH_CROSS_USER => 0, + REACH_NETWORK => 1, + REACH_LOCAL => 2, + REACH_NOT_REACHABLE => 3, + _ => 4, + } +} + +fn generation_of(lock: &StackLock) -> String { + lock.generation_label + .clone() + .filter(|label| !label.is_empty()) + .unwrap_or_else(|| lock.toolchain.label()) +} + +fn module_label(package: &LockPackage) -> String { + let mut label = format!( + "{}/{}-{}", + package.name, + package.version, + package.toolchain.label() + ); + if let Some(suffix) = package + .versionsuffix + .as_deref() + .filter(|suffix| !suffix.is_empty()) + { + label.push_str(suffix); + } + label +} + +fn advisory_ids(cves: &[String], cache: Option<&Path>) -> Result> { + if !cves.is_empty() { + let mut ids = cves.to_vec(); + ids.sort(); + ids.dedup(); + return Ok(ids); + } + let Some(dir) = cache else { + bail!("--backport-list needs --cve or --osv-cache"); + }; + let manifest_path = dir.join("queries.json"); + let text = std::fs::read_to_string(&manifest_path) + .with_context(|| format!("read {}", manifest_path.display()))?; + let manifest: serde_json::Value = serde_json::from_str(&text) + .with_context(|| format!("parse {}", manifest_path.display()))?; + let queries = manifest + .get("queries") + .and_then(|value| value.as_array()) + .ok_or_else(|| anyhow!("{} has no queries", manifest_path.display()))?; + let mut ids = Vec::new(); + for query in queries { + let url = query + .get("url") + .and_then(|value| value.as_str()) + .ok_or_else(|| anyhow!("{} query has no url", manifest_path.display()))?; + let id = url + .rsplit('/') + .next() + .filter(|id| !id.is_empty()) + .ok_or_else(|| anyhow!("{url} has no advisory id"))?; + ids.push(id.to_string()); + } + if ids.is_empty() { + bail!("{} names no advisory", manifest_path.display()); + } + ids.sort(); + ids.dedup(); + Ok(ids) +} + +fn resolve_ids(ids: &[String], cache: Option<&Path>) -> Result> { + let recorded = match cache { + Some(dir) => Some(load_recorded_osv(dir).map_err(|error| anyhow!(error))?), + None => None, + }; + let live = UreqClient; + let client: &dyn RegistryClient = match &recorded { + Some(map) => map, + None => &live, + }; + let mut out = Vec::new(); + for id in ids { + out.push(resolve_vulnerability(id, client).map_err(|error| anyhow!(error))?); + } + Ok(out) +} + +fn load_reach(path: Option<&Path>) -> Result { + let Some(path) = path else { + return Ok(ReachTable::builtin()); + }; + let text = std::fs::read_to_string(path) + .with_context(|| format!("--reachability {}: not readable", path.display()))?; + ReachTable::builtin() + .overlay(&text, path.display().to_string()) + .map_err(|error| usage(format!("--reachability {}: {error}", path.display()))) +} + +fn blast_sets(lock: &StackLock, lock_path: &Path) -> HashMap> { + let edges = closure_edges_for_lock(lock, lock_path); + let mut reverse: HashMap<&str, Vec<&str>> = HashMap::new(); + for (from, deps) in &edges { + for dep in deps { + reverse.entry(dep.as_str()).or_default().push(from.as_str()); + } + } + let labels: HashMap = lock + .packages + .iter() + .map(|package| (package_identity(package), module_label(package))) + .collect(); + let mut out = HashMap::new(); + for package in &lock.packages { + let root = package_identity(package); + let mut seen = BTreeSet::new(); + let mut queue = VecDeque::new(); + queue.push_back(root.as_str()); + seen.insert(root.clone()); + while let Some(node) = queue.pop_front() { + if let Some(next) = reverse.get(node) { + for dependent in next { + if seen.insert((*dependent).to_string()) { + queue.push_back(dependent); + } + } + } + } + let modules = seen + .into_iter() + .map(|id| labels.get(&id).cloned().unwrap_or(id)) + .collect(); + out.insert(root, modules); + } + out +} diff --git a/src/component_reach.rs b/src/component_reach.rs index 49239bae..05e8dd78 100644 --- a/src/component_reach.rs +++ b/src/component_reach.rs @@ -15,7 +15,11 @@ use serde::Serialize; use std::collections::BTreeSet; /// Schema of [`ReachDocument`]. -pub const COMPONENT_REACH_SCHEMA_VERSION: u32 = 1; +/// +/// Version 2 adds `reach`: `cross-user`, `network-facing`, `local-only`, +/// `not-reachable`, or `unclassified` when the built-in names do not include +/// the component. `component-reach` reads no site file. +pub const COMPONENT_REACH_SCHEMA_VERSION: u32 = 2; /// One run. `version_req` is an exact version, `*`, or a requirement. pub struct ReachQuery<'a> { @@ -93,6 +97,12 @@ pub struct ReachDocument { pub ecosystem: Option, /// Toolchain generation. pub generation: String, + /// Reach class of the component name. + /// + /// `cross-user` and `network-facing` justify a backport. `local-only` + /// and `not-reachable` do not. `unclassified` means the built-in names + /// do not include this component. + pub reach: String, /// Who carries it. pub carriers: Vec, /// Recipes that contain the component, fed to the closure. @@ -201,6 +211,8 @@ pub fn component_reach(query: &ReachQuery<'_>) -> ReachDocument { closure_counts.review = 1; } notes.extend(extra); + let reach = + crate::findings::ReachTable::builtin().class_of(query.component, &[query.component]); let mut version_reqs = Vec::new(); if !query.version_req.is_empty() { version_reqs.push(query.version_req.to_string()); @@ -220,6 +232,7 @@ pub fn component_reach(query: &ReachQuery<'_>) -> ReachDocument { purl: query.purl.map(str::to_string), ecosystem: query.ecosystem.map(str::to_string), generation: query.generation.to_string(), + reach, carriers, changed, closure_counts, @@ -242,11 +255,12 @@ pub fn render_markdown(document: &ReachDocument) -> String { "*".to_string() }; out.push_str(&format!( - "Computed {} for `{}` `{requirement}` in generation `{}`.\n\n{} carriers. Closure: {} must-rebuild, {} review, {} no-rebuild.\n\n", + "Computed {} for `{}` `{requirement}` in generation `{}`.\n\n{} carriers. Reach: `{}`. Closure: {} must-rebuild, {} review, {} no-rebuild.\n\n", document.generated_at, document.component, document.generation, document.carriers.len(), + document.reach, document.closure_counts.must_rebuild, document.closure_counts.review, document.closure_counts.no_rebuild @@ -1170,7 +1184,11 @@ mod tests { .any(|row| row.dependent.starts_with("git/") && row.class == "no-rebuild")); let rendered = to_toml(&curl).unwrap(); assert!(rendered.starts_with(HEADER)); + assert_eq!(curl.reach, "network-facing"); + assert!(render_markdown(&curl).contains("Reach: `network-facing`")); assert!(render_markdown(&curl).contains("no-rebuild")); + assert_eq!(run("jupyterlab").reach, "cross-user"); + assert_eq!(run("zlib").reach, "unclassified"); assert!(run("urllib3") .carriers .iter() diff --git a/src/findings.rs b/src/findings.rs index 29461caa..f157645e 100644 --- a/src/findings.rs +++ b/src/findings.rs @@ -233,6 +233,28 @@ impl ReachTable { } } +/// Reach class of each easyconfig filename. +/// +/// The name is the toolchain split, then the same fallback as +/// [`crate::gap::software_name`]. The class is [`ReachTable::class_of`] on +/// that name. Filenames stay in the order `files` gives them only as map +/// keys; a [`std::collections::BTreeMap`] sorts them. +pub fn reach_for_easyconfigs(table: &ReachTable, files: &[String]) -> BTreeMap { + files + .iter() + .map(|file| { + let owned = crate::gap::software_name(file).unwrap_or_else(|| { + file.rsplit(['/', '\\']) + .next() + .unwrap_or(file.as_str()) + .to_string() + }); + let class = table.class_of(&owned, &[owned.as_str()]); + (file.clone(), class) + }) + .collect() +} + fn parse_reachability(text: &str) -> Result, ReachabilityError> { let mut site = BTreeMap::new(); for (index, raw) in text.lines().enumerate() { @@ -1778,6 +1800,7 @@ Medium,\"GHSA-a,b\",\"py \"\"x\"\"\",2.0,2.1,not-fixed,PyApp/1.0-GCCcore-15.2.0\ must_rebuild: 0, }, rows, + reach: BTreeMap::new(), } } diff --git a/src/fix_cost.rs b/src/fix_cost.rs index b7c920b0..4e2719fc 100644 --- a/src/fix_cost.rs +++ b/src/fix_cost.rs @@ -1457,6 +1457,7 @@ mod tests { review: 0, must_rebuild: 0, }, + reach: std::collections::BTreeMap::new(), rows, } } diff --git a/src/lib.rs b/src/lib.rs index ca5609d6..b926b312 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod arch_known_failures; pub mod arch_support; pub mod artifact_class; +pub mod backport; pub mod batch; pub mod bot_result; pub mod build_backend; diff --git a/src/main.rs b/src/main.rs index e37960da..d66542ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1075,7 +1075,7 @@ enum StackCommand { Solve(Box), /// Emit CycloneDX from an existing stack lock. #[command( - after_long_help = "Examples:\n eb-stack stack sbom --lock stack.lock.json --out stack.cdx.json" + after_long_help = "Examples:\n eb-stack stack sbom --lock stack.lock.json --out stack.cdx.json\n eb-stack stack sbom --lock stack.lock.json --backport-list backport.md --osv-cache osv/ --reachability reach.txt" )] Sbom { /// Stack lock JSON written by `stack solve`. @@ -1084,6 +1084,23 @@ enum StackCommand { /// CycloneDX file to write. A single file, so this is `--out` rather than `--out-dir`. #[arg(long, default_value = "stack.cdx.json")] out: PathBuf, + /// Markdown backport list for this lock. A path ending in `.md` also + /// writes the sibling `.json`. + #[arg(long)] + backport_list: Option, + /// Advisory id to resolve through OSV. Repeatable. Needs `--backport-list`. + #[arg(long)] + cve: Vec, + /// Directory of recorded OSV responses (`queries.json` and the bodies + /// it names). Needs `--backport-list`. Without `--cve`, every id in + /// that file is resolved. + #[arg(long)] + osv_cache: Option, + /// Site reachability file, one `name class` per line. Needs + /// `--backport-list`. The classes are `cross-user`, `network-facing`, + /// `local-only`, and `not-reachable`. + #[arg(long)] + reachability: Option, }, /// Order the builds a set of roots needs, as a graph rather than a stack. /// @@ -3292,7 +3309,24 @@ fn run_stack(command: StackCommand) -> Result<()> { } Ok(()) } - StackCommand::Sbom { lock, out } => { + StackCommand::Sbom { + lock, + out, + backport_list, + cve, + osv_cache, + reachability, + } => { + if backport_list.is_none() + && (!cve.is_empty() || osv_cache.is_some() || reachability.is_some()) + { + return Err(usage( + "--cve, --osv-cache, and --reachability need --backport-list", + )); + } + if backport_list.is_some() && cve.is_empty() && osv_cache.is_none() { + return Err(usage("--backport-list needs --cve or --osv-cache")); + } let loaded: StackLock = load_lock_file(&lock)?; let (sbom, stats) = eb_stack::lock_to_cyclonedx_from_recipes(&loaded, &lock); write_json_pretty(&out, &sbom)?; @@ -3304,6 +3338,24 @@ fn run_stack(command: StackCommand) -> Result<()> { stats.recipes_unreadable, stats.with_artifact_facts ); + if let Some(path) = &backport_list { + let document = + eb_stack::backport::backport_list(&eb_stack::backport::BackportRequest { + lock: &loaded, + lock_path: &lock, + cves: &cve, + osv_cache: osv_cache.as_deref(), + reachability: reachability.as_deref(), + })?; + eb_stack::backport::write_backport(path, &document)?; + println!( + "backport={} recorded={} stack={} out={}", + document.listed, + document.recorded, + document.stack_modules, + path.display() + ); + } Ok(()) } } diff --git a/src/mcp.rs b/src/mcp.rs index 7484f61b..854535e7 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -343,9 +343,15 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_sbom", - "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names.", + "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names. With backport_list, also write the modules whose reach class justifies a backport.", &[("lock", "string")], - &[("out", "string")], + &[ + ("out", "string"), + ("backport_list", "string"), + ("cve", "array"), + ("osv_cache", "string"), + ("reachability", "string"), + ], ), tool_with_optional( "eb_stack_order", @@ -2876,6 +2882,17 @@ fn stack_solve(arguments: &Value) -> Result { fn stack_sbom(arguments: &Value) -> Result { let lock_path = required_path(arguments, "lock")?; let out = optional_path(arguments, "out").unwrap_or_else(|| PathBuf::from("stack.cdx.json")); + let backport_list = optional_path(arguments, "backport_list"); + let cve = string_array(arguments, "cve")?; + let osv_cache = optional_path(arguments, "osv_cache"); + let reachability = optional_path(arguments, "reachability"); + if backport_list.is_none() && (!cve.is_empty() || osv_cache.is_some() || reachability.is_some()) + { + return Err("--cve, --osv-cache, and --reachability need backport_list".into()); + } + if backport_list.is_some() && cve.is_empty() && osv_cache.is_none() { + return Err("backport_list needs cve or osv_cache".into()); + } let lock: StackLock = load_json_file(&lock_path).map_err(|error| error.to_string())?; let (sbom, stats) = lock_to_cyclonedx_from_recipes(&lock, &lock_path); write_json_pretty(&out, &sbom).map_err(|error| error.to_string())?; @@ -2884,14 +2901,30 @@ fn stack_sbom(arguments: &Value) -> Result { .and_then(Value::as_array) .map(Vec::len) .unwrap_or(0); - Ok(json!({ + let mut body = json!({ "components": components, "lock": lock_path, "out": out, "recipes_read": stats.recipes_read, "recipes_unreadable": stats.recipes_unreadable, "with_artifact_facts": stats.with_artifact_facts, - })) + }); + if let Some(path) = &backport_list { + let document = crate::backport::backport_list(&crate::backport::BackportRequest { + lock: &lock, + lock_path: &lock_path, + cves: &cve, + osv_cache: osv_cache.as_deref(), + reachability: reachability.as_deref(), + }) + .map_err(|error| error.to_string())?; + crate::backport::write_backport(path, &document).map_err(|error| error.to_string())?; + body["listed"] = json!(document.listed); + body["recorded"] = json!(document.recorded); + body["stack"] = json!(document.stack_modules); + body["backport_list"] = json!(path); + } + Ok(body) } fn workflow_error(error: anyhow::Error) -> String { diff --git a/src/rebuild_set.rs b/src/rebuild_set.rs index 833bfbd3..9e54215e 100644 --- a/src/rebuild_set.rs +++ b/src/rebuild_set.rs @@ -48,7 +48,9 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use thiserror::Error; /// Schema of [`RebuildDocument`]. Bumped when a field changes meaning. -pub const REBUILD_SET_SCHEMA_VERSION: u32 = 4; +/// +/// Version 5 adds `reach`, the class of each patched easyconfig. +pub const REBUILD_SET_SCHEMA_VERSION: u32 = 5; /// Class for a runtime dependency, direct or through other runtime dependencies. pub const CLASS_NO_REBUILD: &str = "no-rebuild"; @@ -179,6 +181,11 @@ pub struct RebuildDocument { pub easystacks: Vec, /// Patched easyconfig filenames, sorted. pub changed: Vec, + /// Reach class of each filename in [`Self::changed`], including missing + /// ones. `cross-user` and `network-facing` justify a backport. + /// `local-only` and `not-reachable` do not. `unclassified` means neither + /// the built-in names nor a site file names the package. + pub reach: BTreeMap, /// Changed filenames that are not in the deployed graph. pub missing: Vec, /// Requirements no easyconfig satisfies, and easystack entries the robot @@ -321,6 +328,8 @@ pub fn rebuild_set(query: &RebuildQuery) -> Result Result String { inlined static link is reported as no-rebuild. A short name shared with another \ library is reported as must-rebuild. An empty export set stays in review. Scripts, \ bytecode, and libraries opened with dlopen are invisible. A static archive shows \ - that a static link was possible, not that it happened.\n", + that a static link was possible, not that it happened.\n\n", + ); + out.push_str( + "Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` is not a judgment.\n\n", ); + for (name, class) in &document.reach { + out.push_str(&format!("- `{name}`: {class}\n")); + } out } @@ -1151,6 +1167,16 @@ mod tests { assert!(markdown.contains("`zlib-1.3.1-GCCcore-15.2.0.eb`")); assert!(markdown.contains("2 must-rebuild, 2 review, 4 no-rebuild.")); assert!(markdown.contains("--check-static")); + assert!(markdown.contains( + "Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport." + )); + assert_eq!( + document + .reach + .get("zlib-1.3.1-GCCcore-15.2.0.eb") + .map(String::as_str), + Some("unclassified") + ); } #[test] diff --git a/src/site_commands.rs b/src/site_commands.rs index 30a9646c..8e5a5605 100644 --- a/src/site_commands.rs +++ b/src/site_commands.rs @@ -349,10 +349,14 @@ impl ComponentReachRun { ); } format!( - "component={} version={} generation={} carriers={} changed={} must-rebuild={} review={} no-rebuild={} out={}", + "component={} version={} generation={} reach={} carriers={} changed={} must-rebuild={} review={} no-rebuild={} out={}", self.component, self.version, self.generation, + self.document + .get("reach") + .and_then(|value| value.as_str()) + .unwrap_or("unclassified"), self.carriers, self.changed, self.must_rebuild, @@ -769,7 +773,7 @@ pub fn run_rebuild_set(args: &RebuildSetRequest) -> Result { } else { crate::fix_cost::EvidenceScan::default() }; - let document = rebuild_set::rebuild_set(&rebuild_set::RebuildQuery { + let mut document = rebuild_set::rebuild_set(&rebuild_set::RebuildQuery { base: robot.base, commits: robot.commits, easystacks, @@ -786,6 +790,7 @@ pub fn run_rebuild_set(args: &RebuildSetRequest) -> Result { rebuild_set::RebuildError::CheckStaticNeedsPrefix => usage("--check-static needs --prefix"), other => anyhow!(other), })?; + document.reach = crate::findings::reach_for_easyconfigs(&reach, &document.changed); std::fs::create_dir_all(&args.out_dir)?; write_json_pretty(&args.out_dir.join("rebuild-set.json"), &document)?; diff --git a/tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..ab20918f --- /dev/null +++ b/tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb @@ -0,0 +1,4 @@ +name = 'App' +version = '1.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} +dependencies = [('zlib', '1.3.1')] diff --git a/tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..6c2679dd --- /dev/null +++ b/tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'Mystery' +version = '1.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..687f825a --- /dev/null +++ b/tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'WinOnly' +version = '1.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..1d2f3b1b --- /dev/null +++ b/tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'cURL' +version = '8.17.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..b8943919 --- /dev/null +++ b/tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb @@ -0,0 +1,4 @@ +name = 'git' +version = '2.52.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} +dependencies = [('cURL', '8.17.0')] diff --git a/tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..1e4a2c84 --- /dev/null +++ b/tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'notebook' +version = '7.4.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb new file mode 100644 index 00000000..a63a03d0 --- /dev/null +++ b/tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'zlib' +version = '1.3.1' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/osv/GHSA-test-curl-old.json b/tests/backport_fixtures/osv/GHSA-test-curl-old.json new file mode 100644 index 00000000..ee51ba06 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-curl-old.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-curl-old", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "curl", "purl": "pkg:generic/curl"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "8.0.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-curl.json b/tests/backport_fixtures/osv/GHSA-test-curl.json new file mode 100644 index 00000000..ff9af93c --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-curl.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-curl", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "curl", "purl": "pkg:generic/curl"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "8.18.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-missing.json b/tests/backport_fixtures/osv/GHSA-test-missing.json new file mode 100644 index 00000000..55b08a2a --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-missing.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-missing", + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "jupyter_server", "purl": "pkg:pypi/jupyter-server"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "2.20.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-mystery.json b/tests/backport_fixtures/osv/GHSA-test-mystery.json new file mode 100644 index 00000000..a4509c2e --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-mystery.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-mystery", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "Mystery", "purl": "pkg:generic/mystery"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "2.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-notebook.json b/tests/backport_fixtures/osv/GHSA-test-notebook.json new file mode 100644 index 00000000..28359a05 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-notebook.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-notebook", + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "notebook", "purl": "pkg:pypi/notebook"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "7.5.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-unparsed.json b/tests/backport_fixtures/osv/GHSA-test-unparsed.json new file mode 100644 index 00000000..4c5353aa --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-unparsed.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-unparsed", + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "notebook", "purl": "pkg:pypi/notebook"}, + "ranges": [{"type": "SEMVER", "events": [{"limit": "1"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-win.json b/tests/backport_fixtures/osv/GHSA-test-win.json new file mode 100644 index 00000000..4a7c2ba1 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-win.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-win", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "WinOnly", "purl": "pkg:generic/winonly"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.1"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-zlib.json b/tests/backport_fixtures/osv/GHSA-test-zlib.json new file mode 100644 index 00000000..c18279e8 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-zlib.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-zlib", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "zlib", "purl": "pkg:generic/zlib"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.3.2"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/queries.json b/tests/backport_fixtures/osv/queries.json new file mode 100644 index 00000000..9b81705b --- /dev/null +++ b/tests/backport_fixtures/osv/queries.json @@ -0,0 +1,12 @@ +{ + "queries": [ + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-curl", "file": "GHSA-test-curl.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-curl-old", "file": "GHSA-test-curl-old.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-missing", "file": "GHSA-test-missing.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-mystery", "file": "GHSA-test-mystery.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-notebook", "file": "GHSA-test-notebook.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-unparsed", "file": "GHSA-test-unparsed.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-win", "file": "GHSA-test-win.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-zlib", "file": "GHSA-test-zlib.json"} + ] +} diff --git a/tests/backport_fixtures/reach.txt b/tests/backport_fixtures/reach.txt new file mode 100644 index 00000000..30d76ad1 --- /dev/null +++ b/tests/backport_fixtures/reach.txt @@ -0,0 +1,3 @@ +# site overlay +zlib local-only +WinOnly not-reachable diff --git a/tests/backport_fixtures/stack.lock.json b/tests/backport_fixtures/stack.lock.json new file mode 100644 index 00000000..9cbd8659 --- /dev/null +++ b/tests/backport_fixtures/stack.lock.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "generation_label": "GCCcore-15.2.0", + "packages": [ + { + "name": "App", + "version": "1.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/App-1.0-GCCcore-15.2.0.eb" + }, + { + "name": "Mystery", + "version": "1.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/Mystery-1.0-GCCcore-15.2.0.eb" + }, + { + "name": "WinOnly", + "version": "1.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/WinOnly-1.0-GCCcore-15.2.0.eb" + }, + { + "name": "cURL", + "version": "8.17.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/cURL-8.17.0-GCCcore-15.2.0.eb" + }, + { + "name": "git", + "version": "2.52.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/git-2.52.0-GCCcore-15.2.0.eb" + }, + { + "name": "notebook", + "version": "7.4.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/notebook-7.4.0-GCCcore-15.2.0.eb" + }, + { + "name": "zlib", + "version": "1.3.1", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/zlib-1.3.1-GCCcore-15.2.0.eb" + } + ], + "solver": { + "engine": "resolvo", + "engine_version": "test", + "timestamp": "2026-11-01T00:00:00Z", + "criteria": [] + } +} diff --git a/tests/backport_list.rs b/tests/backport_list.rs new file mode 100644 index 00000000..827532c4 --- /dev/null +++ b/tests/backport_list.rs @@ -0,0 +1,245 @@ +//! `stack sbom --backport-list` from a lock, recorded OSV responses, and the +//! SBOM reverse closure. + +use eb_stack::domain::StackLock; +use eb_stack::mcp::handle_message; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/backport_fixtures") +} + +fn lock_path() -> PathBuf { + root().join("stack.lock.json") +} + +fn load_lock() -> StackLock { + serde_json::from_str(&std::fs::read_to_string(lock_path()).unwrap()).unwrap() +} + +#[test] +fn the_lock_lists_cross_user_and_network_facing_and_records_the_rest() { + let lock = load_lock(); + let cache = root().join("osv"); + let reach = root().join("reach.txt"); + let document = eb_stack::backport::backport_list(&eb_stack::backport::BackportRequest { + lock: &lock, + lock_path: &lock_path(), + cves: &[], + osv_cache: Some(&cache), + reachability: Some(&reach), + }) + .expect("backport list"); + assert_eq!(document.schema_version, 1); + assert_eq!(document.generation, "GCCcore-15.2.0"); + assert_eq!(document.stack_modules, 7); + assert_eq!(document.listed, 2); + assert_eq!(document.recorded, 3); + let listed: Vec<(&str, &str, &str, usize)> = document + .rows + .iter() + .filter(|row| row.on_list) + .map(|row| { + ( + row.module.as_str(), + row.reach.as_str(), + row.advisory.as_str(), + row.blast_radius, + ) + }) + .collect(); + assert_eq!( + listed, + [ + ( + "notebook/7.4.0-GCCcore-15.2.0", + "cross-user", + "GHSA-test-notebook", + 1 + ), + ( + "cURL/8.17.0-GCCcore-15.2.0", + "network-facing", + "GHSA-test-curl", + 2 + ), + ] + ); + let curl = document + .rows + .iter() + .find(|row| row.name == "cURL") + .expect("curl"); + assert_eq!(curl.fixed_in, ["8.18.0"]); + assert_eq!( + curl.blast, + [ + "cURL/8.17.0-GCCcore-15.2.0".to_string(), + "git/2.52.0-GCCcore-15.2.0".to_string() + ] + ); + let recorded: Vec<(&str, &str, usize)> = document + .rows + .iter() + .filter(|row| !row.on_list) + .map(|row| (row.module.as_str(), row.reach.as_str(), row.blast_radius)) + .collect(); + assert_eq!( + recorded, + [ + ("zlib/1.3.1-GCCcore-15.2.0", "local-only", 2), + ("WinOnly/1.0-GCCcore-15.2.0", "not-reachable", 1), + ("Mystery/1.0-GCCcore-15.2.0", "unclassified", 1), + ] + ); + let zlib = document + .rows + .iter() + .find(|row| row.name == "zlib") + .expect("zlib"); + assert_eq!( + zlib.blast, + [ + "App/1.0-GCCcore-15.2.0".to_string(), + "zlib/1.3.1-GCCcore-15.2.0".to_string() + ] + ); + assert_eq!( + document.notes, + [ + "GHSA-test-missing names jupyter_server, which this lock does not have".to_string(), + "GHSA-test-unparsed names notebook, but its ranges were not parsed".to_string(), + "cURL/8.17.0-GCCcore-15.2.0 is not in range for GHSA-test-curl-old (>=0,<8.0.0)" + .to_string(), + ] + ); + let markdown = eb_stack::backport::render_markdown(&document); + assert!(markdown.contains("## Backport")); + assert!(markdown.contains( + "These modules stay off the backport list. A backport is cross-user or network-facing." + )); + assert!(markdown.find("notebook/7.4.0").unwrap() < markdown.find("cURL/8.17.0").unwrap()); +} + +#[test] +fn the_cli_writes_the_list_and_rejects_a_partial_flag_set() { + let tmp = tempfile::tempdir().unwrap(); + let md = tmp.path().join("backport.md"); + let sbom = tmp.path().join("stack.cdx.json"); + let output = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args(["stack", "sbom", "--lock"]) + .arg(lock_path()) + .arg("--out") + .arg(&sbom) + .arg("--backport-list") + .arg(&md) + .arg("--osv-cache") + .arg(root().join("osv")) + .arg("--reachability") + .arg(root().join("reach.txt")) + .output() + .expect("run"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stderr}\n{stdout}"); + assert!( + stdout.contains("components=7 recipes_read=7 recipes_unreadable=0"), + "{stdout}" + ); + assert!( + stdout.contains(&format!( + "backport=2 recorded=3 stack=7 out={}", + md.display() + )), + "{stdout}" + ); + assert!(sbom.is_file()); + let json: Value = + serde_json::from_str(&std::fs::read_to_string(md.with_extension("json")).unwrap()).unwrap(); + assert_eq!(json["listed"], 2); + assert_eq!(json["recorded"], 3); + assert!(std::fs::read_to_string(&md) + .unwrap() + .contains("network-facing")); + + let missing = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args([ + "stack", + "sbom", + "--lock", + "/no/such.lock", + "--cve", + "GHSA-test-curl", + ]) + .output() + .expect("run"); + assert_eq!(missing.status.code(), Some(2)); + let err = String::from_utf8_lossy(&missing.stderr); + assert!(err.contains("need --backport-list"), "{err}"); + + let bare = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args([ + "stack", + "sbom", + "--lock", + "/no/such.lock", + "--backport-list", + "/tmp/backport.md", + ]) + .output() + .expect("run"); + assert_eq!(bare.status.code(), Some(2)); + let err = String::from_utf8_lossy(&bare.stderr); + assert!(err.contains("needs --cve or --osv-cache"), "{err}"); +} + +#[test] +fn one_cve_does_not_pull_the_rest_of_the_cache() { + let lock = load_lock(); + let cache = root().join("osv"); + let document = eb_stack::backport::backport_list(&eb_stack::backport::BackportRequest { + lock: &lock, + lock_path: &lock_path(), + cves: &["GHSA-test-curl".into()], + osv_cache: Some(&cache), + reachability: None, + }) + .expect("one advisory"); + assert_eq!(document.advisories, ["GHSA-test-curl"]); + assert_eq!(document.listed, 1); + assert_eq!(document.recorded, 0); + assert!(document.notes.is_empty(), "{:?}", document.notes); + assert_eq!(document.rows[0].reach, "network-facing"); +} + +#[test] +fn mcp_stack_sbom_can_write_the_backport_list() { + let tmp = tempfile::tempdir().unwrap(); + let out = tmp.path().join("stack.cdx.json"); + let md = tmp.path().join("backport.md"); + let response = handle_message(&json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "eb_stack_sbom", + "arguments": { + "lock": lock_path(), + "out": &out, + "backport_list": &md, + "osv_cache": root().join("osv"), + "reachability": root().join("reach.txt") + } + } + })) + .expect("mcp"); + assert_eq!(response["result"]["isError"], false, "{response}"); + let body = &response["result"]["structuredContent"]; + assert_eq!(body["listed"], 2); + assert_eq!(body["recorded"], 3); + assert_eq!(body["stack"], 7); + assert!(md.is_file()); + let _ = Path::new(&out); +} diff --git a/tests/component_reach.rs b/tests/component_reach.rs index 7107ab62..1a2e2151 100644 --- a/tests/component_reach.rs +++ b/tests/component_reach.rs @@ -48,7 +48,8 @@ fn the_cli_writes_toml_json_and_markdown() { let json: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(out.join("component-reach.json")).unwrap()) .unwrap(); - assert_eq!(json["schema_version"], 1); + assert_eq!(json["schema_version"], 2); + assert_eq!(json["reach"], "network-facing"); assert_eq!(json["component"], "curl"); assert!(json["closure_counts"]["no-rebuild"].as_u64().unwrap() >= 1); let toml = std::fs::read_to_string(out.join("component-reach.toml")).unwrap(); diff --git a/tests/rebuild_set.rs b/tests/rebuild_set.rs index cf1f2258..b4aa10f1 100644 --- a/tests/rebuild_set.rs +++ b/tests/rebuild_set.rs @@ -138,7 +138,12 @@ fn the_cli_writes_the_rebuild_set_and_drops_a_stale_easystack() { .1 .as_str(); let value: serde_json::Value = serde_json::from_str(json).expect("parse json"); - assert_eq!(value["schema_version"], 4); + assert_eq!(value["schema_version"], 5); + assert_eq!( + value["reach"]["zlib-1.3.1-GCCcore-15.2.0.eb"], + "unclassified" + ); + assert_eq!(value["reach"]["Absent-1.0.eb"], "unclassified"); assert_eq!(value["symbols_checked"], false); assert_eq!(value["generated_at"], "2026-10-22T12:00:00Z"); assert_eq!(value["installed_source"], "x86_64/amd/zen4");