diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index e400acdf..2a844467 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -160,6 +160,7 @@ jobs: cargo test --locked \ --test arch_known_failures \ --test arch_unsupported \ + --test backport_list \ --test baseline_easystack \ --test batch_versions \ --test bot_classify \ @@ -172,6 +173,7 @@ jobs: --test easyblock_layers \ --test easyblock_owed \ --test extension_provides \ + --test family_site_layer \ --test fetch_prs_script \ --test fix_cost \ --test from_commit \ diff --git a/.gitignore b/.gitignore index f346a0a8..747abc12 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ docs/source/images/ # Local solve outputs stack.lock.json +!tests/backport_fixtures/stack.lock.json stack.cdx.json build.list stack.diff.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bf1cb4d..07b7aabb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,27 @@ All notable changes to this project are documented here. and older-EESSI inventories. A module in the target is `moved` and names its EESSI version; an excluded module is `site-only`; every other missing route is `gone` until it lands. +- `stack solve` warns on standard error when a foss solve's baseline easystack names an LLVM-family recipe (`lfoss`, `lompi`, `lfbf`, or `llvm-compilers`) and the stack diff marks that package added. + The foss universe cannot see that family, so the added row is the site layer by mistake. + The solve still exits 0. + Keep one lock per toolchain family, or pass `--baseline-also-toolchain` for that family. + `llvm-compilers-20.1.8` and `llvm-compilers-21.1.8` are the versions `data/eessi-generations.toml` lists for generations 2025b and 2026.1. +- `component-reach` and `rebuild-set` record a reach class: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified` when no built-in name and no `--reachability` file names the package. + `component-reach` uses the built-in names only. + `cross-user` and `network-facing` justify a backport. + Component-reach schema is 2. + Rebuild-set schema is 5. + The component-reach summary prints `reach=`. + `rebuild-set.md` ends with the class of each patched easyconfig. + `--reachability` on `rebuild-set` applies to that map as well as to `backport.md`. +- `stack sbom --backport-list` writes that list for a stack lock. + `--cve` resolves an advisory through Open Source Vulnerabilities (OSV). + `--osv-cache` reads recorded responses and, without `--cve`, every id in `queries.json`. + `--reachability` is the same `name class` file as `rebuild-set`. + Blast radius is the module plus every locked module that depends on it through a runtime or build edge. + `local-only`, `not-reachable`, and `unclassified` are recorded and left off the list. + A `.md` path also writes the sibling `.json`. + `eb_stack_sbom` takes the same optional arguments and reports `listed`, `recorded`, and `stack` when the list is requested. - `stack shard` splits one Jenkins build list into shards that can build at the same time. A line's set is its closure over `--easyconfigs` minus the modules in `--installed-modules-dir` Lmod diff --git a/docs/orgmode/explanation/sbom-blast-radius.org b/docs/orgmode/explanation/sbom-blast-radius.org index 9b09baa5..3efd40d8 100644 --- a/docs/orgmode/explanation/sbom-blast-radius.org +++ b/docs/orgmode/explanation/sbom-blast-radius.org @@ -804,12 +804,40 @@ sed -n '27,29p' /tmp/sbom-demo/rebuild/rebuild-set.md | `git-2.52.0-GCCcore-15.2.0.eb` | `git/2.52.0-GCCcore-15.2.0` | changed | must-rebuild | patched easyconfig | #+end_src +#+begin_src bash +grep 'Reach class' -A 2 /tmp/sbom-demo/rebuild/rebuild-set.md +#+end_src + +#+begin_src text +Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` is not a judgment. + +- `git-2.52.0-GCCcore-15.2.0.eb`: network-facing +#+end_src + The report has one row: the changed git easyconfig, class must-rebuild. -The easystack entries do not depend on that git module, so the walk does -not add a second row. The report also notes each ~from-commit~ pin whose -tree was not passed and says the release tree was used for that pin. -~--check-static~ was not passed, and the report says so. Classes follow -the easyconfig graph only. +The easystack entries do not depend on that git module, so the walk does not add a second row. +The report also notes each ~from-commit~ pin whose tree was not passed and says the release tree was used for that pin. +~--check-static~ was not passed, and the report says so. +Classes follow the easyconfig graph only. git is ~network-facing~, so this patched easyconfig would justify a backport. + +A stack lock has the same classes. +~stack sbom --backport-list~ reads recorded Open Source Vulnerabilities (OSV) responses and the software bill of materials (SBOM) reverse closure. +~cross-user~ and ~network-facing~ are the list. +The fixture lock has cURL (network-facing, blast radius 2 because git depends on it) and notebook (cross-user). zlib is ~local-only~ in the site file, so it is recorded and left off the list. + +#+begin_src bash +eb-stack stack sbom \ + --lock tests/backport_fixtures/stack.lock.json \ + --out /tmp/sbom-demo/backport.cdx.json \ + --backport-list /tmp/sbom-demo/backport.md \ + --osv-cache tests/backport_fixtures/osv \ + --reachability tests/backport_fixtures/reach.txt +#+end_src + +#+begin_src text +sbom=/tmp/sbom-demo/backport.cdx.json components=7 recipes_read=7 recipes_unreadable=0 with_artifact_facts=0 +backport=2 recorded=3 stack=7 out=/tmp/sbom-demo/backport.md +#+end_src * A component named by a CVE @@ -832,7 +860,7 @@ eb-stack component-reach \ #+end_src #+begin_src text -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl #+end_src #+begin_src bash @@ -840,7 +868,7 @@ sed -n '5,16p' /tmp/sbom-demo/component-curl/component-reach.md #+end_src #+begin_src text -2 carriers. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. +2 carriers. Reach: `network-facing`. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. | Easyconfig | Module | Role | Version | Installed | |---|---|---|---|---| @@ -1023,10 +1051,10 @@ eb_easyblock_reach eb_repro_summary #+end_src -That is 50 tools. ~eb_stack_sbom~ writes the CycloneDX file ~stack sbom~ -writes. Its ~tools/list~ description reads: "Emit CycloneDX from an existing -stack lock, reading dependency edges and source checksums from the -easyconfigs the lock names." ~eb_stack_sbom~ reads the Cbc lock: +That is 50 tools. +~eb_stack_sbom~ writes the CycloneDX file ~stack sbom~ writes. +Its ~tools/list~ description reads: "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names. With backport_list, also write the modules whose reach class justifies a backport." +~eb_stack_sbom~ reads the Cbc lock: #+begin_src bash printf '%s\n' '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"eb_stack_sbom","arguments":{"lock":"/tmp/sbom-demo/stack.lock.json","out":"/tmp/sbom-demo/mcp-stack.cdx.json"}}}' \ diff --git a/docs/orgmode/howto/component-reach.org b/docs/orgmode/howto/component-reach.org index df9e3856..d8ca861c 100644 --- a/docs/orgmode/howto/component-reach.org +++ b/docs/orgmode/howto/component-reach.org @@ -25,7 +25,7 @@ eb-stack component-reach \ The command prints: #+begin_example -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach #+end_example cURL 8.17.0 is the module. git 2.52.0 depends on that cURL and is the @@ -50,12 +50,15 @@ eb-stack component-reach \ The command prints: #+begin_example -component=curl version=>=8,<9 generation=2026.1 carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full #+end_example -The changed files are ~cURL-8.17.0-GCCcore-15.2.0.eb~ and -~cargo-c-0.10.23-GCCcore-15.2.0.eb~. The must-rebuild row is -~cURL/8.17.0-GCCcore-15.2.0~, reason ~patched easyconfig~. +The changed files are ~cURL-8.17.0-GCCcore-15.2.0.eb~ and ~cargo-c-0.10.23-GCCcore-15.2.0.eb~. +The must-rebuild row is ~cURL/8.17.0-GCCcore-15.2.0~, reason ~patched easyconfig~. +The summary's ~reach=~ is the class of the component name: ~cross-user~, ~network-facing~, ~local-only~, ~not-reachable~, or ~unclassified~ when the built-in names do not include it. +~component-reach~ reads no site file. +~cross-user~ and ~network-facing~ justify a backport. +cURL is ~network-facing~. ~--easystack~ and ~--installed~ narrow the walk to a deployed prefix. ~--filter-deps~ drops dependency tuples before the closure walk. When diff --git a/docs/orgmode/howto/rebuild-set.org b/docs/orgmode/howto/rebuild-set.org index ec5db2f6..2078740d 100644 --- a/docs/orgmode/howto/rebuild-set.org +++ b/docs/orgmode/howto/rebuild-set.org @@ -29,6 +29,12 @@ The command prints: changed=2 rows=8 must-rebuild=2 review=2 no-rebuild=4 missing=1 easystacks=2 skipped_easyconfigs=0 out=/tmp/rebuild-out #+end_example +~rebuild-set.md~ ends with the reach class of each changed easyconfig. +~cross-user~ and ~network-facing~ justify a backport. +~local-only~ and ~not-reachable~ do not. +~unclassified~ means neither the built-in names nor a ~--reachability~ file names the package. +zlib and ~Absent-1.0.eb~ on this fixture are ~unclassified~. + * Run it on a mounted stack From a clone of ~EESSI/software-layer~, with the 2026.06 stack mounted. diff --git a/docs/orgmode/howto/solve-lock.org b/docs/orgmode/howto/solve-lock.org index 5cde31da..681cd7c9 100644 --- a/docs/orgmode/howto/solve-lock.org +++ b/docs/orgmode/howto/solve-lock.org @@ -62,6 +62,15 @@ that name is also in the lock. A path that cannot be read contributes nothing for that component. ~--sbom-out~ during ~stack solve~ writes the same document while the candidate graph is still in memory. +~--backport-list~ writes the locked modules whose reach class justifies a backport. +Pass ~--cve~ or ~--osv-cache~. +~--osv-cache~ reads recorded Open Source Vulnerabilities (OSV) responses and does not query the network when ~--cve~ is omitted. +~--reachability~ adds or replaces a class. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +~cross-user~ and ~network-facing~ are the list. +~local-only~, ~not-reachable~, and ~unclassified~ are recorded and left off it. +See [[file:../reference/cli.org][the CLI reference]]. + * Order every pinned build ~stack solve~ picks one version of each name. ~stack order~ keeps every @@ -81,5 +90,46 @@ eb-stack stack order \ requirement. The hash file is one ~ ~ line per build. See [[file:../reference/cli.org][the CLI reference]]. +* One lock per toolchain family + +A foss solve only sees the foss hierarchy. +The LLVM family is ~lfoss~, ~lompi~, ~lfbf~, and ~llvm-compilers~. +OpenMPI under ~llvm-compilers-21.1.8~ is in that family. +~data/eessi-generations.toml~ lists ~llvm-compilers-20.1.8~ for generation 2025b and ~llvm-compilers-21.1.8~ for generation 2026.1, because an LLVM release is not a generation name. + +An easystack entry such as ~GROMACS-2025.0-lfoss-2025b.eb~ is absent from a foss baseline. +The foss solve then selects the foss build of the same name, and the stack diff marks that row added. +Added is the site layer. +Keep one lock per toolchain family: solve ~lfoss~ on its own. +Passing ~--baseline-also-toolchain lfoss/2025b~ keeps that family's hierarchy in the foss baseline, and the warning stays quiet for ~lfoss~. + +#+begin_src bash +eb-stack stack solve \ + --easyconfigs fixtures/gromacs_2025_to_next/easyconfigs \ + --toolchain foss/2025b \ + --root GROMACS \ + --baseline-easystack /tmp/lfoss.yml \ + --lock-out /tmp/foss.lock.json \ + --stack-diff-out /tmp/foss.diff.md +#+end_src + +~/tmp/lfoss.yml~ has one entry: + +#+begin_src yaml +easyconfigs: + - GROMACS-2025.0-lfoss-2025b.eb +#+end_src + +The solve still exits 0. +Standard error names the unresolved entry. +It also prints a warning that the added GROMACS row is the LLVM family: + +#+begin_src text +warning: GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS (fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b. +#+end_src + +The path in that warning is the foss easyconfig the solve selected. +A package the foss closure only version-bumps stays off this warning. + See [[file:../reference/policy.org][the policy reference]] and [[file:../reference/formats.org][the format reference]]. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 741a5eab..6f4e425d 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -564,6 +564,21 @@ is the component property ~eb_stack:buildDependsOn~. ~stack solve --sbom-out~ writes that same document, checksums included. The MCP tool ~eb_stack_sbom~ writes it too. An unreadable path contributes no edge. +~--backport-list~ writes a markdown list of locked modules whose reach class justifies a backport. +A path ending in ~.md~ also writes the sibling ~.json~. +~--cve~ is repeatable and resolves that advisory through Open Source Vulnerabilities (OSV). +~--osv-cache~ is a directory of recorded responses: ~queries.json~ maps each request URL to a body file. +Without ~--cve~, every id in that file is resolved, and the command does not query the network. +~--reachability~ is the same ~name class~ file as ~rebuild-set~. +The classes are ~cross-user~, ~network-facing~, ~local-only~, and ~not-reachable~. +A name neither the built-in table nor that file mentions is ~unclassified~. +Only ~cross-user~ and ~network-facing~ are on the list. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +The command writes the CycloneDX file, then prints ~backport=~, ~recorded=~, ~stack=~, and ~out=~. +~--cve~, ~--osv-cache~, and ~--reachability~ need ~--backport-list~. +~--backport-list~ needs ~--cve~ or ~--osv-cache~. +~eb_stack_sbom~ takes the same arguments as ~backport_list~, ~cve~, ~osv_cache~, and ~reachability~. + ~stack solve~ is the multi-root EasyBuild stack solver. Package-profile stack preferences use the TOML schema documented with the package workflow; whole stack solving uses the JSON ~Policy~ schema. The policy file is @@ -579,26 +594,23 @@ requirement; a builddependency is dashed. A ~.svg~ or ~.png~ path is rendered with the Graphviz ~dot~ program. Any other suffix, including ~.dot~, is the DOT source. The MCP field ~dot_out~ writes the same file. -The baseline is either a previous generation (~--baseline-easyconfigs~, by -default the first ~--easyconfigs~ tree) or what a set of easystacks installs -(~--baseline-easystack~, repeatable). An easystack baseline is every entry -plus the closure its recipes pin, read from the ~--easyconfigs~ trees and kept -to the policy toolchain's hierarchy. An entry pinned with ~from-commit~ is read -from that commit of ~--baseline-commits-from~ when given, and matched by -filename otherwise. Entries that name no recipe are listed on standard error. -~--baseline-also-toolchain~ keeps another family's hierarchy in the baseline, -such as ~lfoss/2026.1~ beside a ~foss/2026.1~ solve. ~--filter-deps~ drops each named dependency before the solve, together -with the subtree under it, using EasyBuild's matching: a bare name, -~name=version~, or ~name=[low:high]~. ~[~ and ~]~ include a limit, the -opposite bracket excludes it, and an empty side is open. The last spec for -a name wins. Pass EESSI-extend's ~EASYBUILD_FILTER_DEPS~ list when solving -against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat -layer already provides. -Against an EESSI version's easystacks, ~unchanged~ in the stack diff means the -software layer already provides that module, and -~--build-list-excludes-baseline~ drops those modules (same name, version, -toolchain and versionsuffix) from the build list, leaving what the site layer -builds on top in dependency order. +The baseline is either a previous generation (~--baseline-easyconfigs~, by default the first ~--easyconfigs~ tree) or what a set of easystacks installs (~--baseline-easystack~, repeatable). +An easystack baseline is every entry plus the closure its recipes pin, read from the ~--easyconfigs~ trees and kept to the policy toolchain's hierarchy. +An entry pinned with ~from-commit~ is read from that commit of ~--baseline-commits-from~ when given, and matched by filename otherwise. +Entries that name no recipe are listed on standard error. +~--baseline-also-toolchain~ keeps another family's hierarchy in the baseline, such as ~lfoss/2026.1~ beside a ~foss/2026.1~ solve. +A foss solve cannot see the LLVM family (~lfoss~, ~lompi~, ~lfbf~, ~llvm-compilers~). +A baseline easystack can name one of those toolchains. +If the stack diff then marks that package added, the solve warns on standard error and still exits 0. +The added row is the site layer because the foss universe dropped the LLVM recipe. +Keep one lock per toolchain family, or pass ~--baseline-also-toolchain~ for that family so its hierarchy stays in the baseline. +See [[file:../howto/solve-lock.org][one lock per toolchain family]]. +~--filter-deps~ drops each named dependency before the solve, together with the subtree under it, using EasyBuild's matching: a bare name, ~name=version~, or ~name=[low:high]~. +~[~ and ~]~ include a limit. +The opposite bracket excludes it, and an empty side is open. +The last spec for a name wins. +Pass EESSI-extend's ~EASYBUILD_FILTER_DEPS~ list when solving against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat layer already provides. +Against an EESSI version's easystacks, ~unchanged~ in the stack diff means the software layer already provides that module, and ~--build-list-excludes-baseline~ drops those modules (same name, version, toolchain and versionsuffix) from the build list, leaving what the site layer builds on top in dependency order. A Jenkins build list is one easyconfig filename per line with optional ~eb~ flags, ~#~ for comments. ~--roots-from-build-list~ takes the package name of @@ -1456,10 +1468,12 @@ review. The scan sees dynamic symbols only, so a hidden or inlined static link is a false no-rebuild, and a short name shared with another library is a false must-rebuild. -The output directory contains ~rebuild-set.json~, ~rebuild-set.md~, and -~rebuilds/-eb--cve-patches.yml~. The easystack contains -class ~must-rebuild~ only, including a dependent that ~--check-static~ -promoted. A second run replaces the ~rebuilds/*.yml~ files. +The output directory contains ~rebuild-set.json~, ~rebuild-set.md~, and ~rebuilds/-eb--cve-patches.yml~. +The easystack contains class ~must-rebuild~ only, including a dependent that ~--check-static~ promoted. +A second run replaces the ~rebuilds/*.yml~ files. +~rebuild-set.json~ and ~rebuild-set.md~ record the reach class of each patched easyconfig, including a filename listed under ~missing~. +~--reachability~ replaces or adds names for that map as well as for ~backport.md~. +~cross-user~ and ~network-facing~ justify a backport. * ~component-reach~ @@ -1482,26 +1496,25 @@ that carries a component, then feeds the recipes that contain it into the rebuild closure. See [[file:../howto/component-reach.org][which modules carry a component]]. -Pass ~--component~ and ~--version~, or ~--purl~, or ~--cve~. ~--version~ -is an exact version, ~*~, or a requirement such as ~>=8,<9~. With -~--purl~ it replaces the version in the URL. The purl type selects the -carriers. ~generic~ and ~github~ match the easyconfig name. ~pypi~, -~cargo~, ~npm~, ~golang~, and ~cran~ match an extension or a bundled -component of that language, plus a crate for ~cargo~ and a wheel for -~pypi~. Another type exits 2: the error says that purl type has no -carrier model. ~pkg:cargo/curl~ does not match the cURL recipe. -~--cve~ resolves the id through Open Source Vulnerabilities (OSV), follows GitHub Security Advisory (GHSA), RUSTSEC, PYSEC, and -GO aliases when the Common Vulnerabilities and Exposures (CVE) record has only a git range, and keeps named -version tags such as ~libXpm-3.5.17~. A GIT range is mapped onto -release versions: the affected entry's ~versions~ list when it has one, -and otherwise tags from the upstream repository (~git ls-remote --tags~, -then ~git merge-base --is-ancestor~ on a blobless fetch). That lookup is -cached and is skipped when it cannot reach the network, and the range -then stays under Unresolved ranges. An event that does not parse is -reported there too. A package whose ranges were all unparsed counts as -review. Several affected packages are written as one document with -~kind~ ~cve~. The summary line includes ~changed~, ~must-rebuild~, -~review~, and ~no-rebuild~. +Pass ~--component~ and ~--version~, or ~--purl~, or ~--cve~. +~--version~ is an exact version, ~*~, or a requirement such as ~>=8,<9~. +With ~--purl~ it replaces the version in the URL. +The purl type selects the carriers. +~generic~ and ~github~ match the easyconfig name. +~pypi~, ~cargo~, ~npm~, ~golang~, and ~cran~ match an extension or a bundled component of that language, plus a crate for ~cargo~ and a wheel for ~pypi~. +Another type exits 2: the error says that purl type has no carrier model. +~pkg:cargo/curl~ does not match the cURL recipe. +~--cve~ resolves the id through Open Source Vulnerabilities (OSV). +When the Common Vulnerabilities and Exposures (CVE) record has only a git range, it follows GitHub Security Advisory (GHSA), ~RUSTSEC~, ~PYSEC~, and ~GO~ aliases. +It keeps named version tags such as ~libXpm-3.5.17~. +A ~GIT~ range is mapped onto release versions: the affected entry's ~versions~ list when it has one, and otherwise tags from the upstream repository (~git ls-remote --tags~, then ~git merge-base --is-ancestor~ on a blobless fetch). +That lookup is cached and is skipped when it cannot reach the network, and the range then stays under Unresolved ranges. +An event that does not parse is reported there too. +A package whose ranges were all unparsed counts as review. +An advisory that affects more than one package is written as one document with ~kind~ ~cve~. +The summary line includes ~reach~, ~changed~, ~must-rebuild~, ~review~, and ~no-rebuild~. +~reach~ is the class of the component name: ~cross-user~, ~network-facing~, ~local-only~, ~not-reachable~, or ~unclassified~. +A ~--cve~ summary has no single class. Names match ignoring ASCII letter case, on a whole token. A recipe is in the generation when its toolchain version equals ~--generation~, or when the diff --git a/docs/orgmode/reference/formats.org b/docs/orgmode/reference/formats.org index 3b27f936..9c49c4ac 100644 --- a/docs/orgmode/reference/formats.org +++ b/docs/orgmode/reference/formats.org @@ -534,13 +534,14 @@ filename carried a version, the file is ~unknown~. | Field | Meaning | |- -| ~schema_version~ | ~4~ | +| ~schema_version~ | ~5~ | | ~generated_at~ | RFC 3339 timestamp of the run | | ~installed_source~ | Label of the installed list | | ~prefix~ | Installed prefix passed to ~--prefix~. Omitted when the flag was not passed | | ~symbols_checked~ | ~true~ when ~--check-static~ ran | | ~easystacks~ | Easystack paths that were read, sorted | | ~changed~ | Patched easyconfig filenames, sorted | +| ~reach~ | Reach class of each filename in ~changed~, including missing ones. ~cross-user~ and ~network-facing~ justify a backport. ~local-only~ and ~not-reachable~ do not. ~unclassified~ means neither the built-in names nor ~--reachability~ names the package | | ~missing~ | Changed filenames that are not in the deployed graph | | ~unresolved~ | Requirements no easyconfig satisfies, and easystack entries the robot path does not have | | ~notes~ | A missing easyblock class, a ~from-commit~ whose tree was not given, or an install directory that was not found. Empty when there is nothing to say | @@ -554,8 +555,8 @@ when an easystack filename carried one. On a review row, ~changed~ is the patched filenames joined by ~, ~. After ~--check-static~, a hit keeps only the filenames whose exports were defined. -~rebuild-set.md~ is the same rows as a table, the counts line, the missing -filenames, and each note as ~- note:~. +~rebuild-set.md~ is the same rows as a table, the counts line, the missing filenames, and each note as ~- note:~. +It ends with the reach class of each patched easyconfig. ** ~findings.json~, ~findings.md~, ~findings.csv~ @@ -674,7 +675,7 @@ One run writes ~component-reach.toml~, ~component-reach.json~, and | Field | Meaning | |- -| ~schema_version~ | ~1~ | +| ~schema_version~ | ~2~ | | ~generated_at~ | RFC 3339 timestamp of the run | | ~component~ | Name that was asked for | | ~version_req~ | Exact version, ~*~, or a requirement. Empty when the match is only ~versions~ | @@ -683,6 +684,7 @@ One run writes ~component-reach.toml~, ~component-reach.json~, and | ~purl~ | Package URL, when the query was a purl or a resolved CVE | | ~ecosystem~ | Ecosystem from the purl or the Open Source Vulnerabilities (OSV) record | | ~generation~ | Toolchain generation | +| ~reach~ | Class of the component name: ~cross-user~, ~network-facing~, ~local-only~, ~not-reachable~, or ~unclassified~ | | ~carriers~ | Easyconfigs and modules that carry it | | ~changed~ | Filenames that contain it, fed to the closure | | ~closure_counts~ | Row counts, keys ~no-rebuild~, ~review~, ~must-rebuild~ | @@ -700,15 +702,40 @@ compat-layer and installed-only rows), ~module~, and ~installed~ when a list was passed. A ~--cve~ run writes the same three filenames with a different document. -~kind~ is ~cve~. ~cve~ is the id. ~resolved~ is the packages OSV named -(~ecosystem~, ~name~, ~purl~, ~version_reqs~, ~versions~). ~results~ is -one component-reach document per package. ~notes~ contains alias fetches -that failed. Schema version stays ~1~. +~kind~ is ~cve~. +~cve~ is the id. +~resolved~ is the packages the Open Source Vulnerabilities (OSV) record named (~ecosystem~, ~name~, ~purl~, ~version_reqs~, ~versions~). +~results~ is one component-reach document per package. +~notes~ contains alias fetches that failed. +Schema version stays ~2~. +Each package document carries its own ~reach~. A closure row has the rebuild-set fields ~changed~, ~dependent~, ~dependent_easyconfig~, ~edge_kind~, ~class~, ~reason~, and ~eb_version~ when an easystack filename carried one. With no easystack the synthetic stack is ~generation-.yml~ and ~eb_version~ is absent. -~component-reach.md~ is the carrier table, then the closure table, the -counts line, and each note as ~- note:~. +~component-reach.md~ is the carrier table, then the closure table, the counts line, and each note as ~- note:~. +The carriers line names the reach class. + +* Backport list (~eb-stack stack sbom --backport-list~) + +A path ending in ~.md~ writes that markdown and a sibling ~.json~. + +| Field | Meaning | +|- +| ~schema_version~ | ~1~ | +| ~lock~ | Lock path, as it was passed | +| ~generation~ | Toolchain label of the lock | +| ~advisories~ | Advisory ids that were resolved, sorted | +| ~stack_modules~ | Packages in the lock | +| ~listed~ | Rows whose reach class justifies a backport | +| ~recorded~ | Rows left off the list | +| ~rows~ | One entry per locked module an advisory matched. On-list rows come first | +| ~notes~ | An advisory names a package the lock lacks, a locked version is outside the range, or a range did not parse | + +Each row has ~module~ (~Name/version-toolchain~, plus the versionsuffix when the lock has one), ~name~, ~version~, ~reach~, ~advisory~, ~fixed_in~ (upper bounds from ~<~ clauses; a ~<=~ bound is last-affected and is not a fix), ~blast_radius~, ~blast~ (those module labels, sorted), and ~on_list~. +Blast radius counts the module and every locked module that depends on it through a runtime or build edge. + +The markdown lists each module in a backport section or a recorded section. +The recorded sentence is: these modules stay off the backport list, and a backport is cross-user or network-facing. diff --git a/docs/orgmode/reference/glossary.org b/docs/orgmode/reference/glossary.org index 1a3f0caa..cdad0ab5 100644 --- a/docs/orgmode/reference/glossary.org +++ b/docs/orgmode/reference/glossary.org @@ -29,27 +29,29 @@ Flags and exit status stay in the [[file:cli.org][command-line reference]]. ~--baseline-easyconfigs~ and the first ~--easyconfigs~ tree is the baseline. See [[file:../howto/emit-reports.org][emit a build list and stack diff]]. -- toolchain family :: The toolchain name without its version (~foss~, - ~GCCcore~, ~gfbf~). A bump retargets an explicit dependency's family - onto the new generation. ~--baseline-also-toolchain~ keeps another - family's hierarchy in the baseline. This manual does not use the phrase - "family lock": a lock is one solved selection, and a family is only the - toolchain name. On an architecture, family is the text before the first - slash (~x86_64~, ~aarch64~). See - [[file:../howto/arch-unsupported.org][refuse a plan an architecture cannot build]]. +- toolchain family :: The toolchain name without its version (~foss~, ~GCCcore~, ~gfbf~). + A bump retargets an explicit dependency's family onto the new generation. + ~--baseline-also-toolchain~ keeps another family's hierarchy in the baseline. + Keep one lock per toolchain family. + A foss solve that marks an ~lfoss~, ~lompi~, ~lfbf~, or ~llvm-compilers~ package added warns on standard error; see [[file:../howto/solve-lock.org][one lock per toolchain family]]. + This manual does not use the phrase "family lock": a lock is one solved selection, and a family is only the toolchain name. + On an architecture, family is the text before the first slash (~x86_64~, ~aarch64~). + See [[file:../howto/arch-unsupported.org][refuse a plan an architecture cannot build]]. - lock :: The Resolvo result for one selection. A package bundle has ~locks/.lock.json~ per profile. ~stack solve~ writes ~stack.lock.json~ for a whole stack. A ~preferred~ pin may fall back inside the solver. A ~locked~ pin may not. -- reachability :: Four checks, not one word. Host reachability is whether - a target's SSH, rsync, or staging path answers; a failure is a - ~transport~ finding. ~component-reach~ lists modules in one generation - that carry a named component. ~easyblock-reach~ lists easyconfigs that - use a changed class. ~stack arch --probe-sources~ reports ~unreachable~ - when no probed architecture family can fetch a source, and - ~arch-missing~ when another family can. +- reachability :: Four checks, not one word. + Host reachability is whether a target's SSH, rsync, or staging path answers; a failure is a ~transport~ finding. + ~component-reach~ lists modules in one generation that carry a named component. + ~easyblock-reach~ lists easyconfigs that use a changed class. + ~stack arch --probe-sources~ reports ~unreachable~ when no probed architecture family can fetch a source, and ~arch-missing~ when another family can. + A security finding's class is ~cross-user~, ~network-facing~, ~local-only~, or ~not-reachable~. + ~component-reach~ and ~rebuild-set~ record that class. + ~unclassified~ means neither the built-in names nor a ~--reachability~ file names the package. + Only ~cross-user~ and ~network-facing~ justify a backport. - Resolvo :: The conflict-driven clause learning (CDCL) Boolean satisfiability (SAT) solver diff --git a/llms-full.txt b/llms-full.txt index cfe70331..b4a7afcc 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1813,6 +1813,15 @@ that name is also in the lock. A path that cannot be read contributes nothing for that component. `--sbom-out` during `stack solve` writes the same document while the candidate graph is still in memory. +`--backport-list` writes the locked modules whose reach class justifies a backport. +Pass `--cve` or `--osv-cache`. +`--osv-cache` reads recorded Open Source Vulnerabilities (OSV) responses and does not query the network when `--cve` is omitted. +`--reachability` adds or replaces a class. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +`cross-user` and `network-facing` are the list. +`local-only`, `not-reachable`, and `unclassified` are recorded and left off it. +See [the CLI reference](https://eb-stack.rgoswami.me/../reference/cli.html.md). + # Order every pinned build `stack solve` picks one version of each name. `stack order` keeps every @@ -1832,6 +1841,47 @@ eb-stack stack order \ requirement. The hash file is one ` ` line per build. See [the CLI reference](https://eb-stack.rgoswami.me/../reference/cli.html.md). +# One lock per toolchain family + +A foss solve only sees the foss hierarchy. +The LLVM family is `lfoss`, `lompi`, `lfbf`, and `llvm-compilers`. +OpenMPI under `llvm-compilers-21.1.8` is in that family. +`data/eessi-generations.toml` lists `llvm-compilers-20.1.8` for generation 2025b and `llvm-compilers-21.1.8` for generation 2026.1, because an LLVM release is not a generation name. + +An easystack entry such as `GROMACS-2025.0-lfoss-2025b.eb` is absent from a foss baseline. +The foss solve then selects the foss build of the same name, and the stack diff marks that row added. +Added is the site layer. +Keep one lock per toolchain family: solve `lfoss` on its own. +Passing `--baseline-also-toolchain lfoss/2025b` keeps that family's hierarchy in the foss baseline, and the warning stays quiet for `lfoss`. + +```bash +eb-stack stack solve \ + --easyconfigs fixtures/gromacs_2025_to_next/easyconfigs \ + --toolchain foss/2025b \ + --root GROMACS \ + --baseline-easystack /tmp/lfoss.yml \ + --lock-out /tmp/foss.lock.json \ + --stack-diff-out /tmp/foss.diff.md +``` + +`/tmp/lfoss.yml` has one entry: + +```yaml +easyconfigs: + - GROMACS-2025.0-lfoss-2025b.eb +``` + +The solve still exits 0. +Standard error names the unresolved entry. +It also prints a warning that the added GROMACS row is the LLVM family: + +```text +warning: GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS (fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b. +``` + +The path in that warning is the foss easyconfig the solve selected. +A package the foss closure only version-bumps stays off this warning. + See [the policy reference](https://eb-stack.rgoswami.me/../reference/policy.html.md) and [the format reference](https://eb-stack.rgoswami.me/../reference/formats.html.md). @@ -3026,6 +3076,12 @@ The command prints: changed=2 rows=8 must-rebuild=2 review=2 no-rebuild=4 missing=1 easystacks=2 skipped_easyconfigs=0 out=/tmp/rebuild-out ``` +`rebuild-set.md` ends with the reach class of each changed easyconfig. +`cross-user` and `network-facing` justify a backport. +`local-only` and `not-reachable` do not. +`unclassified` means neither the built-in names nor a `--reachability` file names the package. +zlib and `Absent-1.0.eb` on this fixture are `unclassified`. + # Run it on a mounted stack From a clone of `EESSI/software-layer`, with the 2026.06 stack mounted. @@ -3254,7 +3310,7 @@ eb-stack component-reach \ The command prints: ``` -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/component-reach ``` cURL 8.17.0 is the module. git 2.52.0 depends on that cURL and is the @@ -3279,12 +3335,15 @@ eb-stack component-reach \ The command prints: ``` -component=curl version=>=8,<9 generation=2026.1 carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=23 changed=2 must-rebuild=1 review=106 no-rebuild=16 out=/tmp/component-reach-full ``` -The changed files are `cURL-8.17.0-GCCcore-15.2.0.eb` and -`cargo-c-0.10.23-GCCcore-15.2.0.eb`. The must-rebuild row is -`cURL/8.17.0-GCCcore-15.2.0`, reason `patched easyconfig`. +The changed files are `cURL-8.17.0-GCCcore-15.2.0.eb` and `cargo-c-0.10.23-GCCcore-15.2.0.eb`. +The must-rebuild row is `cURL/8.17.0-GCCcore-15.2.0`, reason `patched easyconfig`. +The summary's `reach=` is the class of the component name: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified` when the built-in names do not include it. +`component-reach` reads no site file. +`cross-user` and `network-facing` justify a backport. +cURL is `network-facing`. `--easystack` and `--installed` narrow the walk to a deployed prefix. `--filter-deps` drops dependency tuples before the closure walk. When @@ -5124,27 +5183,29 @@ Flags and exit status stay in the [command-line reference](https://eb-stack.rgos `--baseline-easyconfigs` and the first `--easyconfigs` tree is the baseline. See [emit a build list and stack diff](https://eb-stack.rgoswami.me/../howto/emit-reports.html.md). -- toolchain family :: The toolchain name without its version (`foss`, - `GCCcore`, `gfbf`). A bump retargets an explicit dependency's family - onto the new generation. `--baseline-also-toolchain` keeps another - family's hierarchy in the baseline. This manual does not use the phrase - "family lock": a lock is one solved selection, and a family is only the - toolchain name. On an architecture, family is the text before the first - slash (`x86_64`, `aarch64`). See - [refuse a plan an architecture cannot build](https://eb-stack.rgoswami.me/../howto/arch-unsupported.html.md). +- toolchain family :: The toolchain name without its version (`foss`, `GCCcore`, `gfbf`). + A bump retargets an explicit dependency's family onto the new generation. + `--baseline-also-toolchain` keeps another family's hierarchy in the baseline. + Keep one lock per toolchain family. + A foss solve that marks an `lfoss`, `lompi`, `lfbf`, or `llvm-compilers` package added warns on standard error; see [one lock per toolchain family](https://eb-stack.rgoswami.me/../howto/solve-lock.html.md). + This manual does not use the phrase "family lock": a lock is one solved selection, and a family is only the toolchain name. + On an architecture, family is the text before the first slash (`x86_64`, `aarch64`). + See [refuse a plan an architecture cannot build](https://eb-stack.rgoswami.me/../howto/arch-unsupported.html.md). - lock :: The Resolvo result for one selection. A package bundle has `locks/.lock.json` per profile. `stack solve` writes `stack.lock.json` for a whole stack. A `preferred` pin may fall back inside the solver. A `locked` pin may not. -- reachability :: Four checks, not one word. Host reachability is whether - a target's SSH, rsync, or staging path answers; a failure is a - `transport` finding. `component-reach` lists modules in one generation - that carry a named component. `easyblock-reach` lists easyconfigs that - use a changed class. `stack arch --probe-sources` reports `unreachable` - when no probed architecture family can fetch a source, and - `arch-missing` when another family can. +- reachability :: Four checks, not one word. + Host reachability is whether a target's SSH, rsync, or staging path answers; a failure is a `transport` finding. + `component-reach` lists modules in one generation that carry a named component. + `easyblock-reach` lists easyconfigs that use a changed class. + `stack arch --probe-sources` reports `unreachable` when no probed architecture family can fetch a source, and `arch-missing` when another family can. + A security finding's class is `cross-user`, `network-facing`, `local-only`, or `not-reachable`. + `component-reach` and `rebuild-set` record that class. + `unclassified` means neither the built-in names nor a `--reachability` file names the package. + Only `cross-user` and `network-facing` justify a backport. - Resolvo :: The conflict-driven clause learning (CDCL) Boolean satisfiability (SAT) solver @@ -5772,6 +5833,21 @@ is the component property `eb_stack:buildDependsOn`. `stack solve --sbom-out` writes that same document, checksums included. The MCP tool `eb_stack_sbom` writes it too. An unreadable path contributes no edge. +`--backport-list` writes a markdown list of locked modules whose reach class justifies a backport. +A path ending in `.md` also writes the sibling `.json`. +`--cve` is repeatable and resolves that advisory through Open Source Vulnerabilities (OSV). +`--osv-cache` is a directory of recorded responses: `queries.json` maps each request URL to a body file. +Without `--cve`, every id in that file is resolved, and the command does not query the network. +`--reachability` is the same `name class` file as `rebuild-set`. +The classes are `cross-user`, `network-facing`, `local-only`, and `not-reachable`. +A name neither the built-in table nor that file mentions is `unclassified`. +Only `cross-user` and `network-facing` are on the list. +Blast radius is the module and every locked module that depends on it through a runtime or build edge. +The command writes the CycloneDX file, then prints `backport=`, `recorded=`, `stack=`, and `out=`. +`--cve`, `--osv-cache`, and `--reachability` need `--backport-list`. +`--backport-list` needs `--cve` or `--osv-cache`. +`eb_stack_sbom` takes the same arguments as `backport_list`, `cve`, `osv_cache`, and `reachability`. + `stack solve` is the multi-root EasyBuild stack solver. Package-profile stack preferences use the TOML schema documented with the package workflow; whole stack solving uses the JSON `Policy` schema. The policy file is @@ -5787,26 +5863,23 @@ requirement; a builddependency is dashed. A `.svg` or `.png` path is rendered with the Graphviz `dot` program. Any other suffix, including `.dot`, is the DOT source. The MCP field `dot_out` writes the same file. -The baseline is either a previous generation (`--baseline-easyconfigs`, by -default the first `--easyconfigs` tree) or what a set of easystacks installs -(`--baseline-easystack`, repeatable). An easystack baseline is every entry -plus the closure its recipes pin, read from the `--easyconfigs` trees and kept -to the policy toolchain's hierarchy. An entry pinned with `from-commit` is read -from that commit of `--baseline-commits-from` when given, and matched by -filename otherwise. Entries that name no recipe are listed on standard error. -`--baseline-also-toolchain` keeps another family's hierarchy in the baseline, -such as `lfoss/2026.1` beside a `foss/2026.1` solve. `--filter-deps` drops each named dependency before the solve, together -with the subtree under it, using EasyBuild's matching: a bare name, -`name=version`, or `name=[low:high]`. `[` and `]` include a limit, the -opposite bracket excludes it, and an empty side is open. The last spec for -a name wins. Pass EESSI-extend's `EASYBUILD_FILTER_DEPS` list when solving -against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat -layer already provides. -Against an EESSI version's easystacks, `unchanged` in the stack diff means the -software layer already provides that module, and -`--build-list-excludes-baseline` drops those modules (same name, version, -toolchain and versionsuffix) from the build list, leaving what the site layer -builds on top in dependency order. +The baseline is either a previous generation (`--baseline-easyconfigs`, by default the first `--easyconfigs` tree) or what a set of easystacks installs (`--baseline-easystack`, repeatable). +An easystack baseline is every entry plus the closure its recipes pin, read from the `--easyconfigs` trees and kept to the policy toolchain's hierarchy. +An entry pinned with `from-commit` is read from that commit of `--baseline-commits-from` when given, and matched by filename otherwise. +Entries that name no recipe are listed on standard error. +`--baseline-also-toolchain` keeps another family's hierarchy in the baseline, such as `lfoss/2026.1` beside a `foss/2026.1` solve. +A foss solve cannot see the LLVM family (`lfoss`, `lompi`, `lfbf`, `llvm-compilers`). +A baseline easystack can name one of those toolchains. +If the stack diff then marks that package added, the solve warns on standard error and still exits 0. +The added row is the site layer because the foss universe dropped the LLVM recipe. +Keep one lock per toolchain family, or pass `--baseline-also-toolchain` for that family so its hierarchy stays in the baseline. +See [one lock per toolchain family](https://eb-stack.rgoswami.me/../howto/solve-lock.html.md). +`--filter-deps` drops each named dependency before the solve, together with the subtree under it, using EasyBuild's matching: a bare name, `name=version`, or `name=[low:high]`. +`[` and `]` include a limit. +The opposite bracket excludes it, and an empty side is open. +The last spec for a name wins. +Pass EESSI-extend's `EASYBUILD_FILTER_DEPS` list when solving against a European Environment for Scientific Software Installations (EESSI) baseline, or the build list carries modules the compat layer already provides. +Against an EESSI version's easystacks, `unchanged` in the stack diff means the software layer already provides that module, and `--build-list-excludes-baseline` drops those modules (same name, version, toolchain and versionsuffix) from the build list, leaving what the site layer builds on top in dependency order. A Jenkins build list is one easyconfig filename per line with optional `eb` flags, `#` for comments. `--roots-from-build-list` takes the package name of @@ -6664,10 +6737,12 @@ review. The scan sees dynamic symbols only, so a hidden or inlined static link is a false no-rebuild, and a short name shared with another library is a false must-rebuild. -The output directory contains `rebuild-set.json`, `rebuild-set.md`, and -`rebuilds/-eb--cve-patches.yml`. The easystack contains -class `must-rebuild` only, including a dependent that `--check-static` -promoted. A second run replaces the `rebuilds/*.yml` files. +The output directory contains `rebuild-set.json`, `rebuild-set.md`, and `rebuilds/-eb--cve-patches.yml`. +The easystack contains class `must-rebuild` only, including a dependent that `--check-static` promoted. +A second run replaces the `rebuilds/*.yml` files. +`rebuild-set.json` and `rebuild-set.md` record the reach class of each patched easyconfig, including a filename listed under `missing`. +`--reachability` replaces or adds names for that map as well as for `backport.md`. +`cross-user` and `network-facing` justify a backport. # `component-reach` @@ -6690,26 +6765,25 @@ that carries a component, then feeds the recipes that contain it into the rebuild closure. See [which modules carry a component](https://eb-stack.rgoswami.me/../howto/component-reach.html.md). -Pass `--component` and `--version`, or `--purl`, or `--cve`. `--version` -is an exact version, `*`, or a requirement such as `>=8,<9`. With -`--purl` it replaces the version in the URL. The purl type selects the -carriers. `generic` and `github` match the easyconfig name. `pypi`, -`cargo`, `npm`, `golang`, and `cran` match an extension or a bundled -component of that language, plus a crate for `cargo` and a wheel for -`pypi`. Another type exits 2: the error says that purl type has no -carrier model. `pkg:cargo/curl` does not match the cURL recipe. -`--cve` resolves the id through Open Source Vulnerabilities (OSV), follows GitHub Security Advisory (GHSA), RUSTSEC, PYSEC, and -GO aliases when the Common Vulnerabilities and Exposures (CVE) record has only a git range, and keeps named -version tags such as `libXpm-3.5.17`. A GIT range is mapped onto -release versions: the affected entry's `versions` list when it has one, -and otherwise tags from the upstream repository (`git ls-remote --tags`, -then `git merge-base --is-ancestor` on a blobless fetch). That lookup is -cached and is skipped when it cannot reach the network, and the range -then stays under Unresolved ranges. An event that does not parse is -reported there too. A package whose ranges were all unparsed counts as -review. Several affected packages are written as one document with -`kind` `cve`. The summary line includes `changed`, `must-rebuild`, -`review`, and `no-rebuild`. +Pass `--component` and `--version`, or `--purl`, or `--cve`. +`--version` is an exact version, `*`, or a requirement such as `>=8,<9`. +With `--purl` it replaces the version in the URL. +The purl type selects the carriers. +`generic` and `github` match the easyconfig name. +`pypi`, `cargo`, `npm`, `golang`, and `cran` match an extension or a bundled component of that language, plus a crate for `cargo` and a wheel for `pypi`. +Another type exits 2: the error says that purl type has no carrier model. +`pkg:cargo/curl` does not match the cURL recipe. +`--cve` resolves the id through Open Source Vulnerabilities (OSV). +When the Common Vulnerabilities and Exposures (CVE) record has only a git range, it follows GitHub Security Advisory (GHSA), `RUSTSEC`, `PYSEC`, and `GO` aliases. +It keeps named version tags such as `libXpm-3.5.17`. +A `GIT` range is mapped onto release versions: the affected entry's `versions` list when it has one, and otherwise tags from the upstream repository (`git ls-remote --tags`, then `git merge-base --is-ancestor` on a blobless fetch). +That lookup is cached and is skipped when it cannot reach the network, and the range then stays under Unresolved ranges. +An event that does not parse is reported there too. +A package whose ranges were all unparsed counts as review. +An advisory that affects more than one package is written as one document with `kind` `cve`. +The summary line includes `reach`, `changed`, `must-rebuild`, `review`, and `no-rebuild`. +`reach` is the class of the component name: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified`. +A `--cve` summary has no single class. Names match ignoring ASCII letter case, on a whole token. A recipe is in the generation when its toolchain version equals `--generation`, or when the @@ -8582,13 +8656,14 @@ filename carried a version, the file is `unknown`. | Field | Meaning | | --- | --- | -| `schema_version` | `4` | +| `schema_version` | `5` | | `generated_at` | RFC 3339 timestamp of the run | | `installed_source` | Label of the installed list | | `prefix` | Installed prefix passed to `--prefix`. Omitted when the flag was not passed | | `symbols_checked` | `true` when `--check-static` ran | | `easystacks` | Easystack paths that were read, sorted | | `changed` | Patched easyconfig filenames, sorted | +| `reach` | Reach class of each filename in `changed`, including missing ones. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` means neither the built-in names nor `--reachability` names the package | | `missing` | Changed filenames that are not in the deployed graph | | `unresolved` | Requirements no easyconfig satisfies, and easystack entries the robot path does not have | | `notes` | A missing easyblock class, a `from-commit` whose tree was not given, or an install directory that was not found. Empty when there is nothing to say | @@ -8602,8 +8677,8 @@ when an easystack filename carried one. On a review row, `changed` is the patched filenames joined by `, `. After `--check-static`, a hit keeps only the filenames whose exports were defined. -`rebuild-set.md` is the same rows as a table, the counts line, the missing -filenames, and each note as `- note:`. +`rebuild-set.md` is the same rows as a table, the counts line, the missing filenames, and each note as `- note:`. +It ends with the reach class of each patched easyconfig. ## `findings.json`, `findings.md`, `findings.csv` @@ -8722,7 +8797,7 @@ One run writes `component-reach.toml`, `component-reach.json`, and | Field | Meaning | | --- | --- | -| `schema_version` | `1` | +| `schema_version` | `2` | | `generated_at` | RFC 3339 timestamp of the run | | `component` | Name that was asked for | | `version_req` | Exact version, `*`, or a requirement. Empty when the match is only `versions` | @@ -8731,6 +8806,7 @@ One run writes `component-reach.toml`, `component-reach.json`, and | `purl` | Package URL, when the query was a purl or a resolved CVE | | `ecosystem` | Ecosystem from the purl or the Open Source Vulnerabilities (OSV) record | | `generation` | Toolchain generation | +| `reach` | Class of the component name: `cross-user`, `network-facing`, `local-only`, `not-reachable`, or `unclassified` | | `carriers` | Easyconfigs and modules that carry it | | `changed` | Filenames that contain it, fed to the closure | | `closure_counts` | Row counts, keys `no-rebuild`, `review`, `must-rebuild` | @@ -8748,18 +8824,43 @@ compat-layer and installed-only rows), `module`, and `installed` when a list was passed. A `--cve` run writes the same three filenames with a different document. -`kind` is `cve`. `cve` is the id. `resolved` is the packages OSV named -(`ecosystem`, `name`, `purl`, `version_reqs`, `versions`). `results` is -one component-reach document per package. `notes` contains alias fetches -that failed. Schema version stays `1`. +`kind` is `cve`. +`cve` is the id. +`resolved` is the packages the Open Source Vulnerabilities (OSV) record named (`ecosystem`, `name`, `purl`, `version_reqs`, `versions`). +`results` is one component-reach document per package. +`notes` contains alias fetches that failed. +Schema version stays `2`. +Each package document carries its own `reach`. A closure row has the rebuild-set fields `changed`, `dependent`, `dependent_easyconfig`, `edge_kind`, `class`, `reason`, and `eb_version` when an easystack filename carried one. With no easystack the synthetic stack is `generation-.yml` and `eb_version` is absent. -`component-reach.md` is the carrier table, then the closure table, the -counts line, and each note as `- note:`. +`component-reach.md` is the carrier table, then the closure table, the counts line, and each note as `- note:`. +The carriers line names the reach class. + +# Backport list (`eb-stack stack sbom --backport-list`) + +A path ending in `.md` writes that markdown and a sibling `.json`. + +| Field | Meaning | +| --- | --- | +| `schema_version` | `1` | +| `lock` | Lock path, as it was passed | +| `generation` | Toolchain label of the lock | +| `advisories` | Advisory ids that were resolved, sorted | +| `stack_modules` | Packages in the lock | +| `listed` | Rows whose reach class justifies a backport | +| `recorded` | Rows left off the list | +| `rows` | One entry per locked module an advisory matched. On-list rows come first | +| `notes` | An advisory names a package the lock lacks, a locked version is outside the range, or a range did not parse | + +Each row has `module` (`Name/version-toolchain`, plus the versionsuffix when the lock has one), `name`, `version`, `reach`, `advisory`, `fixed_in` (upper bounds from `<` clauses; a `<=` bound is last-affected and is not a fix), `blast_radius`, `blast` (those module labels, sorted), and `on_list`. +Blast radius counts the module and every locked module that depends on it through a runtime or build edge. + +The markdown lists each module in a backport section or a recorded section. +The recorded sentence is: these modules stay off the backport list, and a backport is cross-user or network-facing. # From recipe to verified package @@ -10762,12 +10863,40 @@ sed -n '27,29p' /tmp/sbom-demo/rebuild/rebuild-set.md | `git-2.52.0-GCCcore-15.2.0.eb` | `git/2.52.0-GCCcore-15.2.0` | changed | must-rebuild | patched easyconfig | ``` +```bash +grep 'Reach class' -A 2 /tmp/sbom-demo/rebuild/rebuild-set.md +``` + +```text +Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` is not a judgment. + +- `git-2.52.0-GCCcore-15.2.0.eb`: network-facing +``` + The report has one row: the changed git easyconfig, class must-rebuild. -The easystack entries do not depend on that git module, so the walk does -not add a second row. The report also notes each `from-commit` pin whose -tree was not passed and says the release tree was used for that pin. -`--check-static` was not passed, and the report says so. Classes follow -the easyconfig graph only. +The easystack entries do not depend on that git module, so the walk does not add a second row. +The report also notes each `from-commit` pin whose tree was not passed and says the release tree was used for that pin. +`--check-static` was not passed, and the report says so. +Classes follow the easyconfig graph only. git is `network-facing`, so this patched easyconfig would justify a backport. + +A stack lock has the same classes. +`stack sbom --backport-list` reads recorded Open Source Vulnerabilities (OSV) responses and the software bill of materials (SBOM) reverse closure. +`cross-user` and `network-facing` are the list. +The fixture lock has cURL (network-facing, blast radius 2 because git depends on it) and notebook (cross-user). zlib is `local-only` in the site file, so it is recorded and left off the list. + +```bash +eb-stack stack sbom \ + --lock tests/backport_fixtures/stack.lock.json \ + --out /tmp/sbom-demo/backport.cdx.json \ + --backport-list /tmp/sbom-demo/backport.md \ + --osv-cache tests/backport_fixtures/osv \ + --reachability tests/backport_fixtures/reach.txt +``` + +```text +sbom=/tmp/sbom-demo/backport.cdx.json components=7 recipes_read=7 recipes_unreadable=0 with_artifact_facts=0 +backport=2 recorded=3 stack=7 out=/tmp/sbom-demo/backport.md +``` # A component named by a CVE @@ -10790,7 +10919,7 @@ eb-stack component-reach \ ``` ```text -component=curl version=>=8,<9 generation=2026.1 carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl +component=curl version=>=8,<9 generation=2026.1 reach=network-facing carriers=2 changed=1 must-rebuild=1 review=0 no-rebuild=1 out=/tmp/sbom-demo/component-curl ``` ```bash @@ -10798,7 +10927,7 @@ sed -n '5,16p' /tmp/sbom-demo/component-curl/component-reach.md ``` ```text -2 carriers. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. +2 carriers. Reach: `network-facing`. Closure: 1 must-rebuild, 0 review, 1 no-rebuild. | Easyconfig | Module | Role | Version | Installed | |---|---|---|---|---| @@ -10981,10 +11110,10 @@ eb_easyblock_reach eb_repro_summary ``` -That is 50 tools. `eb_stack_sbom` writes the CycloneDX file `stack sbom` -writes. Its `tools/list` description reads: "Emit CycloneDX from an existing -stack lock, reading dependency edges and source checksums from the -easyconfigs the lock names." `eb_stack_sbom` reads the Cbc lock: +That is 50 tools. +`eb_stack_sbom` writes the CycloneDX file `stack sbom` writes. +Its `tools/list` description reads: "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names. With backport_list, also write the modules whose reach class justifies a backport." +`eb_stack_sbom` reads the Cbc lock: ```bash printf '%s\n' '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"eb_stack_sbom","arguments":{"lock":"/tmp/sbom-demo/stack.lock.json","out":"/tmp/sbom-demo/mcp-stack.cdx.json"}}}' \ diff --git a/skills/mcp/SKILL.md b/skills/mcp/SKILL.md index 6e50fbda..57829367 100644 --- a/skills/mcp/SKILL.md +++ b/skills/mcp/SKILL.md @@ -70,7 +70,15 @@ proc.wait(timeout=5) PY ``` -`tools/list` returns the catalog. This page does not keep a count. Read the names from that response. The names that mirror the commands above are `eb_package_inspect`, `eb_package_plan`, `eb_package_bump`, `eb_stack_sbom`, and `eb_pr_overlap`. The input schema requires `toolchain` as `name/version` for inspect, plan, and bump. Inspect and plan also accept `toolchain_version` alone and default the name to `foss`. Bump does not: it requires `toolchain`, or both `toolchain_name` and `toolchain_version`. `eb_stack_sbom` takes `lock` and optional `out`. `eb_pr_overlap` takes `pr` as an array of paths and `easyconfigs` as an array of directories. +`tools/list` returns the catalog. +This page does not keep a count. +Read the names from that response. +The names that mirror the commands above are `eb_package_inspect`, `eb_package_plan`, `eb_package_bump`, `eb_stack_sbom`, and `eb_pr_overlap`. +Each package tool's input schema requires `toolchain` as `name/version`. +Inspect and plan also accept `toolchain_version` alone and default the name to `foss`. +Bump does not: it requires `toolchain`, or both `toolchain_name` and `toolchain_version`. +`eb_stack_sbom` takes `lock` and optional `out`, `backport_list`, `cve`, `osv_cache`, and `reachability`. +`eb_pr_overlap` takes `pr` as an array of paths and `easyconfigs` as an array of directories. `eb_build_test` runs only when `EB_STACK_ALLOW_BUILD=1`. diff --git a/src/backport.rs b/src/backport.rs new file mode 100644 index 00000000..cf6f2202 --- /dev/null +++ b/src/backport.rs @@ -0,0 +1,459 @@ +//! Backport list for one stack lock. +//! +//! A locked module is on the list when an advisory matches its version and +//! its reach class is `cross-user` or `network-facing`. `local-only`, +//! `not-reachable`, and `unclassified` are recorded and left off the list. +//! Blast radius is the SBOM closure: the module and every locked module that +//! depends on it through a runtime or build edge. + +use crate::cli_parse::usage; +use crate::domain::{LockPackage, StackLock}; +use crate::findings::{ + justifies_backport, reach_key, ReachTable, REACH_CROSS_USER, REACH_LOCAL, REACH_NETWORK, + REACH_NOT_REACHABLE, +}; +use crate::osv::{load_recorded_osv, resolve_vulnerability, ResolvedComponent}; +use crate::registry::{RegistryClient, UreqClient}; +use crate::sbom::{closure_edges_for_lock, package_identity}; +use crate::version::matches_req; +use anyhow::{anyhow, bail, Context, Result}; +use serde::Serialize; +use std::collections::{BTreeSet, HashMap, VecDeque}; +use std::path::Path; + +/// Schema of [`BackportDocument`]. +pub const BACKPORT_LIST_SCHEMA_VERSION: u32 = 1; + +/// What `stack sbom --backport-list` reads. +pub struct BackportRequest<'a> { + /// Stack lock. + pub lock: &'a StackLock, + /// Path the lock was read from. Relative recipe paths resolve beside it. + pub lock_path: &'a Path, + /// Advisory ids. Empty means every id in [`Self::osv_cache`]. + pub cves: &'a [String], + /// Recorded OSV responses. Absent means a live fetch of [`Self::cves`]. + pub osv_cache: Option<&'a Path>, + /// Site file of `name class` lines. Absent uses the built-in names. + pub reachability: Option<&'a Path>, +} + +/// One locked module an advisory matched. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackportRow { + /// `Name/version-toolchain`, plus the versionsuffix when the lock has one. + pub module: String, + /// Package name. + pub name: String, + /// Locked version. + pub version: String, + /// Reach class. + pub reach: String, + /// Advisory id. + pub advisory: String, + /// Upper bounds from a less-than clause. A less-than-or-equal bound is last-affected, not a fix. + pub fixed_in: Vec, + /// The module plus every locked module that depends on it. + pub blast_radius: usize, + /// Those module labels, sorted. + pub blast: Vec, + /// `true` when [`justifies_backport`] is true for [`Self::reach`]. + pub on_list: bool, +} + +/// The backport list for one lock. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackportDocument { + /// [`BACKPORT_LIST_SCHEMA_VERSION`]. + pub schema_version: u32, + /// Lock path, as it was passed. + pub lock: String, + /// Toolchain label of the lock. + pub generation: String, + /// Advisory ids that were resolved, sorted. + pub advisories: Vec, + /// Packages in the lock. + pub stack_modules: usize, + /// Rows with [`BackportRow::on_list`]. + pub listed: usize, + /// Rows left off the list. + pub recorded: usize, + /// Matching modules. On-list rows come first. + pub rows: Vec, + /// Advisories that name a package the lock lacks, a version outside the + /// range, or a range that did not parse. + pub notes: Vec, +} + +/// Resolve the advisories and classify every matching locked module. +pub fn backport_list(request: &BackportRequest<'_>) -> Result { + let ids = advisory_ids(request.cves, request.osv_cache)?; + let resolutions = resolve_ids(&ids, request.osv_cache)?; + let reach = load_reach(request.reachability)?; + let blasts = blast_sets(request.lock, request.lock_path); + let mut rows = Vec::new(); + let mut notes = BTreeSet::new(); + for resolution in &resolutions { + for component in &resolution.components { + classify_component( + request.lock, + &resolution.id, + component, + &reach, + &blasts, + &mut rows, + &mut notes, + ); + } + for item in &resolution.unresolved { + notes.insert(format!("{}: {item}", resolution.id)); + } + } + rows.sort_by(|left, right| { + right + .on_list + .cmp(&left.on_list) + .then(rank(&left.reach).cmp(&rank(&right.reach))) + .then(right.blast_radius.cmp(&left.blast_radius)) + .then(left.module.cmp(&right.module)) + .then(left.advisory.cmp(&right.advisory)) + }); + let listed = rows.iter().filter(|row| row.on_list).count(); + Ok(BackportDocument { + schema_version: BACKPORT_LIST_SCHEMA_VERSION, + lock: request.lock_path.display().to_string(), + generation: generation_of(request.lock), + advisories: ids, + stack_modules: request.lock.packages.len(), + listed, + recorded: rows.len() - listed, + rows, + notes: notes.into_iter().collect(), + }) +} + +/// Markdown for [`BackportDocument`]. +pub fn render_markdown(document: &BackportDocument) -> String { + let mut out = String::from("# Backport list\n\n"); + out.push_str(&format!( + "Lock `{}`. Generation `{}`. {} modules.\n\n", + document.lock, document.generation, document.stack_modules + )); + out.push_str( + "A module is on this list when its reach class is `cross-user` or \ + `network-facing`. Those two classes justify a backport. Blast radius \ + is the SBOM closure: the module and every locked module that depends \ + on it through a runtime or build edge.\n\n", + ); + if !document.advisories.is_empty() { + let ids = document + .advisories + .iter() + .map(|id| format!("`{id}`")) + .collect::>() + .join(", "); + out.push_str(&format!("Advisories: {ids}.\n\n")); + } + out.push_str("## Backport\n\n"); + write_rows( + &mut out, + document.rows.iter().filter(|row| row.on_list), + "No module on this lock is `cross-user` or `network-facing`.\n", + ); + out.push_str("\n## Recorded\n\n"); + out.push_str( + "These modules stay off the backport list. A backport is cross-user or network-facing.\n\n", + ); + write_rows( + &mut out, + document.rows.iter().filter(|row| !row.on_list), + "No module was recorded off the list.\n", + ); + if !document.notes.is_empty() { + out.push_str("\n## Notes\n\n"); + for note in &document.notes { + out.push_str(&format!("- {note}\n")); + } + } + out +} + +/// Write the markdown. A path ending in `.md` also gets a sibling `.json`. +pub fn write_backport(path: &Path, document: &BackportDocument) -> Result<()> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create {}", parent.display()))?; + } + } + std::fs::write(path, render_markdown(document)) + .with_context(|| format!("write {}", path.display()))?; + if path.extension().and_then(|ext| ext.to_str()) == Some("md") { + crate::write_json_pretty(&path.with_extension("json"), document)?; + } + Ok(()) +} + +fn write_rows<'a>(out: &mut String, rows: impl Iterator, empty: &str) { + let rows: Vec<&BackportRow> = rows.collect(); + if rows.is_empty() { + out.push_str(empty); + return; + } + out.push_str("| Module | Reach | Advisory | Fixed in | Blast radius |\n"); + out.push_str("|---|---|---|---|---|\n"); + for row in rows { + let fixed = if row.fixed_in.is_empty() { + "-".to_string() + } else { + row.fixed_in.join(", ") + }; + out.push_str(&format!( + "| `{}` | {} | {} | {} | {} |\n", + row.module, row.reach, row.advisory, fixed, row.blast_radius + )); + } +} + +fn classify_component( + lock: &StackLock, + id: &str, + component: &ResolvedComponent, + reach: &ReachTable, + blasts: &HashMap>, + rows: &mut Vec, + notes: &mut BTreeSet, +) { + let named: Vec<&LockPackage> = lock + .packages + .iter() + .filter(|package| reach_key(&package.name) == reach_key(&component.name)) + .collect(); + if named.is_empty() { + notes.insert(format!( + "{id} names {}, which this lock does not have", + component.name + )); + return; + } + if component.version_reqs.is_empty() && component.versions.is_empty() { + notes.insert(format!( + "{id} names {}, but its ranges were not parsed", + component.name + )); + return; + } + let hits: Vec<&LockPackage> = named + .iter() + .copied() + .filter(|package| version_matches(package, component)) + .collect(); + if hits.is_empty() { + let present = named + .iter() + .map(|package| module_label(package)) + .collect::>() + .join(", "); + let verb = if named.len() == 1 { "is" } else { "are" }; + notes.insert(format!( + "{present} {verb} not in range for {id} ({})", + range_detail(component) + )); + return; + } + let fixed_in = fixed_bounds(&component.version_reqs); + for package in hits { + let identity = package_identity(package); + let blast = blasts.get(&identity).cloned().unwrap_or_default(); + let class = reach.class_of(&package.name, &[&component.name, &package.name]); + rows.push(BackportRow { + module: module_label(package), + name: package.name.clone(), + version: package.version.clone(), + reach: class.clone(), + advisory: id.to_string(), + fixed_in: fixed_in.clone(), + blast_radius: blast.len().max(1), + blast: blast.into_iter().collect(), + on_list: justifies_backport(&class), + }); + } +} + +fn version_matches(package: &LockPackage, component: &ResolvedComponent) -> bool { + component + .versions + .iter() + .any(|version| version == &package.version) + || component + .version_reqs + .iter() + .any(|req| matches_req(&package.version, req)) +} + +fn range_detail(component: &ResolvedComponent) -> String { + let mut parts = component.version_reqs.clone(); + for version in &component.versions { + parts.push(format!("version {version}")); + } + parts.join(" or ") +} + +/// Upper bounds introduced by a less-than clause. Less-than-or-equal is last-affected, not a fix. +fn fixed_bounds(reqs: &[String]) -> Vec { + let mut out = Vec::new(); + for req in reqs { + for clause in req.split(',') { + let Some(bound) = clause.trim().strip_prefix('<') else { + continue; + }; + if bound.is_empty() || bound.starts_with('=') { + continue; + } + out.push(bound.to_string()); + } + } + out.sort(); + out.dedup(); + out +} + +fn rank(class: &str) -> u8 { + match class { + REACH_CROSS_USER => 0, + REACH_NETWORK => 1, + REACH_LOCAL => 2, + REACH_NOT_REACHABLE => 3, + _ => 4, + } +} + +fn generation_of(lock: &StackLock) -> String { + lock.generation_label + .clone() + .filter(|label| !label.is_empty()) + .unwrap_or_else(|| lock.toolchain.label()) +} + +fn module_label(package: &LockPackage) -> String { + let mut label = format!( + "{}/{}-{}", + package.name, + package.version, + package.toolchain.label() + ); + if let Some(suffix) = package + .versionsuffix + .as_deref() + .filter(|suffix| !suffix.is_empty()) + { + label.push_str(suffix); + } + label +} + +fn advisory_ids(cves: &[String], cache: Option<&Path>) -> Result> { + if !cves.is_empty() { + let mut ids = cves.to_vec(); + ids.sort(); + ids.dedup(); + return Ok(ids); + } + let Some(dir) = cache else { + bail!("--backport-list needs --cve or --osv-cache"); + }; + let manifest_path = dir.join("queries.json"); + let text = std::fs::read_to_string(&manifest_path) + .with_context(|| format!("read {}", manifest_path.display()))?; + let manifest: serde_json::Value = serde_json::from_str(&text) + .with_context(|| format!("parse {}", manifest_path.display()))?; + let queries = manifest + .get("queries") + .and_then(|value| value.as_array()) + .ok_or_else(|| anyhow!("{} has no queries", manifest_path.display()))?; + let mut ids = Vec::new(); + for query in queries { + let url = query + .get("url") + .and_then(|value| value.as_str()) + .ok_or_else(|| anyhow!("{} query has no url", manifest_path.display()))?; + let id = url + .rsplit('/') + .next() + .filter(|id| !id.is_empty()) + .ok_or_else(|| anyhow!("{url} has no advisory id"))?; + ids.push(id.to_string()); + } + if ids.is_empty() { + bail!("{} names no advisory", manifest_path.display()); + } + ids.sort(); + ids.dedup(); + Ok(ids) +} + +fn resolve_ids(ids: &[String], cache: Option<&Path>) -> Result> { + let recorded = match cache { + Some(dir) => Some(load_recorded_osv(dir).map_err(|error| anyhow!(error))?), + None => None, + }; + let live = UreqClient; + let client: &dyn RegistryClient = match &recorded { + Some(map) => map, + None => &live, + }; + let mut out = Vec::new(); + for id in ids { + out.push(resolve_vulnerability(id, client).map_err(|error| anyhow!(error))?); + } + Ok(out) +} + +fn load_reach(path: Option<&Path>) -> Result { + let Some(path) = path else { + return Ok(ReachTable::builtin()); + }; + let text = std::fs::read_to_string(path) + .with_context(|| format!("--reachability {}: not readable", path.display()))?; + ReachTable::builtin() + .overlay(&text, path.display().to_string()) + .map_err(|error| usage(format!("--reachability {}: {error}", path.display()))) +} + +fn blast_sets(lock: &StackLock, lock_path: &Path) -> HashMap> { + let edges = closure_edges_for_lock(lock, lock_path); + let mut reverse: HashMap<&str, Vec<&str>> = HashMap::new(); + for (from, deps) in &edges { + for dep in deps { + reverse.entry(dep.as_str()).or_default().push(from.as_str()); + } + } + let labels: HashMap = lock + .packages + .iter() + .map(|package| (package_identity(package), module_label(package))) + .collect(); + let mut out = HashMap::new(); + for package in &lock.packages { + let root = package_identity(package); + let mut seen = BTreeSet::new(); + let mut queue = VecDeque::new(); + queue.push_back(root.as_str()); + seen.insert(root.clone()); + while let Some(node) = queue.pop_front() { + if let Some(next) = reverse.get(node) { + for dependent in next { + if seen.insert((*dependent).to_string()) { + queue.push_back(dependent); + } + } + } + } + let modules = seen + .into_iter() + .map(|id| labels.get(&id).cloned().unwrap_or(id)) + .collect(); + out.insert(root, modules); + } + out +} diff --git a/src/component_reach.rs b/src/component_reach.rs index 49239bae..05e8dd78 100644 --- a/src/component_reach.rs +++ b/src/component_reach.rs @@ -15,7 +15,11 @@ use serde::Serialize; use std::collections::BTreeSet; /// Schema of [`ReachDocument`]. -pub const COMPONENT_REACH_SCHEMA_VERSION: u32 = 1; +/// +/// Version 2 adds `reach`: `cross-user`, `network-facing`, `local-only`, +/// `not-reachable`, or `unclassified` when the built-in names do not include +/// the component. `component-reach` reads no site file. +pub const COMPONENT_REACH_SCHEMA_VERSION: u32 = 2; /// One run. `version_req` is an exact version, `*`, or a requirement. pub struct ReachQuery<'a> { @@ -93,6 +97,12 @@ pub struct ReachDocument { pub ecosystem: Option, /// Toolchain generation. pub generation: String, + /// Reach class of the component name. + /// + /// `cross-user` and `network-facing` justify a backport. `local-only` + /// and `not-reachable` do not. `unclassified` means the built-in names + /// do not include this component. + pub reach: String, /// Who carries it. pub carriers: Vec, /// Recipes that contain the component, fed to the closure. @@ -201,6 +211,8 @@ pub fn component_reach(query: &ReachQuery<'_>) -> ReachDocument { closure_counts.review = 1; } notes.extend(extra); + let reach = + crate::findings::ReachTable::builtin().class_of(query.component, &[query.component]); let mut version_reqs = Vec::new(); if !query.version_req.is_empty() { version_reqs.push(query.version_req.to_string()); @@ -220,6 +232,7 @@ pub fn component_reach(query: &ReachQuery<'_>) -> ReachDocument { purl: query.purl.map(str::to_string), ecosystem: query.ecosystem.map(str::to_string), generation: query.generation.to_string(), + reach, carriers, changed, closure_counts, @@ -242,11 +255,12 @@ pub fn render_markdown(document: &ReachDocument) -> String { "*".to_string() }; out.push_str(&format!( - "Computed {} for `{}` `{requirement}` in generation `{}`.\n\n{} carriers. Closure: {} must-rebuild, {} review, {} no-rebuild.\n\n", + "Computed {} for `{}` `{requirement}` in generation `{}`.\n\n{} carriers. Reach: `{}`. Closure: {} must-rebuild, {} review, {} no-rebuild.\n\n", document.generated_at, document.component, document.generation, document.carriers.len(), + document.reach, document.closure_counts.must_rebuild, document.closure_counts.review, document.closure_counts.no_rebuild @@ -1170,7 +1184,11 @@ mod tests { .any(|row| row.dependent.starts_with("git/") && row.class == "no-rebuild")); let rendered = to_toml(&curl).unwrap(); assert!(rendered.starts_with(HEADER)); + assert_eq!(curl.reach, "network-facing"); + assert!(render_markdown(&curl).contains("Reach: `network-facing`")); assert!(render_markdown(&curl).contains("no-rebuild")); + assert_eq!(run("jupyterlab").reach, "cross-user"); + assert_eq!(run("zlib").reach, "unclassified"); assert!(run("urllib3") .carriers .iter() diff --git a/src/easystack.rs b/src/easystack.rs index 2a04fe72..fe671ef2 100644 --- a/src/easystack.rs +++ b/src/easystack.rs @@ -215,6 +215,57 @@ pub struct EessiLayout { pub members: Vec, } +/// Toolchain names a foss solve cannot see. +/// +/// The LLVM family is `lfoss`, `lompi`, `lfbf`, and `llvm-compilers`. +/// OpenMPI under `llvm-compilers-21.1.8` is in that family. A foss solve's +/// universe is the foss hierarchy, so a package that exists only on one of +/// these toolchains is absent from the foss lock and a later diff marks it +/// site layer. `data/eessi-generations.toml` lists `llvm-compilers-20.1.8` +/// (generation 2025b) and `llvm-compilers-21.1.8` (generation 2026.1) by +/// hand, because an LLVM release is not a generation name. +pub const LLVM_FAMILY: &[&str] = &["lfoss", "lompi", "lfbf", "llvm-compilers"]; + +/// The LLVM family `label` names, or `None` when it is some other toolchain. +/// +/// `label` is a toolchain name (`lfoss`) or a `name-version` label whose +/// version starts with a digit (`llvm-compilers-21.1.8`, `lfoss-2026.1`). +/// The longest family name wins, so `llvm-compilers` is not cut at a shorter +/// prefix. +pub fn llvm_family_name(label: &str) -> Option<&'static str> { + LLVM_FAMILY + .iter() + .copied() + .filter(|family| llvm_family_version_rest(label, family).is_some()) + .max_by_key(|family| family.len()) +} + +/// The version after `family` in `label`, when `label` is that family. +/// +/// `llvm-compilers-21.1.8` and `llvm-compilers` yield `Some("21.1.8")` and +/// `Some("")`. A label that is only the family name has an empty version. +pub fn llvm_family_version(label: &str, family: &str) -> Option { + let rest = llvm_family_version_rest(label, family)?; + Some(rest.to_string()) +} + +/// `Some` when `label` is `family` or `family-` with a digit version. +/// The returned string is the version, empty when `label` is the bare name. +fn llvm_family_version_rest<'a>(label: &'a str, family: &str) -> Option<&'a str> { + if label.eq_ignore_ascii_case(family) { + return Some(""); + } + if label.len() <= family.len() || !label[..family.len()].eq_ignore_ascii_case(family) { + return None; + } + let rest = label[family.len()..].strip_prefix('-')?; + if rest.starts_with(|c: char| c.is_ascii_digit()) { + Some(rest) + } else { + None + } +} + /// Toolchain names whose presence in a filename means it is not a system /// recipe, followed by the toolchain version up to the next dash. static TOOLCHAIN_IN_FILENAME: LazyLock = LazyLock::new(|| { @@ -408,6 +459,31 @@ mod tests { assert_eq!(split("foss-2025b.eb"), None); } + #[test] + fn llvm_family_name_takes_the_longest_family_and_a_digit_version() { + assert_eq!(llvm_family_name("lfoss"), Some("lfoss")); + assert_eq!(llvm_family_name("LFOSS-2026.1"), Some("lfoss")); + assert_eq!(llvm_family_name("lompi-2025b"), Some("lompi")); + assert_eq!(llvm_family_name("lfbf"), Some("lfbf")); + assert_eq!( + llvm_family_name("llvm-compilers-21.1.8"), + Some("llvm-compilers") + ); + assert_eq!( + llvm_family_version("llvm-compilers-21.1.8", "llvm-compilers").as_deref(), + Some("21.1.8") + ); + assert_eq!( + llvm_family_version("lfoss-2025b", "lfoss").as_deref(), + Some("2025b") + ); + assert_eq!(llvm_family_version("lfoss", "lfoss").as_deref(), Some("")); + assert_eq!(llvm_family_name("foss-2025b"), None); + assert_eq!(llvm_family_name("GCCcore-15.2.0"), None); + assert_eq!(llvm_family_name("lfoss-v2025b"), None); + assert_eq!(llvm_family_version("foss-2025b", "lfoss"), None); + } + #[test] fn eessi_easystacks_split_by_file_and_keep_options() { let mut options = EasystackOptions::new(); diff --git a/src/findings.rs b/src/findings.rs index 29461caa..f157645e 100644 --- a/src/findings.rs +++ b/src/findings.rs @@ -233,6 +233,28 @@ impl ReachTable { } } +/// Reach class of each easyconfig filename. +/// +/// The name is the toolchain split, then the same fallback as +/// [`crate::gap::software_name`]. The class is [`ReachTable::class_of`] on +/// that name. Filenames stay in the order `files` gives them only as map +/// keys; a [`std::collections::BTreeMap`] sorts them. +pub fn reach_for_easyconfigs(table: &ReachTable, files: &[String]) -> BTreeMap { + files + .iter() + .map(|file| { + let owned = crate::gap::software_name(file).unwrap_or_else(|| { + file.rsplit(['/', '\\']) + .next() + .unwrap_or(file.as_str()) + .to_string() + }); + let class = table.class_of(&owned, &[owned.as_str()]); + (file.clone(), class) + }) + .collect() +} + fn parse_reachability(text: &str) -> Result, ReachabilityError> { let mut site = BTreeMap::new(); for (index, raw) in text.lines().enumerate() { @@ -1778,6 +1800,7 @@ Medium,\"GHSA-a,b\",\"py \"\"x\"\"\",2.0,2.1,not-fixed,PyApp/1.0-GCCcore-15.2.0\ must_rebuild: 0, }, rows, + reach: BTreeMap::new(), } } diff --git a/src/fix_cost.rs b/src/fix_cost.rs index b7c920b0..4e2719fc 100644 --- a/src/fix_cost.rs +++ b/src/fix_cost.rs @@ -1457,6 +1457,7 @@ mod tests { review: 0, must_rebuild: 0, }, + reach: std::collections::BTreeMap::new(), rows, } } diff --git a/src/generation_lint.rs b/src/generation_lint.rs index d5c0a78c..2e5ed100 100644 --- a/src/generation_lint.rs +++ b/src/generation_lint.rs @@ -633,6 +633,156 @@ pub fn run_generation_lint(request: &GenerationLintRequest) -> Result Vec { + if crate::easystack::llvm_family_name(&policy.name).is_some() { + return Vec::new(); + } + let suppressed: BTreeSet<&str> = also + .iter() + .filter_map(|tc| crate::easystack::llvm_family_name(&tc.name)) + .collect(); + let mut added_by_name: BTreeMap)> = BTreeMap::new(); + for (name, path) in added { + let entry = added_by_name + .entry(name.to_ascii_lowercase()) + .or_insert_with(|| (name.clone(), BTreeSet::new())); + if !path.is_empty() { + entry.1.insert(path.clone()); + } + } + + struct Group { + name: String, + family: &'static str, + versions: BTreeSet, + files: BTreeSet, + from_easystack: bool, + solved: BTreeSet, + } + + let mut groups: BTreeMap<(String, &'static str), Group> = BTreeMap::new(); + for file in easystack_files { + let Some(split) = filename_toolchain(file) else { + continue; + }; + let Some(family) = crate::easystack::llvm_family_name(&split.toolchain) else { + continue; + }; + if suppressed.contains(family) { + continue; + } + let Some((display, solved)) = added_by_name.get(&split.name.to_ascii_lowercase()) else { + continue; + }; + let key = (split.name.to_ascii_lowercase(), family); + let group = groups.entry(key).or_insert_with(|| Group { + name: display.clone(), + family, + versions: BTreeSet::new(), + files: BTreeSet::new(), + from_easystack: true, + solved: solved.clone(), + }); + let base = file.rsplit(['/', '\\']).next().unwrap_or(file.as_str()); + group.files.insert(base.to_string()); + if let Some(version) = crate::easystack::llvm_family_version(&split.toolchain, family) { + if !version.is_empty() { + group.versions.insert(version); + } + } + } + for (name, path) in added { + if path.is_empty() { + continue; + } + let Some(split) = filename_toolchain(path) else { + continue; + }; + let Some(family) = crate::easystack::llvm_family_name(&split.toolchain) else { + continue; + }; + if suppressed.contains(family) { + continue; + } + let key = (name.to_ascii_lowercase(), family); + if groups.contains_key(&key) { + continue; + } + let group = groups.entry(key).or_insert_with(|| Group { + name: name.clone(), + family, + versions: BTreeSet::new(), + files: BTreeSet::new(), + from_easystack: false, + solved: BTreeSet::new(), + }); + group.solved.insert(path.clone()); + if let Some(version) = crate::easystack::llvm_family_version(&split.toolchain, family) { + if !version.is_empty() { + group.versions.insert(version); + } + } + } + + groups + .into_values() + .map(|group| { + let version = if group.versions.len() == 1 { + group.versions.iter().next().expect("one version").clone() + } else { + "".to_string() + }; + let flag = format!("{family}/{version}", family = group.family); + if group.from_easystack { + let files = group.files.into_iter().collect::>().join(", "); + let solved = if group.solved.is_empty() { + group.name.clone() + } else { + format!( + "{name} ({paths})", + name = group.name, + paths = group.solved.into_iter().collect::>().join(", ") + ) + }; + format!( + "{name} ({files}) is the LLVM family ({family}). A {policy} solve cannot reach it, so the site layer includes {solved}. Keep one lock per toolchain family: solve {family} on its own, or pass --baseline-also-toolchain {flag}.", + name = group.name, + family = group.family, + policy = policy.name, + ) + } else { + let solved = group.solved.into_iter().collect::>().join(", "); + format!( + "{name} ({solved}) is the LLVM family ({family}). This {policy} solve marked it site layer. Keep one lock per toolchain family: solve {family} on its own, or pass --baseline-also-toolchain {flag}.", + name = group.name, + family = group.family, + policy = policy.name, + ) + } + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -893,4 +1043,98 @@ mod tests { Some("2025b") ); } + + fn foss() -> Toolchain { + Toolchain { + name: "foss".into(), + version: "2025b".into(), + } + } + + #[test] + fn a_foss_solve_warns_when_an_lfoss_easystack_entry_is_site_layer() { + let added = [( + "GROMACS".into(), + "fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb" + .into(), + )]; + let files = ["GROMACS-2025.0-lfoss-2025b.eb".to_string()]; + let warnings = llvm_site_layer_warnings(&foss(), &[], &files, &added); + assert_eq!( + warnings, + vec![ + "GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS (fixtures/gromacs_2025_to_next/easyconfigs/foss-2025b/GROMACS-2025.0-foss-2025b.eb). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b.".to_string() + ] + ); + } + + #[test] + fn an_also_toolchain_or_an_llvm_policy_suppresses_the_site_layer_warning() { + let added = [( + "OpenMPI".into(), + "easyconfigs/llvm-compilers-21.1.8/OpenMPI-5.0.7-llvm-compilers-21.1.8.eb".into(), + )]; + let files = ["OpenMPI-5.0.7-llvm-compilers-21.1.8.eb".to_string()]; + let also = [Toolchain { + name: "llvm-compilers".into(), + version: "21.1.8".into(), + }]; + assert!(llvm_site_layer_warnings(&foss(), &also, &files, &added).is_empty()); + let lfoss = Toolchain { + name: "lfoss".into(), + version: "2026.1".into(), + }; + assert!(llvm_site_layer_warnings(&lfoss, &[], &files, &added).is_empty()); + } + + #[test] + fn a_version_bump_and_a_foss_easystack_entry_are_not_site_layer() { + let files = [ + "GROMACS-2024.4-foss-2025b.eb".to_string(), + "OpenBLAS-0.3.32-lfoss-2025b.eb".to_string(), + ]; + let added = [( + "FFTW".into(), + "easyconfigs/foss-2025b/FFTW-3.3.10-foss-2025b.eb".into(), + )]; + assert!(llvm_site_layer_warnings(&foss(), &[], &files, &added).is_empty()); + } + + #[test] + fn a_solved_llvm_easyconfig_with_no_easystack_entry_is_site_layer() { + let added = [( + "OpenBLAS".into(), + "easyconfigs/lfbf-2026.1/OpenBLAS-0.3.32-lfbf-2026.1.eb".into(), + )]; + let warnings = llvm_site_layer_warnings(&foss(), &[], &[], &added); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert_eq!( + warnings[0], + "OpenBLAS (easyconfigs/lfbf-2026.1/OpenBLAS-0.3.32-lfbf-2026.1.eb) is the LLVM family (lfbf). This foss solve marked it site layer. Keep one lock per toolchain family: solve lfbf on its own, or pass --baseline-also-toolchain lfbf/2026.1." + ); + } + + #[test] + fn two_versions_of_one_llvm_family_leave_the_flag_version_unnamed() { + let added = [( + "OpenBLAS".into(), + "easyconfigs/foss-2025b/OpenBLAS-0.3.32-foss-2025b.eb".into(), + )]; + let files = [ + "OpenBLAS-0.3.32-lfoss-2026.1.eb".to_string(), + "OpenBLAS-0.3.30-lfoss-2025b.eb".to_string(), + ]; + let warnings = llvm_site_layer_warnings(&foss(), &[], &files, &added); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!( + warnings[0].contains( + "OpenBLAS (OpenBLAS-0.3.30-lfoss-2025b.eb, OpenBLAS-0.3.32-lfoss-2026.1.eb) is the LLVM family (lfoss)." + ), + "{warnings:?}" + ); + assert!( + warnings[0].contains("--baseline-also-toolchain lfoss/"), + "{warnings:?}" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 2b375509..b926b312 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod arch_known_failures; pub mod arch_support; pub mod artifact_class; +pub mod backport; pub mod batch; pub mod bot_result; pub mod build_backend; @@ -878,10 +879,53 @@ pub fn solve_with_baseline_source( anyhow::anyhow!("{text}{hint}{constraints}") })?; validate_lock_deps(&lock, &universe.candidates).map_err(|e| anyhow::anyhow!(e))?; + if let Some(base) = baseline.as_ref() { + let (easystack_files, also) = llvm_baseline_inputs(baseline_source)?; + let added: Vec<(String, String)> = classify_stack_diff(base, &lock) + .into_iter() + .filter(|change| change.kind == PackageChangeKind::Added) + .map(|change| { + ( + change.name, + change.solved_easyconfig_path.unwrap_or_default(), + ) + }) + .collect(); + for warning in generation_lint::llvm_site_layer_warnings( + &policy.toolchain, + &also, + &easystack_files, + &added, + ) { + eprintln!("warning: {warning}"); + } + } write_lock_sbom_and_extras(&lock, baseline.as_ref(), lock_out, sbom_out, extra)?; Ok(lock) } +/// Easyconfig filenames a baseline easystack names, and the extra families +/// the solve was asked to keep. A tree baseline has neither. +fn llvm_baseline_inputs(source: BaselineSource<'_>) -> Result<(Vec, Vec)> { + let BaselineSource::Easystacks { + files, + also_toolchains, + .. + } = source + else { + return Ok((Vec::new(), Vec::new())); + }; + let mut names = Vec::new(); + for path in files { + let text = + std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + let entries = + parse_easystack_entries(&text).with_context(|| format!("parse {}", path.display()))?; + names.extend(entries.into_iter().map(|entry| entry.file)); + } + Ok((names, also_toolchains.to_vec())) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/main.rs b/src/main.rs index e37960da..d66542ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1075,7 +1075,7 @@ enum StackCommand { Solve(Box), /// Emit CycloneDX from an existing stack lock. #[command( - after_long_help = "Examples:\n eb-stack stack sbom --lock stack.lock.json --out stack.cdx.json" + after_long_help = "Examples:\n eb-stack stack sbom --lock stack.lock.json --out stack.cdx.json\n eb-stack stack sbom --lock stack.lock.json --backport-list backport.md --osv-cache osv/ --reachability reach.txt" )] Sbom { /// Stack lock JSON written by `stack solve`. @@ -1084,6 +1084,23 @@ enum StackCommand { /// CycloneDX file to write. A single file, so this is `--out` rather than `--out-dir`. #[arg(long, default_value = "stack.cdx.json")] out: PathBuf, + /// Markdown backport list for this lock. A path ending in `.md` also + /// writes the sibling `.json`. + #[arg(long)] + backport_list: Option, + /// Advisory id to resolve through OSV. Repeatable. Needs `--backport-list`. + #[arg(long)] + cve: Vec, + /// Directory of recorded OSV responses (`queries.json` and the bodies + /// it names). Needs `--backport-list`. Without `--cve`, every id in + /// that file is resolved. + #[arg(long)] + osv_cache: Option, + /// Site reachability file, one `name class` per line. Needs + /// `--backport-list`. The classes are `cross-user`, `network-facing`, + /// `local-only`, and `not-reachable`. + #[arg(long)] + reachability: Option, }, /// Order the builds a set of roots needs, as a graph rather than a stack. /// @@ -3292,7 +3309,24 @@ fn run_stack(command: StackCommand) -> Result<()> { } Ok(()) } - StackCommand::Sbom { lock, out } => { + StackCommand::Sbom { + lock, + out, + backport_list, + cve, + osv_cache, + reachability, + } => { + if backport_list.is_none() + && (!cve.is_empty() || osv_cache.is_some() || reachability.is_some()) + { + return Err(usage( + "--cve, --osv-cache, and --reachability need --backport-list", + )); + } + if backport_list.is_some() && cve.is_empty() && osv_cache.is_none() { + return Err(usage("--backport-list needs --cve or --osv-cache")); + } let loaded: StackLock = load_lock_file(&lock)?; let (sbom, stats) = eb_stack::lock_to_cyclonedx_from_recipes(&loaded, &lock); write_json_pretty(&out, &sbom)?; @@ -3304,6 +3338,24 @@ fn run_stack(command: StackCommand) -> Result<()> { stats.recipes_unreadable, stats.with_artifact_facts ); + if let Some(path) = &backport_list { + let document = + eb_stack::backport::backport_list(&eb_stack::backport::BackportRequest { + lock: &loaded, + lock_path: &lock, + cves: &cve, + osv_cache: osv_cache.as_deref(), + reachability: reachability.as_deref(), + })?; + eb_stack::backport::write_backport(path, &document)?; + println!( + "backport={} recorded={} stack={} out={}", + document.listed, + document.recorded, + document.stack_modules, + path.display() + ); + } Ok(()) } } diff --git a/src/mcp.rs b/src/mcp.rs index 7484f61b..854535e7 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -343,9 +343,15 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_sbom", - "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names.", + "Emit CycloneDX from an existing stack lock, reading dependency edges and source checksums from the easyconfigs the lock names. With backport_list, also write the modules whose reach class justifies a backport.", &[("lock", "string")], - &[("out", "string")], + &[ + ("out", "string"), + ("backport_list", "string"), + ("cve", "array"), + ("osv_cache", "string"), + ("reachability", "string"), + ], ), tool_with_optional( "eb_stack_order", @@ -2876,6 +2882,17 @@ fn stack_solve(arguments: &Value) -> Result { fn stack_sbom(arguments: &Value) -> Result { let lock_path = required_path(arguments, "lock")?; let out = optional_path(arguments, "out").unwrap_or_else(|| PathBuf::from("stack.cdx.json")); + let backport_list = optional_path(arguments, "backport_list"); + let cve = string_array(arguments, "cve")?; + let osv_cache = optional_path(arguments, "osv_cache"); + let reachability = optional_path(arguments, "reachability"); + if backport_list.is_none() && (!cve.is_empty() || osv_cache.is_some() || reachability.is_some()) + { + return Err("--cve, --osv-cache, and --reachability need backport_list".into()); + } + if backport_list.is_some() && cve.is_empty() && osv_cache.is_none() { + return Err("backport_list needs cve or osv_cache".into()); + } let lock: StackLock = load_json_file(&lock_path).map_err(|error| error.to_string())?; let (sbom, stats) = lock_to_cyclonedx_from_recipes(&lock, &lock_path); write_json_pretty(&out, &sbom).map_err(|error| error.to_string())?; @@ -2884,14 +2901,30 @@ fn stack_sbom(arguments: &Value) -> Result { .and_then(Value::as_array) .map(Vec::len) .unwrap_or(0); - Ok(json!({ + let mut body = json!({ "components": components, "lock": lock_path, "out": out, "recipes_read": stats.recipes_read, "recipes_unreadable": stats.recipes_unreadable, "with_artifact_facts": stats.with_artifact_facts, - })) + }); + if let Some(path) = &backport_list { + let document = crate::backport::backport_list(&crate::backport::BackportRequest { + lock: &lock, + lock_path: &lock_path, + cves: &cve, + osv_cache: osv_cache.as_deref(), + reachability: reachability.as_deref(), + }) + .map_err(|error| error.to_string())?; + crate::backport::write_backport(path, &document).map_err(|error| error.to_string())?; + body["listed"] = json!(document.listed); + body["recorded"] = json!(document.recorded); + body["stack"] = json!(document.stack_modules); + body["backport_list"] = json!(path); + } + Ok(body) } fn workflow_error(error: anyhow::Error) -> String { diff --git a/src/provided.rs b/src/provided.rs index 1ea83b83..cbea19a4 100644 --- a/src/provided.rs +++ b/src/provided.rs @@ -15,7 +15,7 @@ use crate::cli_parse::usage; use crate::domain::{StackLock, Toolchain}; -use crate::easystack::filename_toolchain; +use crate::easystack::{filename_toolchain, llvm_family_name}; use crate::gap::{names_from_listing, software_name}; use crate::hierarchy::known_hierarchy; use crate::jenkins::parse_build_list; @@ -25,9 +25,6 @@ use serde::Serialize; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -/// Toolchain names of the LLVM family. A foss solve cannot see them. -const LLVM_FAMILY: &[&str] = &["lfoss", "lompi", "lfbf", "llvm-compilers"]; - /// How `stack provided` prints. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProvidedFormat { @@ -558,19 +555,12 @@ fn toolchain_detail(file: &str, families: &[FamilyLock], allowed: &[String]) -> } fn llvm_suffix(toolchain_name: &str, families: &[FamilyLock]) -> String { - if !LLVM_FAMILY - .iter() - .any(|family| family.eq_ignore_ascii_case(toolchain_name)) - { + if llvm_family_name(toolchain_name).is_none() { return String::new(); } let locks: Vec<&FamilyLock> = families .iter() - .filter(|family| { - LLVM_FAMILY - .iter() - .any(|name| name.eq_ignore_ascii_case(&family.toolchain_name)) - }) + .filter(|family| llvm_family_name(&family.toolchain_name).is_some()) .collect(); if locks.is_empty() { return format!(" {toolchain_name} is the LLVM family, not a site-layer toolchain"); diff --git a/src/rebuild_set.rs b/src/rebuild_set.rs index 833bfbd3..9e54215e 100644 --- a/src/rebuild_set.rs +++ b/src/rebuild_set.rs @@ -48,7 +48,9 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use thiserror::Error; /// Schema of [`RebuildDocument`]. Bumped when a field changes meaning. -pub const REBUILD_SET_SCHEMA_VERSION: u32 = 4; +/// +/// Version 5 adds `reach`, the class of each patched easyconfig. +pub const REBUILD_SET_SCHEMA_VERSION: u32 = 5; /// Class for a runtime dependency, direct or through other runtime dependencies. pub const CLASS_NO_REBUILD: &str = "no-rebuild"; @@ -179,6 +181,11 @@ pub struct RebuildDocument { pub easystacks: Vec, /// Patched easyconfig filenames, sorted. pub changed: Vec, + /// Reach class of each filename in [`Self::changed`], including missing + /// ones. `cross-user` and `network-facing` justify a backport. + /// `local-only` and `not-reachable` do not. `unclassified` means neither + /// the built-in names nor a site file names the package. + pub reach: BTreeMap, /// Changed filenames that are not in the deployed graph. pub missing: Vec, /// Requirements no easyconfig satisfies, and easystack entries the robot @@ -321,6 +328,8 @@ pub fn rebuild_set(query: &RebuildQuery) -> Result Result String { inlined static link is reported as no-rebuild. A short name shared with another \ library is reported as must-rebuild. An empty export set stays in review. Scripts, \ bytecode, and libraries opened with dlopen are invisible. A static archive shows \ - that a static link was possible, not that it happened.\n", + that a static link was possible, not that it happened.\n\n", + ); + out.push_str( + "Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport. `local-only` and `not-reachable` do not. `unclassified` is not a judgment.\n\n", ); + for (name, class) in &document.reach { + out.push_str(&format!("- `{name}`: {class}\n")); + } out } @@ -1151,6 +1167,16 @@ mod tests { assert!(markdown.contains("`zlib-1.3.1-GCCcore-15.2.0.eb`")); assert!(markdown.contains("2 must-rebuild, 2 review, 4 no-rebuild.")); assert!(markdown.contains("--check-static")); + assert!(markdown.contains( + "Reach class of each patched easyconfig. `cross-user` and `network-facing` justify a backport." + )); + assert_eq!( + document + .reach + .get("zlib-1.3.1-GCCcore-15.2.0.eb") + .map(String::as_str), + Some("unclassified") + ); } #[test] diff --git a/src/site_commands.rs b/src/site_commands.rs index 30a9646c..8e5a5605 100644 --- a/src/site_commands.rs +++ b/src/site_commands.rs @@ -349,10 +349,14 @@ impl ComponentReachRun { ); } format!( - "component={} version={} generation={} carriers={} changed={} must-rebuild={} review={} no-rebuild={} out={}", + "component={} version={} generation={} reach={} carriers={} changed={} must-rebuild={} review={} no-rebuild={} out={}", self.component, self.version, self.generation, + self.document + .get("reach") + .and_then(|value| value.as_str()) + .unwrap_or("unclassified"), self.carriers, self.changed, self.must_rebuild, @@ -769,7 +773,7 @@ pub fn run_rebuild_set(args: &RebuildSetRequest) -> Result { } else { crate::fix_cost::EvidenceScan::default() }; - let document = rebuild_set::rebuild_set(&rebuild_set::RebuildQuery { + let mut document = rebuild_set::rebuild_set(&rebuild_set::RebuildQuery { base: robot.base, commits: robot.commits, easystacks, @@ -786,6 +790,7 @@ pub fn run_rebuild_set(args: &RebuildSetRequest) -> Result { rebuild_set::RebuildError::CheckStaticNeedsPrefix => usage("--check-static needs --prefix"), other => anyhow!(other), })?; + document.reach = crate::findings::reach_for_easyconfigs(&reach, &document.changed); std::fs::create_dir_all(&args.out_dir)?; write_json_pretty(&args.out_dir.join("rebuild-set.json"), &document)?; diff --git a/tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..ab20918f --- /dev/null +++ b/tests/backport_fixtures/eb/App-1.0-GCCcore-15.2.0.eb @@ -0,0 +1,4 @@ +name = 'App' +version = '1.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} +dependencies = [('zlib', '1.3.1')] diff --git a/tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..6c2679dd --- /dev/null +++ b/tests/backport_fixtures/eb/Mystery-1.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'Mystery' +version = '1.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..687f825a --- /dev/null +++ b/tests/backport_fixtures/eb/WinOnly-1.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'WinOnly' +version = '1.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..1d2f3b1b --- /dev/null +++ b/tests/backport_fixtures/eb/cURL-8.17.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'cURL' +version = '8.17.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..b8943919 --- /dev/null +++ b/tests/backport_fixtures/eb/git-2.52.0-GCCcore-15.2.0.eb @@ -0,0 +1,4 @@ +name = 'git' +version = '2.52.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} +dependencies = [('cURL', '8.17.0')] diff --git a/tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb new file mode 100644 index 00000000..1e4a2c84 --- /dev/null +++ b/tests/backport_fixtures/eb/notebook-7.4.0-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'notebook' +version = '7.4.0' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb b/tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb new file mode 100644 index 00000000..a63a03d0 --- /dev/null +++ b/tests/backport_fixtures/eb/zlib-1.3.1-GCCcore-15.2.0.eb @@ -0,0 +1,3 @@ +name = 'zlib' +version = '1.3.1' +toolchain = {'name': 'GCCcore', 'version': '15.2.0'} diff --git a/tests/backport_fixtures/osv/GHSA-test-curl-old.json b/tests/backport_fixtures/osv/GHSA-test-curl-old.json new file mode 100644 index 00000000..ee51ba06 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-curl-old.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-curl-old", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "curl", "purl": "pkg:generic/curl"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "8.0.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-curl.json b/tests/backport_fixtures/osv/GHSA-test-curl.json new file mode 100644 index 00000000..ff9af93c --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-curl.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-curl", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "curl", "purl": "pkg:generic/curl"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "8.18.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-missing.json b/tests/backport_fixtures/osv/GHSA-test-missing.json new file mode 100644 index 00000000..55b08a2a --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-missing.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-missing", + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "jupyter_server", "purl": "pkg:pypi/jupyter-server"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "2.20.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-mystery.json b/tests/backport_fixtures/osv/GHSA-test-mystery.json new file mode 100644 index 00000000..a4509c2e --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-mystery.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-mystery", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "Mystery", "purl": "pkg:generic/mystery"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "2.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-notebook.json b/tests/backport_fixtures/osv/GHSA-test-notebook.json new file mode 100644 index 00000000..28359a05 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-notebook.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-notebook", + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "notebook", "purl": "pkg:pypi/notebook"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "7.5.0"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-unparsed.json b/tests/backport_fixtures/osv/GHSA-test-unparsed.json new file mode 100644 index 00000000..4c5353aa --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-unparsed.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-unparsed", + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "notebook", "purl": "pkg:pypi/notebook"}, + "ranges": [{"type": "SEMVER", "events": [{"limit": "1"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-win.json b/tests/backport_fixtures/osv/GHSA-test-win.json new file mode 100644 index 00000000..4a7c2ba1 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-win.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-win", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "WinOnly", "purl": "pkg:generic/winonly"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.1"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/GHSA-test-zlib.json b/tests/backport_fixtures/osv/GHSA-test-zlib.json new file mode 100644 index 00000000..c18279e8 --- /dev/null +++ b/tests/backport_fixtures/osv/GHSA-test-zlib.json @@ -0,0 +1,7 @@ +{ + "id": "GHSA-test-zlib", + "affected": [{ + "package": {"ecosystem": "OSS-Fuzz", "name": "zlib", "purl": "pkg:generic/zlib"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.3.2"}]}] + }] +} diff --git a/tests/backport_fixtures/osv/queries.json b/tests/backport_fixtures/osv/queries.json new file mode 100644 index 00000000..9b81705b --- /dev/null +++ b/tests/backport_fixtures/osv/queries.json @@ -0,0 +1,12 @@ +{ + "queries": [ + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-curl", "file": "GHSA-test-curl.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-curl-old", "file": "GHSA-test-curl-old.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-missing", "file": "GHSA-test-missing.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-mystery", "file": "GHSA-test-mystery.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-notebook", "file": "GHSA-test-notebook.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-unparsed", "file": "GHSA-test-unparsed.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-win", "file": "GHSA-test-win.json"}, + {"url": "https://api.osv.dev/v1/vulns/GHSA-test-zlib", "file": "GHSA-test-zlib.json"} + ] +} diff --git a/tests/backport_fixtures/reach.txt b/tests/backport_fixtures/reach.txt new file mode 100644 index 00000000..30d76ad1 --- /dev/null +++ b/tests/backport_fixtures/reach.txt @@ -0,0 +1,3 @@ +# site overlay +zlib local-only +WinOnly not-reachable diff --git a/tests/backport_fixtures/stack.lock.json b/tests/backport_fixtures/stack.lock.json new file mode 100644 index 00000000..9cbd8659 --- /dev/null +++ b/tests/backport_fixtures/stack.lock.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "generation_label": "GCCcore-15.2.0", + "packages": [ + { + "name": "App", + "version": "1.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/App-1.0-GCCcore-15.2.0.eb" + }, + { + "name": "Mystery", + "version": "1.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/Mystery-1.0-GCCcore-15.2.0.eb" + }, + { + "name": "WinOnly", + "version": "1.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/WinOnly-1.0-GCCcore-15.2.0.eb" + }, + { + "name": "cURL", + "version": "8.17.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/cURL-8.17.0-GCCcore-15.2.0.eb" + }, + { + "name": "git", + "version": "2.52.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/git-2.52.0-GCCcore-15.2.0.eb" + }, + { + "name": "notebook", + "version": "7.4.0", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/notebook-7.4.0-GCCcore-15.2.0.eb" + }, + { + "name": "zlib", + "version": "1.3.1", + "toolchain": { + "name": "GCCcore", + "version": "15.2.0" + }, + "easyconfig_path": "eb/zlib-1.3.1-GCCcore-15.2.0.eb" + } + ], + "solver": { + "engine": "resolvo", + "engine_version": "test", + "timestamp": "2026-11-01T00:00:00Z", + "criteria": [] + } +} diff --git a/tests/backport_list.rs b/tests/backport_list.rs new file mode 100644 index 00000000..827532c4 --- /dev/null +++ b/tests/backport_list.rs @@ -0,0 +1,245 @@ +//! `stack sbom --backport-list` from a lock, recorded OSV responses, and the +//! SBOM reverse closure. + +use eb_stack::domain::StackLock; +use eb_stack::mcp::handle_message; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/backport_fixtures") +} + +fn lock_path() -> PathBuf { + root().join("stack.lock.json") +} + +fn load_lock() -> StackLock { + serde_json::from_str(&std::fs::read_to_string(lock_path()).unwrap()).unwrap() +} + +#[test] +fn the_lock_lists_cross_user_and_network_facing_and_records_the_rest() { + let lock = load_lock(); + let cache = root().join("osv"); + let reach = root().join("reach.txt"); + let document = eb_stack::backport::backport_list(&eb_stack::backport::BackportRequest { + lock: &lock, + lock_path: &lock_path(), + cves: &[], + osv_cache: Some(&cache), + reachability: Some(&reach), + }) + .expect("backport list"); + assert_eq!(document.schema_version, 1); + assert_eq!(document.generation, "GCCcore-15.2.0"); + assert_eq!(document.stack_modules, 7); + assert_eq!(document.listed, 2); + assert_eq!(document.recorded, 3); + let listed: Vec<(&str, &str, &str, usize)> = document + .rows + .iter() + .filter(|row| row.on_list) + .map(|row| { + ( + row.module.as_str(), + row.reach.as_str(), + row.advisory.as_str(), + row.blast_radius, + ) + }) + .collect(); + assert_eq!( + listed, + [ + ( + "notebook/7.4.0-GCCcore-15.2.0", + "cross-user", + "GHSA-test-notebook", + 1 + ), + ( + "cURL/8.17.0-GCCcore-15.2.0", + "network-facing", + "GHSA-test-curl", + 2 + ), + ] + ); + let curl = document + .rows + .iter() + .find(|row| row.name == "cURL") + .expect("curl"); + assert_eq!(curl.fixed_in, ["8.18.0"]); + assert_eq!( + curl.blast, + [ + "cURL/8.17.0-GCCcore-15.2.0".to_string(), + "git/2.52.0-GCCcore-15.2.0".to_string() + ] + ); + let recorded: Vec<(&str, &str, usize)> = document + .rows + .iter() + .filter(|row| !row.on_list) + .map(|row| (row.module.as_str(), row.reach.as_str(), row.blast_radius)) + .collect(); + assert_eq!( + recorded, + [ + ("zlib/1.3.1-GCCcore-15.2.0", "local-only", 2), + ("WinOnly/1.0-GCCcore-15.2.0", "not-reachable", 1), + ("Mystery/1.0-GCCcore-15.2.0", "unclassified", 1), + ] + ); + let zlib = document + .rows + .iter() + .find(|row| row.name == "zlib") + .expect("zlib"); + assert_eq!( + zlib.blast, + [ + "App/1.0-GCCcore-15.2.0".to_string(), + "zlib/1.3.1-GCCcore-15.2.0".to_string() + ] + ); + assert_eq!( + document.notes, + [ + "GHSA-test-missing names jupyter_server, which this lock does not have".to_string(), + "GHSA-test-unparsed names notebook, but its ranges were not parsed".to_string(), + "cURL/8.17.0-GCCcore-15.2.0 is not in range for GHSA-test-curl-old (>=0,<8.0.0)" + .to_string(), + ] + ); + let markdown = eb_stack::backport::render_markdown(&document); + assert!(markdown.contains("## Backport")); + assert!(markdown.contains( + "These modules stay off the backport list. A backport is cross-user or network-facing." + )); + assert!(markdown.find("notebook/7.4.0").unwrap() < markdown.find("cURL/8.17.0").unwrap()); +} + +#[test] +fn the_cli_writes_the_list_and_rejects_a_partial_flag_set() { + let tmp = tempfile::tempdir().unwrap(); + let md = tmp.path().join("backport.md"); + let sbom = tmp.path().join("stack.cdx.json"); + let output = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args(["stack", "sbom", "--lock"]) + .arg(lock_path()) + .arg("--out") + .arg(&sbom) + .arg("--backport-list") + .arg(&md) + .arg("--osv-cache") + .arg(root().join("osv")) + .arg("--reachability") + .arg(root().join("reach.txt")) + .output() + .expect("run"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stderr}\n{stdout}"); + assert!( + stdout.contains("components=7 recipes_read=7 recipes_unreadable=0"), + "{stdout}" + ); + assert!( + stdout.contains(&format!( + "backport=2 recorded=3 stack=7 out={}", + md.display() + )), + "{stdout}" + ); + assert!(sbom.is_file()); + let json: Value = + serde_json::from_str(&std::fs::read_to_string(md.with_extension("json")).unwrap()).unwrap(); + assert_eq!(json["listed"], 2); + assert_eq!(json["recorded"], 3); + assert!(std::fs::read_to_string(&md) + .unwrap() + .contains("network-facing")); + + let missing = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args([ + "stack", + "sbom", + "--lock", + "/no/such.lock", + "--cve", + "GHSA-test-curl", + ]) + .output() + .expect("run"); + assert_eq!(missing.status.code(), Some(2)); + let err = String::from_utf8_lossy(&missing.stderr); + assert!(err.contains("need --backport-list"), "{err}"); + + let bare = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args([ + "stack", + "sbom", + "--lock", + "/no/such.lock", + "--backport-list", + "/tmp/backport.md", + ]) + .output() + .expect("run"); + assert_eq!(bare.status.code(), Some(2)); + let err = String::from_utf8_lossy(&bare.stderr); + assert!(err.contains("needs --cve or --osv-cache"), "{err}"); +} + +#[test] +fn one_cve_does_not_pull_the_rest_of_the_cache() { + let lock = load_lock(); + let cache = root().join("osv"); + let document = eb_stack::backport::backport_list(&eb_stack::backport::BackportRequest { + lock: &lock, + lock_path: &lock_path(), + cves: &["GHSA-test-curl".into()], + osv_cache: Some(&cache), + reachability: None, + }) + .expect("one advisory"); + assert_eq!(document.advisories, ["GHSA-test-curl"]); + assert_eq!(document.listed, 1); + assert_eq!(document.recorded, 0); + assert!(document.notes.is_empty(), "{:?}", document.notes); + assert_eq!(document.rows[0].reach, "network-facing"); +} + +#[test] +fn mcp_stack_sbom_can_write_the_backport_list() { + let tmp = tempfile::tempdir().unwrap(); + let out = tmp.path().join("stack.cdx.json"); + let md = tmp.path().join("backport.md"); + let response = handle_message(&json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "eb_stack_sbom", + "arguments": { + "lock": lock_path(), + "out": &out, + "backport_list": &md, + "osv_cache": root().join("osv"), + "reachability": root().join("reach.txt") + } + } + })) + .expect("mcp"); + assert_eq!(response["result"]["isError"], false, "{response}"); + let body = &response["result"]["structuredContent"]; + assert_eq!(body["listed"], 2); + assert_eq!(body["recorded"], 3); + assert_eq!(body["stack"], 7); + assert!(md.is_file()); + let _ = Path::new(&out); +} diff --git a/tests/component_reach.rs b/tests/component_reach.rs index 7107ab62..1a2e2151 100644 --- a/tests/component_reach.rs +++ b/tests/component_reach.rs @@ -48,7 +48,8 @@ fn the_cli_writes_toml_json_and_markdown() { let json: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(out.join("component-reach.json")).unwrap()) .unwrap(); - assert_eq!(json["schema_version"], 1); + assert_eq!(json["schema_version"], 2); + assert_eq!(json["reach"], "network-facing"); assert_eq!(json["component"], "curl"); assert!(json["closure_counts"]["no-rebuild"].as_u64().unwrap() >= 1); let toml = std::fs::read_to_string(out.join("component-reach.toml")).unwrap(); diff --git a/tests/family_site_layer.rs b/tests/family_site_layer.rs new file mode 100644 index 00000000..eef1715c --- /dev/null +++ b/tests/family_site_layer.rs @@ -0,0 +1,76 @@ +//! A foss solve warns when an LLVM-family easystack entry looks like site layer. + +use std::path::PathBuf; +use std::process::Command; + +fn easyconfigs() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/gromacs_2025_to_next/easyconfigs") +} + +fn solve(stack: &str) -> std::process::Output { + let tmp = tempfile::tempdir().unwrap(); + let easystack = tmp.path().join("baseline.yml"); + std::fs::write(&easystack, stack).unwrap(); + let lock = tmp.path().join("stack.lock.json"); + let diff = tmp.path().join("stack.diff.md"); + let output = Command::new(env!("CARGO_BIN_EXE_eb-stack")) + .args(["stack", "solve", "--easyconfigs"]) + .arg(easyconfigs()) + .args(["--toolchain", "foss/2025b", "--root", "GROMACS"]) + .arg("--baseline-easystack") + .arg(&easystack) + .arg("--lock-out") + .arg(&lock) + .arg("--stack-diff-out") + .arg(&diff) + .output() + .expect("run eb-stack"); + // Keep the temp dir alive until the command has finished reading it. + drop(tmp); + output +} + +#[test] +fn a_foss_solve_warns_that_an_lfoss_easystack_entry_is_not_site_layer() { + let output = solve("easyconfigs:\n - GROMACS-2025.0-lfoss-2025b.eb\n"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "solve failed: {stderr}\n{}", + String::from_utf8_lossy(&output.stdout) + ); + let solved = easyconfigs().join("foss-2025b/GROMACS-2025.0-foss-2025b.eb"); + let expected = format!( + "warning: GROMACS (GROMACS-2025.0-lfoss-2025b.eb) is the LLVM family (lfoss). A foss solve cannot reach it, so the site layer includes GROMACS ({}). Keep one lock per toolchain family: solve lfoss on its own, or pass --baseline-also-toolchain lfoss/2025b.", + solved.display() + ); + assert!( + stderr.contains(&expected), + "stderr missing the LLVM warning:\n{stderr}\nexpected:\n{expected}" + ); + assert!( + stderr.contains("GROMACS-2025.0-lfoss-2025b.eb"), + "the unresolved easystack entry is still reported: {stderr}" + ); +} + +#[test] +fn a_version_bumped_lfoss_name_is_not_reported_as_site_layer() { + let output = solve( + "easyconfigs:\n - GROMACS-2024.4-foss-2025b.eb\n - OpenBLAS-0.3.32-lfoss-2025b.eb\n", + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "solve failed: {stderr}\n{}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + stderr.contains("OpenBLAS-0.3.32-lfoss-2025b.eb"), + "an entry with no recipe is reported: {stderr}" + ); + assert!( + !stderr.contains("LLVM family"), + "a version bump is not site layer: {stderr}" + ); +} diff --git a/tests/rebuild_set.rs b/tests/rebuild_set.rs index cf1f2258..b4aa10f1 100644 --- a/tests/rebuild_set.rs +++ b/tests/rebuild_set.rs @@ -138,7 +138,12 @@ fn the_cli_writes_the_rebuild_set_and_drops_a_stale_easystack() { .1 .as_str(); let value: serde_json::Value = serde_json::from_str(json).expect("parse json"); - assert_eq!(value["schema_version"], 4); + assert_eq!(value["schema_version"], 5); + assert_eq!( + value["reach"]["zlib-1.3.1-GCCcore-15.2.0.eb"], + "unclassified" + ); + assert_eq!(value["reach"]["Absent-1.0.eb"], "unclassified"); assert_eq!(value["symbols_checked"], false); assert_eq!(value["generated_at"], "2026-10-22T12:00:00Z"); assert_eq!(value["installed_source"], "x86_64/amd/zen4");