From 5484b09e72377367c7c4b1c12304cf1c7f3fc972 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:02:18 +0000 Subject: [PATCH 1/7] feat(stack): read PARALLELISM_LIMITS from eb_hooks.py in stack oom The `stack oom` command warned on a non-integer divide_by_factor only in the hook rows copied into `data/eessi-bot-resources.toml`. The `--hooks FILE` option reads PARALLELISM_LIMITS from a software-layer-scripts eb_hooks.py with rustpython-parser, without running the file, and uses those rows in place of the table's. A key may be a module-level string constant such as CPU_TARGET_A64FX, and a repeated key keeps its last value, as in Python. A float factor is a warning that names the file and the line, since 16 // 2.5 is 6.0. These exit 1 with the line: an op other than divide_by_factor or set_maximum, a `** unpacking`, and a non-string key. The fixtures are `hooks/divide-2.5.py`, which warns, and `hooks/integers.py`, which does not and gives the bundled table's parallel levels. The `eb_stack_oom` tool takes hooks. --- CHANGELOG.md | 9 + docs/orgmode/howto/oom-profile.org | 57 ++- docs/orgmode/reference/cli.org | 26 +- llms-full.txt | 83 +++- skills/eessi-extend/SKILL.md | 4 +- src/eb_hooks.rs | 441 ++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 22 +- src/mcp.rs | 4 +- src/oom_profile.rs | 49 +- src/site_commands.rs | 1 + tests/mcp_v1.rs | 24 + tests/oom_profile.rs | 101 +++- .../oom_profile_fixtures/hooks/divide-2.5.py | 23 + tests/oom_profile_fixtures/hooks/integers.py | 22 + 15 files changed, 826 insertions(+), 41 deletions(-) create mode 100644 src/eb_hooks.rs create mode 100644 tests/oom_profile_fixtures/hooks/divide-2.5.py create mode 100644 tests/oom_profile_fixtures/hooks/integers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b1898e..a10609a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,15 @@ All notable changes to this project are documented here. records the closures, the shard weights and a SHA-256 of the inputs. A line that cannot be resolved exits 1 and writes nothing. `eb_stack_shard` is the same command over MCP. +- `stack oom --hooks FILE` reads `PARALLELISM_LIMITS` from an EESSI + `eb_hooks.py` in place of the hook rows of + `data/eessi-bot-resources.toml`. The file is parsed as Python and + never run. A key may be a module-level string constant such as + `CPU_TARGET_A64FX`, and a repeated key keeps its last value. A + `divide_by_factor` that is not an integer is a warning that names the + file and line (`16 // 2.5` is `6.0`, a float). An op other than + `divide_by_factor` or `set_maximum`, a `**` unpacking, or a key that + is not a string exits 1 with the line. `eb_stack_oom` takes `hooks`. ### Documentation diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index 4cda00a6..7beeeab6 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -112,9 +112,11 @@ bigger node. The parallel level comes from the hook limits in ~data/eessi-bot-resources.toml~. Pass ~--resources~ to try other limits -or other nodes. Pull request 1608 ran out of memory in the PETSc test -step on zen5 before software-layer-scripts #298 halved PETSc. With the -current hooks, PETSc fits: +or other nodes, or ~--hooks~ to read the limits from an ~eb_hooks.py~ +(see [[*Read the limits from eb_hooks.py][below]]). Pull request 1608 +ran out of memory in the PETSc test step on zen5 before +software-layer-scripts #298 halved PETSc. With the current hooks, PETSc +fits: #+begin_src bash eb-stack stack oom \ @@ -223,6 +225,52 @@ is 24 GiB and the 32 GiB zen4 job fits. With the integer divider of 2 the same build is the warning in the first example. The command still exits 0. +* Read the limits from eb_hooks.py + +The hook rows in ~data/eessi-bot-resources.toml~ are a copy of +~PARALLELISM_LIMITS~. ~--hooks~ reads that dict from an ~eb_hooks.py~ +instead, so a hook change can be checked before it is merged. The file +is parsed as Python, and nothing in it runs. The fixture +~hooks/divide-2.5.py~ has the layout of the real file and the libxc +divider of 2.5: + +#+begin_src bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-2.5.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-2.5.py +warning: tests/oom_profile_fixtures/hooks/divide-2.5.py:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. +oom=0 fits=2 builds=5 nodes=2 +#+end_example + +The first line says how many rows were read, and from which file. The +warning names the line of the ~(divide_by_factor, 2.5)~ tuple. The nodes +still come from the bot resource table, so both zen4 nodes are checked. +With every factor an integer there is no warning: + +#+begin_src bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/integers.py +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +#+end_example + +The libxc and A64FX rows of ~integers.py~ are the ones in the bot +resource table, so ~--node~ prints the same parallel levels with and +without it. For the real hooks, pass the file from a software-layer-scripts +checkout, ~--hooks software-layer-scripts/eb_hooks.py~. + +A key is a string or a module-level string constant such as +~CPU_TARGET_A64FX~. A key the dict repeats keeps its last value, as in +Python. An op other than ~divide_by_factor~ or ~set_maximum~, a ~**~ +unpacking, or a key that is not a string stops the command with the +file and the line: the parallel levels would be wrong without that row. + * A memory limit instead of a core fraction A core fraction does not travel. Half of 16 cores and half of 64 cores @@ -364,6 +412,9 @@ then. steps, test steps that run with a different parallel level, or memory that other jobs on the node use. It does not model ~ulimit -u~, which ~det_parallelism~ also consults. +- ~--hooks~ reads the ~PARALLELISM_LIMITS~ literal. It does not run + ~post_ready_hook~; the rules that pick a row are the ones described + under [[*Where the numbers come from][where the numbers come from]]. - ~limit_by_memory~ does not replace the estimate. The estimate follows the hooks that are in ~eb_hooks.py~. The ~--node~ report is the comparison. A package with no measured ~mib_per_job~ stays diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 6bca18bf..7b1286b9 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -17,7 +17,7 @@ reachability, toolchain family, and the claim ladder are defined in | 1 | The work was attempted and failed: a missing or unreadable file, a document that does not parse, a check that found errors, a network fetch that failed, a doctor probe, or a campaign whose build failed | | 2 | The command line was rejected before that work: a missing or unknown flag (clap), or a flag value or combination the command does not accept | -A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails ~recipe lint~ or ~recipe check~ is status 1: the files were read and the check failed. ~stack from-commit~ is status 1 when any pin is ~released~, and status 0 when every pin is ~newer~ or ~unknown~ or the files name no pin. A file name with no ~eb-X.Y.Z~, and no ~--easybuild-version~, is status 2. ~stack arch~ is status 1 when an entry is unsupported or flagged, or when ~--probe-sources~ reports ~unreachable~ or ~arch-missing~ sources. ~stack files~ is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are ~no_url~ or ~unchecked~. ~stack generation~ is status 1 on any finding, and status 2 when an easystack path names no EESSI version and ~--eessi-version~ is absent. ~stack known-failures~ is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with ~--strict~ it is status 1 when a warning remains, and status 0 when every hit is remedied. ~stack oom~ is status 0 when it warns: the warnings are advice, not a failed check. A ~--node~ value that is not ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~ is status 2, and so is a command with neither ~--easystack~ nor ~--node~. ~stack bot-classify~ is status 0 for every class, and status 2 when a path is a directory. ~stack batch~ is status 1 when the plan includes, or depends on, an entry the architecture cannot build. ~stack provided~ is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets ~--hooks~, or belongs in a ~--family-lock~. A group-restricted warning is status 0 unless ~--strict~. Status 2 is when neither ~--installed~ nor ~--baseline-lock~ is set. ~stack shard~ is status 1 when a build-list line cannot be resolved, and writes nothing; ~--shards 0~ and a list with no lines are status 2. ~stack shadow~ is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. +A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails ~recipe lint~ or ~recipe check~ is status 1: the files were read and the check failed. ~stack from-commit~ is status 1 when any pin is ~released~, and status 0 when every pin is ~newer~ or ~unknown~ or the files name no pin. A file name with no ~eb-X.Y.Z~, and no ~--easybuild-version~, is status 2. ~stack arch~ is status 1 when an entry is unsupported or flagged, or when ~--probe-sources~ reports ~unreachable~ or ~arch-missing~ sources. ~stack files~ is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are ~no_url~ or ~unchecked~. ~stack generation~ is status 1 on any finding, and status 2 when an easystack path names no EESSI version and ~--eessi-version~ is absent. ~stack known-failures~ is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with ~--strict~ it is status 1 when a warning remains, and status 0 when every hit is remedied. ~stack oom~ is status 0 when it warns: the warnings are advice, not a failed check. A ~--node~ value that is not ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~ is status 2, and so is a command with none of ~--easystack~, ~--node~, and ~--hooks~. A ~--hooks~ file that is not valid Python, has no module-level ~PARALLELISM_LIMITS~, or has a row the model cannot follow is status 1, and the message names the line. ~stack bot-classify~ is status 0 for every class, and status 2 when a path is a directory. ~stack batch~ is status 1 when the plan includes, or depends on, an entry the architecture cannot build. ~stack provided~ is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets ~--hooks~, or belongs in a ~--family-lock~. A group-restricted warning is status 0 unless ~--strict~. Status 2 is when neither ~--installed~ nor ~--baseline-lock~ is set. ~stack shard~ is status 1 when a build-list line cannot be resolved, and writes nothing; ~--shards 0~ and a list with no lines are status 2. ~stack shadow~ is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. ~--help~ for every command has an example and a description of every flag. A default that the command applies is printed in that help. A directory that receives several artifacts is ~--out-dir~. A single output file is ~--out~ (~--lock-out~ for the lock ~stack solve~ writes). ~--installed~, ~--easyconfigs~, ~--filter-deps~, and ~--generated-at~ use those names on every command that takes them. @@ -1152,8 +1152,20 @@ worst compile has not been measured, and the table does not guess. the current hook leaves beside ~limit_by_memory(parallel, mib_per_job)~, which is ~max(1, min(parallel, memory_mib // mib_per_job))~. The memory a hook would read on the build machine is the smallest cgroup v2 -~memory.max~ on the process's cgroup path, else ~MemTotal~. Pass -~--easystack~, ~--node~, or both. The command exits 0 when it warns. See +~memory.max~ on the process's cgroup path, else ~MemTotal~. + +~--hooks~ reads ~PARALLELISM_LIMITS~ from an EESSI ~eb_hooks.py~ in +place of the hook rows of ~data/eessi-bot-resources.toml~. The nodes +and EasyBuild's ~max-parallel~ still come from the table. The file is +parsed, not run. A key is a string or a module-level string constant +such as ~CPU_TARGET_A64FX~, and a key the dict repeats keeps its last +value, as in Python. An op other than ~divide_by_factor~ or +~set_maximum~, a ~**~ unpacking, or a key that is not a string is +refused with its line: the parallel levels would be wrong without that +row. The output starts with ~hooks: N PARALLELISM_LIMITS rows from +FILE~, and a ~divide~ that is not an integer is a warning that names the +file and the line. Pass ~--easystack~, ~--node~, ~--hooks~, or a mix. +The command exits 0 when it warns. See [[file:../howto/oom-profile.org][warn before a bot job runs out of memory]]. ** ~stack bot-classify~ @@ -1798,10 +1810,10 @@ needs one ~easystack~ or ~pr~. It returns the same report as stdout, and ~hits~, one object per line. Warnings are not a tool error unless ~strict~ is true, when a remaining warning is. ~eb_stack_oom~ takes ~easystack~, ~arch~, ~easyconfigs~, -~installed~, ~profile~, ~resources~, ~node~, and ~job_memory~. It needs -an easystack or a node. It returns the ~stack oom~ text, one object per -warning, and ~factor_warnings~ for a divide factor that is not an -integer. A warning is not a tool error. +~installed~, ~profile~, ~resources~, ~node~, ~job_memory~, and ~hooks~. +It needs an easystack, a node, or hooks. It returns the ~stack oom~ +text, one object per warning, and ~factor_warnings~ for a divide factor +that is not an integer. A warning is not a tool error. ~eb_stack_bot_classify~ requires ~files~. It returns ~text~, the ~stack bot-classify~ stdout, ~jobs~, one object per job, and ~counts~. ~eb_stack_batch~ takes ~arch~, ~unsupported~, ~known_failures~, diff --git a/llms-full.txt b/llms-full.txt index e55505f4..78e03f2c 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4705,9 +4705,11 @@ bigger node. The parallel level comes from the hook limits in `data/eessi-bot-resources.toml`. Pass `--resources` to try other limits -or other nodes. Pull request 1608 ran out of memory in the PETSc test -step on zen5 before software-layer-scripts #298 halved PETSc. With the -current hooks, PETSc fits: +or other nodes, or `--hooks` to read the limits from an `eb_hooks.py` +(see [[*Read the limits from eb_hooks.py][below]]). Pull request 1608 +ran out of memory in the PETSc test step on zen5 before +software-layer-scripts #298 halved PETSc. With the current hooks, PETSc +fits: ```bash eb-stack stack oom \ @@ -4816,6 +4818,52 @@ is 24 GiB and the 32 GiB zen4 job fits. With the integer divider of 2 the same build is the warning in the first example. The command still exits 0. +# Read the limits from eb_hooks.py + +The hook rows in `data/eessi-bot-resources.toml` are a copy of +`PARALLELISM_LIMITS`. `--hooks` reads that dict from an `eb_hooks.py` +instead, so a hook change can be checked before it is merged. The file +is parsed as Python, and nothing in it runs. The fixture +`hooks/divide-2.5.py` has the layout of the real file and the libxc +divider of 2.5: + +```bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-2.5.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-2.5.py +warning: tests/oom_profile_fixtures/hooks/divide-2.5.py:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. +oom=0 fits=2 builds=5 nodes=2 +``` + +The first line says how many rows were read, and from which file. The +warning names the line of the `(divide_by_factor, 2.5)` tuple. The nodes +still come from the bot resource table, so both zen4 nodes are checked. +With every factor an integer there is no warning: + +```bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/integers.py +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +``` + +The libxc and A64FX rows of `integers.py` are the ones in the bot +resource table, so `--node` prints the same parallel levels with and +without it. For the real hooks, pass the file from a software-layer-scripts +checkout, `--hooks software-layer-scripts/eb_hooks.py`. + +A key is a string or a module-level string constant such as +`CPU_TARGET_A64FX`. A key the dict repeats keeps its last value, as in +Python. An op other than `divide_by_factor` or `set_maximum`, a `**` +unpacking, or a key that is not a string stops the command with the +file and the line: the parallel levels would be wrong without that row. + # A memory limit instead of a core fraction A core fraction does not travel. Half of 16 cores and half of 64 cores @@ -4957,6 +5005,9 @@ then. steps, test steps that run with a different parallel level, or memory that other jobs on the node use. It does not model `ulimit -u`, which `det_parallelism` also consults. +- `--hooks` reads the `PARALLELISM_LIMITS` literal. It does not run + `post_ready_hook`; the rules that pick a row are the ones described + under [[*Where the numbers come from][where the numbers come from]]. - `limit_by_memory` does not replace the estimate. The estimate follows the hooks that are in `eb_hooks.py`. The `--node` report is the comparison. A package with no measured `mib_per_job` stays @@ -5851,7 +5902,7 @@ reachability, toolchain family, and the claim ladder are defined in | 1 | The work was attempted and failed: a missing or unreadable file, a document that does not parse, a check that found errors, a network fetch that failed, a doctor probe, or a campaign whose build failed | | 2 | The command line was rejected before that work: a missing or unknown flag (clap), or a flag value or combination the command does not accept | -A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails `recipe lint` or `recipe check` is status 1: the files were read and the check failed. `stack from-commit` is status 1 when any pin is `released`, and status 0 when every pin is `newer` or `unknown` or the files name no pin. A file name with no `eb-X.Y.Z`, and no `--easybuild-version`, is status 2. `stack arch` is status 1 when an entry is unsupported or flagged, or when `--probe-sources` reports `unreachable` or `arch-missing` sources. `stack files` is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are `no_url` or `unchecked`. `stack generation` is status 1 on any finding, and status 2 when an easystack path names no EESSI version and `--eessi-version` is absent. `stack known-failures` is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with `--strict` it is status 1 when a warning remains, and status 0 when every hit is remedied. `stack oom` is status 0 when it warns: the warnings are advice, not a failed check. A `--node` value that is not `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET` is status 2, and so is a command with neither `--easystack` nor `--node`. `stack bot-classify` is status 0 for every class, and status 2 when a path is a directory. `stack batch` is status 1 when the plan includes, or depends on, an entry the architecture cannot build. `stack provided` is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets `--hooks`, or belongs in a `--family-lock`. A group-restricted warning is status 0 unless `--strict`. Status 2 is when neither `--installed` nor `--baseline-lock` is set. `stack shard` is status 1 when a build-list line cannot be resolved, and writes nothing; `--shards 0` and a list with no lines are status 2. `stack shadow` is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. +A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails `recipe lint` or `recipe check` is status 1: the files were read and the check failed. `stack from-commit` is status 1 when any pin is `released`, and status 0 when every pin is `newer` or `unknown` or the files name no pin. A file name with no `eb-X.Y.Z`, and no `--easybuild-version`, is status 2. `stack arch` is status 1 when an entry is unsupported or flagged, or when `--probe-sources` reports `unreachable` or `arch-missing` sources. `stack files` is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are `no_url` or `unchecked`. `stack generation` is status 1 on any finding, and status 2 when an easystack path names no EESSI version and `--eessi-version` is absent. `stack known-failures` is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with `--strict` it is status 1 when a warning remains, and status 0 when every hit is remedied. `stack oom` is status 0 when it warns: the warnings are advice, not a failed check. A `--node` value that is not `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET` is status 2, and so is a command with none of `--easystack`, `--node`, and `--hooks`. A `--hooks` file that is not valid Python, has no module-level `PARALLELISM_LIMITS`, or has a row the model cannot follow is status 1, and the message names the line. `stack bot-classify` is status 0 for every class, and status 2 when a path is a directory. `stack batch` is status 1 when the plan includes, or depends on, an entry the architecture cannot build. `stack provided` is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets `--hooks`, or belongs in a `--family-lock`. A group-restricted warning is status 0 unless `--strict`. Status 2 is when neither `--installed` nor `--baseline-lock` is set. `stack shard` is status 1 when a build-list line cannot be resolved, and writes nothing; `--shards 0` and a list with no lines are status 2. `stack shadow` is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. `--help` for every command has an example and a description of every flag. A default that the command applies is printed in that help. A directory that receives several artifacts is `--out-dir`. A single output file is `--out` (`--lock-out` for the lock `stack solve` writes). `--installed`, `--easyconfigs`, `--filter-deps`, and `--generated-at` use those names on every command that takes them. @@ -6986,8 +7037,20 @@ worst compile has not been measured, and the table does not guess. the current hook leaves beside `limit_by_memory(parallel, mib_per_job)`, which is `max(1, min(parallel, memory_mib // mib_per_job))`. The memory a hook would read on the build machine is the smallest cgroup v2 -`memory.max` on the process's cgroup path, else `MemTotal`. Pass -`--easystack`, `--node`, or both. The command exits 0 when it warns. See +`memory.max` on the process's cgroup path, else `MemTotal`. + +`--hooks` reads `PARALLELISM_LIMITS` from an EESSI `eb_hooks.py` in +place of the hook rows of `data/eessi-bot-resources.toml`. The nodes +and EasyBuild's `max-parallel` still come from the table. The file is +parsed, not run. A key is a string or a module-level string constant +such as `CPU_TARGET_A64FX`, and a key the dict repeats keeps its last +value, as in Python. An op other than `divide_by_factor` or +`set_maximum`, a `**` unpacking, or a key that is not a string is +refused with its line: the parallel levels would be wrong without that +row. The output starts with ~hooks: N PARALLELISM_LIMITS rows from +FILE`, and a `divide~ that is not an integer is a warning that names the +file and the line. Pass `--easystack`, `--node`, `--hooks`, or a mix. +The command exits 0 when it warns. See [warn before a bot job runs out of memory](https://eb-stack.rgoswami.me/../howto/oom-profile.html.md). ## `stack bot-classify` @@ -7632,10 +7695,10 @@ needs one `easystack` or `pr`. It returns the same report as stdout, and `hits`, one object per line. Warnings are not a tool error unless `strict` is true, when a remaining warning is. `eb_stack_oom` takes `easystack`, `arch`, `easyconfigs`, -`installed`, `profile`, `resources`, `node`, and `job_memory`. It needs -an easystack or a node. It returns the `stack oom` text, one object per -warning, and `factor_warnings` for a divide factor that is not an -integer. A warning is not a tool error. +`installed`, `profile`, `resources`, `node`, `job_memory`, and `hooks`. +It needs an easystack, a node, or hooks. It returns the `stack oom` +text, one object per warning, and `factor_warnings` for a divide factor +that is not an integer. A warning is not a tool error. `eb_stack_bot_classify` requires `files`. It returns `text`, the `stack bot-classify` stdout, `jobs`, one object per job, and `counts`. `eb_stack_batch` takes `arch`, `unsupported`, `known_failures`, diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index d6f182fe..7002874b 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -341,7 +341,9 @@ The hook ops are `divide_by_factor` (`max(1, parallel // factor)`) and `set_maximum` (`min(parallel, n)`). A CPU-target key overrides `*`. A64FX divides every other build by 4. A divide factor that is not an integer is a warning: `16 // 2.5` is the float `6.0`. Use an integer. -`eb_hooks.py` also has a `msg+` where the log line should use `+=`; half +`--hooks software-layer-scripts/eb_hooks.py` reads `PARALLELISM_LIMITS` +from the file instead of the bundled copy, and the warning names the +line. `eb_hooks.py` also has a `msg+` where the log line should use `+=`; half the message never prints. That is upstream. Do not patch it from here. `eb-stack stack oom --node 16:32768:x86_64/amd/zen4` compares the hook diff --git a/src/eb_hooks.rs b/src/eb_hooks.rs new file mode 100644 index 00000000..cdbcd18c --- /dev/null +++ b/src/eb_hooks.rs @@ -0,0 +1,441 @@ +//! `PARALLELISM_LIMITS` from an EESSI `eb_hooks.py`, read as Python source. +//! +//! software-layer-scripts keeps its per-package parallelism limits in a +//! module-level dict that `post_ready_hook` reads. A key whose value is an +//! `(op, argument)` tuple is a CPU target's own limit. A key whose value is +//! a dict is a software name, and that dict's keys are CPU targets or +//! `'*'`. A key is a string or a module-level string constant such as +//! `CPU_TARGET_A64FX`. The ops are `divide_by_factor` and `set_maximum`. +//! The file is parsed, never run. A row this reader cannot model is an +//! error that names its line, because the parallel levels would be wrong +//! without it. + +use crate::oom_profile::{DivideFactor, HookLimit, LimitOp}; +use anyhow::{anyhow, bail, Result}; +use rustpython_parser::ast::{self, Ranged}; +use rustpython_parser::text_size::TextSize; +use rustpython_parser::Parse; +use std::collections::BTreeMap; + +/// The dict `post_ready_hook` looks a build up in. +pub const LIMITS_NAME: &str = "PARALLELISM_LIMITS"; + +/// `max(1, parallel // factor)` in `eb_hooks.py`. +const DIVIDE_OP: &str = "divide_by_factor"; + +/// `min(parallel, max_value)` in `eb_hooks.py`. +const MAXIMUM_OP: &str = "set_maximum"; + +/// What `stack oom` reads from one `eb_hooks.py`. +#[derive(Debug, Clone, PartialEq)] +pub struct HooksFile { + /// `PARALLELISM_LIMITS` rows. A key the dict repeats keeps its last + /// value, as Python does. + pub limits: Vec, +} + +/// Read `PARALLELISM_LIMITS` from `text`. +/// +/// `file` names the file in errors and in each row's location. The last +/// module-level assignment to `PARALLELISM_LIMITS` is the one +/// `post_ready_hook` sees, and its keys resolve through the string +/// constants bound before it. +pub fn parse_hooks(text: &str, file: &str) -> Result { + let suite = ast::Suite::parse(text, file).map_err(|error| { + let offset = offset(error.offset).min(text.len()); + anyhow!( + "{file}:{}: not valid Python: {}", + line_at(text, offset), + error.error + ) + })?; + let (value, constants) = + find_limits(&suite).ok_or_else(|| anyhow!("{file}: no module-level {LIMITS_NAME}"))?; + let reader = Reader { + text, + file, + constants: &constants, + }; + let ast::Expr::Dict(dict) = value else { + bail!( + "{file}:{}: {LIMITS_NAME} is not a dict literal", + reader.line(value) + ); + }; + let limits = reader.limits(dict)?; + Ok(HooksFile { limits }) +} + +/// The last module-level `PARALLELISM_LIMITS = ...`, and the string +/// constants bound before it. +fn find_limits(suite: &[ast::Stmt]) -> Option<(&ast::Expr, BTreeMap)> { + let mut constants = BTreeMap::new(); + let mut found = None; + for statement in suite { + let (targets, value): (Vec<&ast::Expr>, &ast::Expr) = match statement { + ast::Stmt::Assign(assign) => (assign.targets.iter().collect(), &assign.value), + ast::Stmt::AnnAssign(assign) => match &assign.value { + Some(value) => (vec![&assign.target], value), + None => continue, + }, + _ => continue, + }; + for target in targets { + let ast::Expr::Name(name) = target else { + continue; + }; + if name.id.as_str() == LIMITS_NAME { + found = Some((value, constants.clone())); + } else if let Some(text) = string_literal(value) { + constants.insert(name.id.to_string(), text.to_string()); + } else { + constants.remove(name.id.as_str()); + } + } + } + found +} + +fn string_literal(expr: &ast::Expr) -> Option<&str> { + match expr { + ast::Expr::Constant(ast::ExprConstant { + value: ast::Constant::Str(text), + .. + }) => Some(text), + _ => None, + } +} + +enum Number { + Int(u32), + Float(f64), +} + +fn number(expr: &ast::Expr) -> Option { + let ast::Expr::Constant(constant) = expr else { + return None; + }; + match &constant.value { + ast::Constant::Int(value) => value.to_string().parse().ok().map(Number::Int), + ast::Constant::Float(value) => Some(Number::Float(*value)), + _ => None, + } +} + +struct Reader<'a> { + text: &'a str, + file: &'a str, + constants: &'a BTreeMap, +} + +impl Reader<'_> { + fn line(&self, node: &impl Ranged) -> usize { + line_at(self.text, offset(node.start())) + } + + fn source(&self, node: &impl Ranged) -> &str { + let range = node.range(); + self.text + .get(offset(range.start())..offset(range.end())) + .unwrap_or("") + } + + fn key(&self, key: &ast::Expr) -> Result { + if let Some(text) = string_literal(key) { + return Ok(text.to_string()); + } + let at = format!("{}:{}", self.file, self.line(key)); + match key { + ast::Expr::Name(name) => { + self.constants + .get(name.id.as_str()) + .cloned() + .ok_or_else(|| { + anyhow!( + "{at}: {LIMITS_NAME} key {} is not a module-level string constant", + name.id + ) + }) + } + other => bail!( + "{at}: {LIMITS_NAME} key {} is not a string", + self.source(other) + ), + } + } + + fn limits(&self, dict: &ast::ExprDict) -> Result> { + let mut rows: Vec = Vec::new(); + for (key, value) in dict.keys.iter().zip(&dict.values) { + let Some(key) = key else { + bail!( + "{}:{}: {LIMITS_NAME} unpacks a dict with **; only written-out rows are read", + self.file, + self.line(value) + ); + }; + let name = self.key(key)?; + rows.retain(|row| { + row.software.as_deref() != Some(name.as_str()) + && !(row.software.is_none() && row.target == name) + }); + match value { + ast::Expr::Tuple(tuple) => { + if name == "*" { + bail!( + "{}:{}: a {LIMITS_NAME} limit for every build needs a CPU target, not '*'", + self.file, + self.line(key) + ); + } + let op = self.op(tuple, "every build", &name)?; + rows.push(HookLimit { + software: None, + target: name, + op, + line: Some(self.line(tuple)), + }); + } + ast::Expr::Dict(inner) => { + if inner.keys.is_empty() { + bail!( + "{}:{}: {LIMITS_NAME} entry for {name} is empty, and post_ready_hook then skips even the CPU target's own limit", + self.file, + self.line(value) + ); + } + for (target, row) in inner.keys.iter().zip(&inner.values) { + let Some(target) = target else { + bail!( + "{}:{}: {LIMITS_NAME} entry for {name} unpacks a dict with **; only written-out rows are read", + self.file, + self.line(row) + ); + }; + let target = self.key(target)?; + let ast::Expr::Tuple(tuple) = row else { + bail!( + "{}:{}: {LIMITS_NAME} row for {name} on {target} is {}, not an (op, argument) tuple", + self.file, + self.line(row), + self.source(row) + ); + }; + let op = self.op(tuple, &name, &target)?; + rows.retain(|limit| { + !(limit.software.as_deref() == Some(name.as_str()) + && limit.target == target) + }); + rows.push(HookLimit { + software: Some(name.clone()), + target, + op, + line: Some(self.line(tuple)), + }); + } + } + other => bail!( + "{}:{}: {LIMITS_NAME} value for {name} is {}, neither an (op, argument) tuple nor a dict of them", + self.file, + self.line(other), + self.source(other) + ), + } + } + Ok(rows) + } + + fn op(&self, tuple: &ast::ExprTuple, who: &str, target: &str) -> Result { + let at = format!( + "{}:{}: {LIMITS_NAME} row for {who} on {target}", + self.file, + self.line(tuple) + ); + let [op, argument] = tuple.elts.as_slice() else { + bail!("{at} is {}, not (op, argument)", self.source(tuple)); + }; + let ast::Expr::Name(op) = op else { + bail!( + "{at} names its op as {}; the ops are {DIVIDE_OP} and {MAXIMUM_OP}", + self.source(op) + ); + }; + match (op.id.as_str(), number(argument)) { + (DIVIDE_OP, Some(Number::Int(factor))) if factor > 0 => { + Ok(LimitOp::Divide(DivideFactor::from_integer(factor))) + } + (DIVIDE_OP, Some(Number::Float(factor))) if factor.is_finite() && factor > 0.0 => { + Ok(LimitOp::Divide(DivideFactor::from_float(factor))) + } + (MAXIMUM_OP, Some(Number::Int(maximum))) if maximum > 0 => { + Ok(LimitOp::Maximum(maximum)) + } + (DIVIDE_OP, _) => bail!( + "{at} divides by {}; {DIVIDE_OP} needs a positive number", + self.source(argument) + ), + (MAXIMUM_OP, _) => bail!( + "{at} caps at {}; {MAXIMUM_OP} needs a positive integer", + self.source(argument) + ), + (other, _) => bail!("{at} uses {other}; the ops are {DIVIDE_OP} and {MAXIMUM_OP}"), + } + } +} + +fn offset(size: TextSize) -> usize { + u32::from(size) as usize +} + +fn line_at(text: &str, offset: usize) -> usize { + text.as_bytes()[..offset.min(text.len())] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits(text: &str) -> Vec { + parse_hooks(text, "eb_hooks.py") + .unwrap_or_else(|error| panic!("{error:#}")) + .limits + } + + fn refusal(text: &str) -> String { + format!("{:#}", parse_hooks(text, "eb_hooks.py").unwrap_err()) + } + + #[test] + fn rows_resolve_constants_and_keep_the_literal_kind() { + let text = "\ +CPU_TARGET_A64FX = 'aarch64/a64fx' + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2.5), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} +"; + let rows = limits(text); + assert_eq!(rows.len(), 3, "{rows:?}"); + assert_eq!(rows[0].software, None); + assert_eq!(rows[0].target, "aarch64/a64fx"); + assert_eq!(rows[0].line, Some(10)); + let LimitOp::Divide(global) = rows[0].op else { + panic!("{:?}", rows[0].op); + }; + assert!(global.integer_literal()); + assert_eq!(global.value(), 4.0); + assert_eq!(rows[1].software.as_deref(), Some("libxc")); + assert_eq!(rows[1].target, "*"); + assert_eq!(rows[1].line, Some(12)); + let LimitOp::Divide(factor) = rows[1].op else { + panic!("{:?}", rows[1].op); + }; + assert!(!factor.integer_literal()); + assert_eq!(factor.value(), 2.5); + assert_eq!(rows[2].target, "aarch64/a64fx"); + assert_eq!(rows[2].op, LimitOp::Maximum(6)); + } + + #[test] + fn a_repeated_key_keeps_the_last_value_as_python_does() { + let text = "\ +PARALLELISM_LIMITS = { + 'libxc': {'*': (divide_by_factor, 2)}, + 'LLVM': {'*': (divide_by_factor, 2), '*': (set_maximum, 8)}, + 'libxc': {'x86_64/amd/zen4': (set_maximum, 4)}, +} +"; + let rows = limits(text); + assert_eq!(rows.len(), 2, "{rows:?}"); + assert!(rows + .iter() + .any(|row| row.software.as_deref() == Some("LLVM") && row.op == LimitOp::Maximum(8))); + let libxc: Vec<&HookLimit> = rows + .iter() + .filter(|row| row.software.as_deref() == Some("libxc")) + .collect(); + assert_eq!(libxc.len(), 1, "{rows:?}"); + assert_eq!(libxc[0].target, "x86_64/amd/zen4"); + } + + #[test] + fn the_last_assignment_wins_and_uses_the_constants_bound_before_it() { + let text = "\ +TARGET = 'aarch64/a64fx' +PARALLELISM_LIMITS = {TARGET: (divide_by_factor, 4)} +TARGET = 'x86_64/amd/zen4' +PARALLELISM_LIMITS = {TARGET: (set_maximum, 8)} +"; + let rows = limits(text); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert_eq!(rows[0].target, "x86_64/amd/zen4"); + assert_eq!(rows[0].op, LimitOp::Maximum(8)); + assert_eq!(rows[0].line, Some(4)); + } + + #[test] + fn rows_the_model_cannot_follow_are_refused_with_their_line() { + let cases = [ + ("x = (\n", "eb_hooks.py:2: not valid Python"), + ("LIMITS = {}\n", "eb_hooks.py: no module-level PARALLELISM_LIMITS"), + ( + "PARALLELISM_LIMITS = dict()\n", + "eb_hooks.py:1: PARALLELISM_LIMITS is not a dict literal", + ), + ( + "PARALLELISM_LIMITS = {\n 'libxc': {'*': (limit_by_memory, 6400)},\n}\n", + "eb_hooks.py:2: PARALLELISM_LIMITS row for libxc on * uses limit_by_memory; the ops are divide_by_factor and set_maximum", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (set_maximum, 8.0)}}\n", + "row for libxc on * caps at 8.0; set_maximum needs a positive integer", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (divide_by_factor, 0)}}\n", + "row for libxc on * divides by 0; divide_by_factor needs a positive number", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (divide_by_factor, -2)}}\n", + "row for libxc on * divides by -2; divide_by_factor needs a positive number", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {UNKNOWN: (set_maximum, 8)}}\n", + "PARALLELISM_LIMITS key UNKNOWN is not a module-level string constant", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (set_maximum, 8, 1)}}\n", + "row for libxc on * is (set_maximum, 8, 1), not (op, argument)", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {}}\n", + "PARALLELISM_LIMITS entry for libxc is empty", + ), + ( + "PARALLELISM_LIMITS = {'*': (divide_by_factor, 2)}\n", + "needs a CPU target, not '*'", + ), + ( + "PARALLELISM_LIMITS = {**OTHER}\n", + "PARALLELISM_LIMITS unpacks a dict with **", + ), + ]; + for (text, expected) in cases { + let error = refusal(text); + assert!(error.contains(expected), "{text:?}: {error}"); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index f41a5da0..76b2e237 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod easystack; pub mod eb_easyblock; mod eb_emit; pub mod eb_engine_export; +pub mod eb_hooks; pub mod eb_maintainer; pub mod eb_module; pub mod eb_parse; diff --git a/src/main.rs b/src/main.rs index 4e824c69..912f465f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1541,12 +1541,13 @@ enum StackCommand { /// the easyconfig. An estimate that reaches the job's memory in /// `data/eessi-bot-resources.toml` is a warning with remedies. A /// `divide_by_factor` that is not an integer is a warning too: Python's - /// `//` then returns a float (`16 // 2.5` is `6.0`). `--node` prints - /// the hook parallelism beside `limit_by_memory` for a core count and - /// a memory. The command does not fail on a warning. Pass `--easystack` - /// or `--node` (or both). + /// `//` then returns a float (`16 // 2.5` is `6.0`). `--hooks` reads + /// the limits from an `eb_hooks.py`. `--node` prints the hook + /// parallelism beside `limit_by_memory` for a core count and a memory. + /// The command does not fail on a warning. Pass `--easystack`, + /// `--node`, `--hooks`, or a mix. #[command( - after_long_help = "Examples:\n eb-stack stack oom --easystack eessi-2026.06-eb-5.4.0-2026.1.yml\n eb-stack stack oom --arch x86_64/amd/zen4 --easystack eessi-2026.06-eb-5.4.0-2026.1.yml --easyconfigs easybuild-easyconfigs/easybuild/easyconfigs --installed installed-zen4.txt\n eb-stack stack oom --node 16:32768:x86_64/amd/zen4 --node 48:30016:aarch64/a64fx" + after_long_help = "Examples:\n eb-stack stack oom --easystack eessi-2026.06-eb-5.4.0-2026.1.yml\n eb-stack stack oom --arch x86_64/amd/zen4 --easystack eessi-2026.06-eb-5.4.0-2026.1.yml --easyconfigs easybuild-easyconfigs/easybuild/easyconfigs --installed installed-zen4.txt\n eb-stack stack oom --node 16:32768:x86_64/amd/zen4 --node 48:30016:aarch64/a64fx\n eb-stack stack oom --hooks software-layer-scripts/eb_hooks.py --node 16:32768:x86_64/amd/zen4" )] Oom { /// CPU target, `x86_64/amd/zen4`, or a prefix such as `aarch64`. @@ -1554,7 +1555,7 @@ enum StackCommand { #[arg(long = "arch")] arch: Vec, /// Easystack whose entries are the plan. Repeatable. Required - /// unless `--node` is passed. + /// unless `--node` or `--hooks` is passed. #[arg(long)] easystack: Vec, /// Robot tree the dependency closure and `maxparallel` are read @@ -1581,6 +1582,13 @@ enum StackCommand { /// Replace the bundled per-package `mib_per_job` table. #[arg(long)] job_memory: Option, + /// EESSI `eb_hooks.py` to read `PARALLELISM_LIMITS` from, in place + /// of the hook rows of the bot resource table. The nodes and + /// EasyBuild's max-parallel still come from the table. The file is + /// parsed, not run. An op other than `divide_by_factor` or + /// `set_maximum` is refused with its line. + #[arg(long)] + hooks: Option, }, /// Say what an EESSI bot job's FAILURE was: a real failure, ERROR-only /// noise, an empty tarball, or infra. @@ -3265,6 +3273,7 @@ fn run_stack(command: StackCommand) -> Result<()> { resources, node, job_memory, + hooks, } => { let report = eb_stack::oom_profile::run_oom_check(&eb_stack::oom_profile::OomCheckRequest { @@ -3276,6 +3285,7 @@ fn run_stack(command: StackCommand) -> Result<()> { resources, nodes: node, job_memory, + hooks, })?; print!("{}", report.stdout); Ok(()) diff --git a/src/mcp.rs b/src/mcp.rs index 32e9bf24..09b05b64 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -540,7 +540,7 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_oom", - "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory. Pass easystack or node. A warning is not an error.", + "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory. Pass easystack, node, or hooks. A warning is not an error.", &[], &[ ("easystack", "array"), @@ -551,6 +551,7 @@ fn tool_catalog() -> Vec { ("resources", "string"), ("node", "array"), ("job_memory", "string"), + ("hooks", "string"), ], ), tool_with_optional( @@ -3416,6 +3417,7 @@ fn stack_oom_tool(arguments: &Value) -> Result { resources: optional_path(arguments, "resources"), nodes: string_array(arguments, "node")?, job_memory: optional_path(arguments, "job_memory"), + hooks: optional_path(arguments, "hooks"), }) .map_err(workflow_error)?; Ok(json!({ diff --git a/src/oom_profile.rs b/src/oom_profile.rs index 299d68b7..0b9ad484 100644 --- a/src/oom_profile.rs +++ b/src/oom_profile.rs @@ -13,9 +13,11 @@ //! The hook ops are `divide_by_factor` and `set_maximum` from //! `eb_hooks.py`. A divide factor that is not an integer makes Python's //! `//` return a float (`16 // 2.5` is `6.0`); the command warns and uses -//! the numeric floor for the estimate. `data/job-memory.toml` holds -//! `mib_per_job` for `limit_by_memory`. A row with no number is -//! unmeasured. `--node` prints the hook parallelism beside that rule. +//! the numeric floor for the estimate. `--hooks` reads the rows from an +//! `eb_hooks.py` instead of the table, through [`crate::eb_hooks`]. +//! `data/job-memory.toml` holds `mib_per_job` for `limit_by_memory`. A row +//! with no number is unmeasured. `--node` prints the hook parallelism +//! beside that rule. use crate::arch_support::{plan_requirers, split_easyconfig_filename}; use crate::build_order::{closure_against_installed, reverse_uses, Choice, ModuleKey}; @@ -364,14 +366,14 @@ impl std::fmt::Debug for DivideFactor { } impl DivideFactor { - fn from_integer(value: u32) -> Self { + pub(crate) fn from_integer(value: u32) -> Self { Self { bits: f64::from(value).to_bits(), integer_literal: true, } } - fn from_float(value: f64) -> Self { + pub(crate) fn from_float(value: f64) -> Self { Self { bits: value.to_bits(), integer_literal: false, @@ -519,6 +521,9 @@ pub struct HookLimit { pub target: String, /// The limit. pub op: LimitOp, + /// Line of the `(op, argument)` tuple in the `eb_hooks.py` the row was + /// read from. `None` for a row of the bot resource table. + pub line: Option, } /// The bot job table and the EESSI hook limits. @@ -688,6 +693,7 @@ impl BotResources { software: limit.software, target: limit.target, op, + line: None, }); } Ok(Self { @@ -731,7 +737,8 @@ impl BotResources { /// /// `eb_hooks.py` computes `max(1, parallel // factor)`. A non-integer /// factor makes `//` return a float: `16 // 2.5` is `6.0`. A whole - /// number written as a float does the same: `16 // 2.0` is `8.0`. + /// number written as a float does the same: `16 // 2.0` is `8.0`. A + /// row read from an `eb_hooks.py` names its file and line. pub fn factor_warnings(&self) -> Vec { let mut lines = Vec::new(); for limit in &self.limits { @@ -748,8 +755,12 @@ impl BotResources { } else { format!("{shown}, which is not an integer") }; + let at = limit + .line + .map(|line| format!("{}:{line}: ", self.hooks_source)) + .unwrap_or_default(); lines.push(format!( - "warning: PARALLELISM_LIMITS divide_by_factor for {who} on {} uses {kind}; eb_hooks.py computes max(1, parallel // factor), and {}. Use an integer factor.", + "warning: {at}PARALLELISM_LIMITS divide_by_factor for {who} on {} uses {kind}; eb_hooks.py computes max(1, parallel // factor), and {}. Use an integer factor.", limit.target, float_example(factor) )); @@ -1489,6 +1500,9 @@ pub struct OomCheckRequest { pub nodes: Vec, /// Override `data/job-memory.toml`. pub job_memory: Option, + /// An EESSI `eb_hooks.py` whose `PARALLELISM_LIMITS` replace the hook + /// rows of the bot resource table. + pub hooks: Option, } /// What `stack oom` prints. @@ -1512,16 +1526,16 @@ pub struct OomCheckReport { /// Estimate each plan build's peak on each bot node and warn where it /// reaches the job's memory. pub fn run_oom_check(request: &OomCheckRequest) -> Result { - if request.easystacks.is_empty() && request.nodes.is_empty() { + if request.easystacks.is_empty() && request.nodes.is_empty() && request.hooks.is_none() { return Err(usage( - "pass at least one --easystack, or --node to compare parallelism", + "pass at least one --easystack, --node to compare parallelism, or --hooks to check an eb_hooks.py", )); } let mut nodes = Vec::new(); for text in &request.nodes { nodes.push(parse_node_size(text).map_err(usage)?); } - let resources = match &request.resources { + let mut resources = match &request.resources { Some(path) => BotResources::load(path)?, None => BotResources::bundled()?, }; @@ -1529,9 +1543,21 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { Some(path) => JobMemoryTable::load(path)?, None => JobMemoryTable::bundled()?, }; + let mut stdout = String::new(); + if let Some(path) = &request.hooks { + let file = path.display().to_string(); + let text = std::fs::read_to_string(path).with_context(|| format!("--hooks {file}"))?; + let hooks = crate::eb_hooks::parse_hooks(&text, &file)?; + stdout.push_str(&format!( + "hooks: {} {} rows from {file}\n", + hooks.limits.len(), + crate::eb_hooks::LIMITS_NAME + )); + resources.limits = hooks.limits; + resources.hooks_source = file; + } let factor_warnings = resources.factor_warnings(); if request.easystacks.is_empty() { - let mut stdout = String::new(); for line in &factor_warnings { stdout.push_str(line); stdout.push('\n'); @@ -1623,7 +1649,6 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { } } - let mut stdout = String::new(); for arch in &unknown_arches { stdout.push_str(&format!( "{arch} has no bot node in the bot resource table, so it is not checked\n" diff --git a/src/site_commands.rs b/src/site_commands.rs index 8e5a5605..61d4e199 100644 --- a/src/site_commands.rs +++ b/src/site_commands.rs @@ -1716,6 +1716,7 @@ pub fn run_stack_batch(args: &StackBatchRequest) -> Result { resources: args.oom_resources.clone(), nodes: Vec::new(), job_memory: None, + hooks: None, })?; let oom = oom_report.warnings; let factor_warnings = oom_report.factor_warnings; diff --git a/tests/mcp_v1.rs b/tests/mcp_v1.rs index bdbac047..d3fca9a5 100644 --- a/tests/mcp_v1.rs +++ b/tests/mcp_v1.rs @@ -3792,3 +3792,27 @@ fn every_cli_subcommand_has_an_mcp_tool() { assert!(listed, "CLI subcommand {rendered} has no MCP tool"); } } + +#[test] +fn mcp_stack_oom_reads_an_eb_hooks_py() { + let hooks = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/oom_profile_fixtures/hooks/divide-2.5.py") + .display() + .to_string(); + let response = call(1, "eb_stack_oom", json!({"hooks": hooks})); + let body = &response["result"]["structuredContent"]; + let warnings = body["factor_warnings"].as_array().expect("factor_warnings"); + assert_eq!(warnings.len(), 1, "{body}"); + let warning = warnings[0].as_str().expect("warning text"); + assert!( + warning.starts_with(&format!( + "warning: {hooks}:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5" + )), + "{warning}" + ); + let text = body["text"].as_str().expect("text"); + assert!( + text.starts_with(&format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n")), + "{text}" + ); +} diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index d754fef3..42b1e33b 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -213,6 +213,7 @@ fn a_target_without_a_bot_node_is_reported_not_checked() { fn an_easystack_is_required_and_a_directory_is_refused() { let (code, _, stderr) = run(&["stack", "oom"]); assert_eq!(code, 2, "{stderr}"); + assert!(stderr.contains("--hooks"), "{stderr}"); let (code, _, stderr) = run(&["stack", "oom", "--easystack", &arg("easystacks")]); assert_eq!(code, 2, "{stderr}"); assert!(stderr.contains("is a directory"), "{stderr}"); @@ -244,6 +245,104 @@ fn a_non_integer_divide_warns_and_the_estimate_uses_six() { ); } +#[test] +fn an_eb_hooks_py_with_a_float_divider_warns_at_its_line() { + let hooks = arg("hooks/divide-2.5.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + warning: {hooks}:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor.\n\ + oom=0 fits=2 builds=5 nodes=2\n" + ) + ); +} + +#[test] +fn an_eb_hooks_py_with_integer_dividers_does_not_warn() { + let hooks = arg("hooks/integers.py"); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", &hooks]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n") + ); + + // The fixture's libxc and A64FX rows are the bundled table's, so the + // parallel levels match. + let nodes = [ + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]; + let (_, bundled, _) = run(&[&["stack", "oom"][..], &nodes].concat()); + let (code, read, stderr) = run(&[&["stack", "oom", "--hooks", &hooks][..], &nodes].concat()); + assert_eq!(code, 0, "{stderr}"); + for line in [ + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n", + " (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured\n", + ] { + assert!(bundled.contains(line), "{bundled}"); + assert!(read.contains(line), "{read}"); + } + assert!(!read.contains("warning:"), "{read}"); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains(" parallel 8: 16 cores, eb_hooks.py divides libxc by 2\n"), + "{stdout}" + ); + assert!(!stdout.contains("warning:"), "{stdout}"); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); +} + +#[test] +fn an_eb_hooks_py_row_the_model_cannot_follow_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let hooks = dir.path().join("eb_hooks.py"); + std::fs::write( + &hooks, + "PARALLELISM_LIMITS = {\n 'libxc': {'*': (limit_by_memory, 6400)},\n}\n", + ) + .unwrap(); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", hooks.to_str().unwrap()]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + assert!( + stderr.contains(&format!( + "{}:2: PARALLELISM_LIMITS row for libxc on * uses limit_by_memory; the ops are divide_by_factor and set_maximum", + hooks.display() + )), + "{stderr}" + ); + assert!(stdout.is_empty(), "{stdout}"); +} + #[test] fn node_sizes_compare_the_hook_with_the_memory_rule() { let (code, stdout, stderr) = run(&[ @@ -443,6 +542,6 @@ fn the_how_to_shows_the_real_output() { "{command}" ); } - assert_eq!(oom_blocks, 7); + assert_eq!(oom_blocks, 9); assert_eq!(rss_blocks, 1); } diff --git a/tests/oom_profile_fixtures/hooks/divide-2.5.py b/tests/oom_profile_fixtures/hooks/divide-2.5.py new file mode 100644 index 00000000..56ca96b3 --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/divide-2.5.py @@ -0,0 +1,23 @@ +# integers.py with the libxc divider the maintainers suggested in +# EESSI/software-layer#1772 (issuecomment-6420263033): 2.5 instead of 2. +# 16 // 2.5 is 6.0 in Python, a float. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2.5), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} diff --git a/tests/oom_profile_fixtures/hooks/integers.py b/tests/oom_profile_fixtures/hooks/integers.py new file mode 100644 index 00000000..1706c63a --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/integers.py @@ -0,0 +1,22 @@ +# PARALLELISM_LIMITS in the layout of software-layer-scripts eb_hooks.py, +# cut down to the A64FX row and libxc. Every divide_by_factor is an integer. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} From 71021bdc209f86171907896f864cc225b275a70e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:10:27 +0000 Subject: [PATCH 2/7] feat(stack): report the dropped msg+ sum in eb_hooks.py In software-layer-scripts eb_hooks.py, the post_ready_hook builds its log line in two statements. Since 8132905f the second is `msg+ "..."` on a line of its own. That computes the longer string and drops it, so the "to avoid out-of-memory failures" half of the message never prints. The `msg+` statement in eb_hooks.py drops its string; the fix belongs upstream. With `--hooks`, the `stack oom` command now walks the parsed file. It looks for an expression statement whose value is `name + ...`. It also looks inside def, class, loop, if, with, match and try bodies. It prints a warning with the file, the line, the statement and the enclosing function. The form `msg += "..."` is an augmented assignment and is not reported. The warning changes no parallel level and not the exit status. The `eb_stack_oom` tool returns these as hook_warnings. The fixtures `hooks/msg-plus.py` and `hooks/msg-plus-equals.py` are the `integers.py` rows with a post_ready_hook in each form. On upstream software-layer-scripts main (43df6f4) the warning is eb_hooks.py:440. --- CHANGELOG.md | 6 + docs/orgmode/howto/oom-profile.org | 22 ++- docs/orgmode/reference/cli.org | 10 +- llms-full.txt | 32 +++- skills/eessi-extend/SKILL.md | 1 + src/eb_hooks.rs | 170 +++++++++++++++++- src/main.rs | 6 +- src/mcp.rs | 3 +- src/oom_profile.rs | 15 +- tests/mcp_v1.rs | 18 ++ tests/oom_profile.rs | 49 ++++- .../hooks/msg-plus-equals.py | 28 +++ tests/oom_profile_fixtures/hooks/msg-plus.py | 31 ++++ 13 files changed, 375 insertions(+), 16 deletions(-) create mode 100644 tests/oom_profile_fixtures/hooks/msg-plus-equals.py create mode 100644 tests/oom_profile_fixtures/hooks/msg-plus.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a10609a7..e7428ff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,12 @@ All notable changes to this project are documented here. file and line (`16 // 2.5` is `6.0`, a float). An op other than `divide_by_factor` or `set_maximum`, a `**` unpacking, or a key that is not a string exits 1 with the line. `eb_stack_oom` takes `hooks`. +- `stack oom --hooks FILE` warns on a statement that adds to a name and + drops the sum, such as `msg+ "..."` in the EESSI `post_ready_hook`: + the second half of its log message never prints. The warning names + the file, the line, and the function; `msg += "..."` is not reported, + and the parallel levels do not change. `eb_stack_oom` returns these + as `hook_warnings`. ### Documentation diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index 7beeeab6..a32c1f03 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -198,7 +198,8 @@ msg+ "to avoid out-of-memory failures during building/testing" The second statement adds nothing. ~msg + "..."~ is an expression, and the result is thrown away. The operator that appends is ~+=~. The line that prints stops after the CPU target. This is upstream ~eb_hooks.py~. -eb-stack does not patch it. +eb-stack does not patch it. ~--hooks~ reports it with its line (see +[[*Read the limits from eb_hooks.py][below]]). * A divide factor that is not an integer @@ -271,6 +272,23 @@ Python. An op other than ~divide_by_factor~ or ~set_maximum~, a ~**~ unpacking, or a key that is not a string stops the command with the file and the line: the parallel levels would be wrong without that row. +The same file is checked for a statement that adds to a name and drops +the sum, such as the ~msg+~ in ~post_ready_hook~. The fixture +~hooks/msg-plus.py~ has that statement: + +#+begin_src bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/msg-plus.py +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/msg-plus.py +warning: tests/oom_profile_fixtures/hooks/msg-plus.py:30: msg+ "to avoid out-of-memory failures during building/testing" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends +#+end_example + +The warning names the function and the line. It does not change a +parallel level, and the command still exits 0. ~msg += "..."~, as in +~hooks/msg-plus-equals.py~, appends and is not reported. + * A memory limit instead of a core fraction A core fraction does not travel. Half of 16 cores and half of 64 cores @@ -415,6 +433,8 @@ then. - ~--hooks~ reads the ~PARALLELISM_LIMITS~ literal. It does not run ~post_ready_hook~; the rules that pick a row are the ones described under [[*Where the numbers come from][where the numbers come from]]. + Of the rest of the file it checks only for a statement that drops a + sum. Other mistakes in the hooks are not found. - ~limit_by_memory~ does not replace the estimate. The estimate follows the hooks that are in ~eb_hooks.py~. The ~--node~ report is the comparison. A package with no measured ~mib_per_job~ stays diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 7b1286b9..7bc47bcf 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -1164,7 +1164,10 @@ value, as in Python. An op other than ~divide_by_factor~ or refused with its line: the parallel levels would be wrong without that row. The output starts with ~hooks: N PARALLELISM_LIMITS rows from FILE~, and a ~divide~ that is not an integer is a warning that names the -file and the line. Pass ~--easystack~, ~--node~, ~--hooks~, or a mix. +file and the line. A statement that adds to a name and drops the sum, +such as ~msg+ "..."~ in ~post_ready_hook~, is a warning that names the +file, the line, and the function; ~msg += "..."~ is not. Pass +~--easystack~, ~--node~, ~--hooks~, or a mix. The command exits 0 when it warns. See [[file:../howto/oom-profile.org][warn before a bot job runs out of memory]]. @@ -1812,8 +1815,9 @@ unless ~strict~ is true, when a remaining warning is. ~eb_stack_oom~ takes ~easystack~, ~arch~, ~easyconfigs~, ~installed~, ~profile~, ~resources~, ~node~, ~job_memory~, and ~hooks~. It needs an easystack, a node, or hooks. It returns the ~stack oom~ -text, one object per warning, and ~factor_warnings~ for a divide factor -that is not an integer. A warning is not a tool error. +text, one object per warning, ~factor_warnings~ for a divide factor +that is not an integer, and ~hook_warnings~ for a dropped sum in the +~hooks~ file. A warning is not a tool error. ~eb_stack_bot_classify~ requires ~files~. It returns ~text~, the ~stack bot-classify~ stdout, ~jobs~, one object per job, and ~counts~. ~eb_stack_batch~ takes ~arch~, ~unsupported~, ~known_failures~, diff --git a/llms-full.txt b/llms-full.txt index 78e03f2c..ab6f488e 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4791,7 +4791,8 @@ msg+ "to avoid out-of-memory failures during building/testing" The second statement adds nothing. `msg + "..."` is an expression, and the result is thrown away. The operator that appends is `+=`. The line that prints stops after the CPU target. This is upstream `eb_hooks.py`. -eb-stack does not patch it. +eb-stack does not patch it. `--hooks` reports it with its line (see +[[*Read the limits from eb_hooks.py][below]]). # A divide factor that is not an integer @@ -4864,6 +4865,23 @@ Python. An op other than `divide_by_factor` or `set_maximum`, a `**` unpacking, or a key that is not a string stops the command with the file and the line: the parallel levels would be wrong without that row. +The same file is checked for a statement that adds to a name and drops +the sum, such as the `msg+` in `post_ready_hook`. The fixture +`hooks/msg-plus.py` has that statement: + +```bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/msg-plus.py +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/msg-plus.py +warning: tests/oom_profile_fixtures/hooks/msg-plus.py:30: msg+ "to avoid out-of-memory failures during building/testing" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends +``` + +The warning names the function and the line. It does not change a +parallel level, and the command still exits 0. `msg += "..."`, as in +`hooks/msg-plus-equals.py`, appends and is not reported. + # A memory limit instead of a core fraction A core fraction does not travel. Half of 16 cores and half of 64 cores @@ -5008,6 +5026,8 @@ then. - `--hooks` reads the `PARALLELISM_LIMITS` literal. It does not run `post_ready_hook`; the rules that pick a row are the ones described under [[*Where the numbers come from][where the numbers come from]]. + Of the rest of the file it checks only for a statement that drops a + sum. Other mistakes in the hooks are not found. - `limit_by_memory` does not replace the estimate. The estimate follows the hooks that are in `eb_hooks.py`. The `--node` report is the comparison. A package with no measured `mib_per_job` stays @@ -7049,7 +7069,10 @@ value, as in Python. An op other than `divide_by_factor` or refused with its line: the parallel levels would be wrong without that row. The output starts with ~hooks: N PARALLELISM_LIMITS rows from FILE`, and a `divide~ that is not an integer is a warning that names the -file and the line. Pass `--easystack`, `--node`, `--hooks`, or a mix. +file and the line. A statement that adds to a name and drops the sum, +such as `msg+ "..."` in `post_ready_hook`, is a warning that names the +file, the line, and the function; `msg += "..."` is not. Pass +`--easystack`, `--node`, `--hooks`, or a mix. The command exits 0 when it warns. See [warn before a bot job runs out of memory](https://eb-stack.rgoswami.me/../howto/oom-profile.html.md). @@ -7697,8 +7720,9 @@ unless `strict` is true, when a remaining warning is. `eb_stack_oom` takes `easystack`, `arch`, `easyconfigs`, `installed`, `profile`, `resources`, `node`, `job_memory`, and `hooks`. It needs an easystack, a node, or hooks. It returns the `stack oom` -text, one object per warning, and `factor_warnings` for a divide factor -that is not an integer. A warning is not a tool error. +text, one object per warning, `factor_warnings` for a divide factor +that is not an integer, and `hook_warnings` for a dropped sum in the +`hooks` file. A warning is not a tool error. `eb_stack_bot_classify` requires `files`. It returns `text`, the `stack bot-classify` stdout, `jobs`, one object per job, and `counts`. `eb_stack_batch` takes `arch`, `unsupported`, `known_failures`, diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index 7002874b..9feeef57 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -345,6 +345,7 @@ integer is a warning: `16 // 2.5` is the float `6.0`. Use an integer. from the file instead of the bundled copy, and the warning names the line. `eb_hooks.py` also has a `msg+` where the log line should use `+=`; half the message never prints. That is upstream. Do not patch it from here. +`--hooks` prints it as a warning with the line and `post_ready_hook`. `eb-stack stack oom --node 16:32768:x86_64/amd/zen4` compares the hook with `limit_by_memory`. `data/job-memory.toml` holds `mib_per_job`. diff --git a/src/eb_hooks.rs b/src/eb_hooks.rs index cdbcd18c..5b0bf27f 100644 --- a/src/eb_hooks.rs +++ b/src/eb_hooks.rs @@ -9,6 +9,10 @@ //! The file is parsed, never run. A row this reader cannot model is an //! error that names its line, because the parallel levels would be wrong //! without it. +//! +//! The same file is checked for a statement that adds to a name and drops +//! the result: `msg+ "..."` on a line of its own builds a longer string and +//! throws it away, where `msg += "..."` appends. use crate::oom_profile::{DivideFactor, HookLimit, LimitOp}; use anyhow::{anyhow, bail, Result}; @@ -32,9 +36,39 @@ pub struct HooksFile { /// `PARALLELISM_LIMITS` rows. A key the dict repeats keeps its last /// value, as Python does. pub limits: Vec, + /// Statements that add to a name and drop the sum. + pub dropped_sums: Vec, +} + +/// `name + ...` as a statement of its own: the sum is computed and lost. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DroppedSum { + /// 1-based line of the statement. + pub line: usize, + /// The innermost `def` around the statement. `None` at module level. + pub function: Option, + /// The name left of `+`, `msg`. + pub name: String, + /// The statement's first source line, trimmed. + pub statement: String, } -/// Read `PARALLELISM_LIMITS` from `text`. +impl DroppedSum { + /// The warning `stack oom` prints, with `file` as the location. + pub fn message(&self, file: &str) -> String { + let place = match &self.function { + Some(function) => format!("in {function}"), + None => "at module level".to_string(), + }; + let name = &self.name; + format!( + "warning: {file}:{}: {} {place} adds to {name} and drops the result, so {name} does not change; {name} += appends", + self.line, self.statement + ) + } +} + +/// Read `PARALLELISM_LIMITS` and the dropped sums from `text`. /// /// `file` names the file in errors and in each row's location. The last /// module-level assignment to `PARALLELISM_LIMITS` is the one @@ -63,7 +97,12 @@ pub fn parse_hooks(text: &str, file: &str) -> Result { ); }; let limits = reader.limits(dict)?; - Ok(HooksFile { limits }) + let mut dropped_sums = Vec::new(); + collect_dropped_sums(&suite, None, text, &mut dropped_sums); + Ok(HooksFile { + limits, + dropped_sums, + }) } /// The last module-level `PARALLELISM_LIMITS = ...`, and the string @@ -283,6 +322,89 @@ impl Reader<'_> { } } +/// Every `name + ...` expression statement, in source order. +fn collect_dropped_sums( + statements: &[ast::Stmt], + function: Option<&str>, + text: &str, + out: &mut Vec, +) { + for statement in statements { + match statement { + ast::Stmt::Expr(expr) => { + let ast::Expr::BinOp(sum) = expr.value.as_ref() else { + continue; + }; + let ast::Expr::Name(name) = sum.left.as_ref() else { + continue; + }; + if !matches!(sum.op, ast::Operator::Add) { + continue; + } + let range = expr.range(); + let source = text + .get(offset(range.start())..offset(range.end())) + .unwrap_or(""); + out.push(DroppedSum { + line: line_at(text, offset(range.start())), + function: function.map(str::to_string), + name: name.id.to_string(), + statement: source.lines().next().unwrap_or("").trim().to_string(), + }); + } + ast::Stmt::FunctionDef(def) => { + collect_dropped_sums(&def.body, Some(def.name.as_str()), text, out); + } + ast::Stmt::AsyncFunctionDef(def) => { + collect_dropped_sums(&def.body, Some(def.name.as_str()), text, out); + } + ast::Stmt::ClassDef(class) => collect_dropped_sums(&class.body, function, text, out), + ast::Stmt::For(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::AsyncFor(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::While(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::If(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::With(block) => collect_dropped_sums(&block.body, function, text, out), + ast::Stmt::AsyncWith(block) => collect_dropped_sums(&block.body, function, text, out), + ast::Stmt::Match(block) => { + for case in &block.cases { + collect_dropped_sums(&case.body, function, text, out); + } + } + ast::Stmt::Try(block) => { + collect_dropped_sums(&block.body, function, text, out); + for handler in &block.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + collect_dropped_sums(&handler.body, function, text, out); + } + collect_dropped_sums(&block.orelse, function, text, out); + collect_dropped_sums(&block.finalbody, function, text, out); + } + ast::Stmt::TryStar(block) => { + collect_dropped_sums(&block.body, function, text, out); + for handler in &block.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + collect_dropped_sums(&handler.body, function, text, out); + } + collect_dropped_sums(&block.orelse, function, text, out); + collect_dropped_sums(&block.finalbody, function, text, out); + } + _ => {} + } + } +} + fn offset(size: TextSize) -> usize { u32::from(size) as usize } @@ -438,4 +560,48 @@ PARALLELISM_LIMITS = {TARGET: (set_maximum, 8)} assert!(error.contains(expected), "{text:?}: {error}"); } } + + #[test] + fn a_sum_statement_is_reported_and_an_augmented_assignment_is_not() { + let text = "\ +def post_ready_hook(self): + msg = 'limiting parallelism to %s ' + msg+ 'to avoid out-of-memory failures' + note = 'a' + note += 'b' + if self: + try: + pass + except OSError: + note + 'c' + return msg + +total = 1 +total + 2 +'''A docstring is an expression statement too, and is not a sum.''' +PARALLELISM_LIMITS = {} +"; + let hooks = parse_hooks(text, "hooks.py").unwrap(); + assert!(hooks.limits.is_empty()); + let found: Vec<(usize, Option<&str>, &str)> = hooks + .dropped_sums + .iter() + .map(|sum| (sum.line, sum.function.as_deref(), sum.name.as_str())) + .collect(); + assert_eq!( + found, + [ + (3, Some("post_ready_hook"), "msg"), + (10, Some("post_ready_hook"), "note"), + (14, None, "total"), + ] + ); + assert_eq!( + hooks.dropped_sums[0].message("hooks.py"), + "warning: hooks.py:3: msg+ 'to avoid out-of-memory failures' in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends" + ); + assert!(hooks.dropped_sums[2] + .message("hooks.py") + .contains("total + 2 at module level adds to total")); + } } diff --git a/src/main.rs b/src/main.rs index 912f465f..fbff7810 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1542,7 +1542,8 @@ enum StackCommand { /// `data/eessi-bot-resources.toml` is a warning with remedies. A /// `divide_by_factor` that is not an integer is a warning too: Python's /// `//` then returns a float (`16 // 2.5` is `6.0`). `--hooks` reads - /// the limits from an `eb_hooks.py`. `--node` prints the hook + /// the limits from an `eb_hooks.py` and also warns on a statement such + /// as `msg+ "..."` that drops its sum. `--node` prints the hook /// parallelism beside `limit_by_memory` for a core count and a memory. /// The command does not fail on a warning. Pass `--easystack`, /// `--node`, `--hooks`, or a mix. @@ -1586,7 +1587,8 @@ enum StackCommand { /// of the hook rows of the bot resource table. The nodes and /// EasyBuild's max-parallel still come from the table. The file is /// parsed, not run. An op other than `divide_by_factor` or - /// `set_maximum` is refused with its line. + /// `set_maximum` is refused with its line. A statement that adds to + /// a name and drops the sum, such as `msg+ "..."`, is a warning. #[arg(long)] hooks: Option, }, diff --git a/src/mcp.rs b/src/mcp.rs index 09b05b64..4e859057 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -540,7 +540,7 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_oom", - "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory. Pass easystack, node, or hooks. A warning is not an error.", + "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows; a statement such as msg+ \"...\" that drops its sum is returned in hook_warnings. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory. Pass easystack, node, or hooks. A warning is not an error.", &[], &[ ("easystack", "array"), @@ -3442,6 +3442,7 @@ fn stack_oom_tool(arguments: &Value) -> Result { "nodes": report.partitions, "unknown_arch": report.unknown_arches, "factor_warnings": report.factor_warnings, + "hook_warnings": report.hook_warnings, })) } diff --git a/src/oom_profile.rs b/src/oom_profile.rs index 0b9ad484..7a2c7b17 100644 --- a/src/oom_profile.rs +++ b/src/oom_profile.rs @@ -1521,6 +1521,9 @@ pub struct OomCheckReport { pub unknown_arches: Vec, /// Divide factors that are not integers. Also included in `stdout`. pub factor_warnings: Vec, + /// `--hooks` statements that add to a name and drop the sum, such as + /// `msg+ "..."`. Also included in `stdout`. + pub hook_warnings: Vec, } /// Estimate each plan build's peak on each bot node and warn where it @@ -1544,6 +1547,7 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { None => JobMemoryTable::bundled()?, }; let mut stdout = String::new(); + let mut hook_warnings = Vec::new(); if let Some(path) = &request.hooks { let file = path.display().to_string(); let text = std::fs::read_to_string(path).with_context(|| format!("--hooks {file}"))?; @@ -1553,12 +1557,17 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { hooks.limits.len(), crate::eb_hooks::LIMITS_NAME )); + hook_warnings = hooks + .dropped_sums + .iter() + .map(|sum| sum.message(&file)) + .collect(); resources.limits = hooks.limits; resources.hooks_source = file; } let factor_warnings = resources.factor_warnings(); if request.easystacks.is_empty() { - for line in &factor_warnings { + for line in factor_warnings.iter().chain(&hook_warnings) { stdout.push_str(line); stdout.push('\n'); } @@ -1573,6 +1582,7 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { partitions: 0, unknown_arches: Vec::new(), factor_warnings, + hook_warnings, }); } let profile = match &request.profile { @@ -1654,7 +1664,7 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { "{arch} has no bot node in the bot resource table, so it is not checked\n" )); } - for line in &factor_warnings { + for line in factor_warnings.iter().chain(&hook_warnings) { stdout.push_str(line); stdout.push('\n'); } @@ -1683,6 +1693,7 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { partitions: partitions.len(), unknown_arches, factor_warnings, + hook_warnings, }) } diff --git a/tests/mcp_v1.rs b/tests/mcp_v1.rs index d3fca9a5..4cf9a18f 100644 --- a/tests/mcp_v1.rs +++ b/tests/mcp_v1.rs @@ -3815,4 +3815,22 @@ fn mcp_stack_oom_reads_an_eb_hooks_py() { text.starts_with(&format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n")), "{text}" ); + assert_eq!(body["hook_warnings"], json!([]), "{body}"); +} + +#[test] +fn mcp_stack_oom_returns_the_dropped_msg_sum() { + let hooks = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/oom_profile_fixtures/hooks/msg-plus.py") + .display() + .to_string(); + let response = call(1, "eb_stack_oom", json!({"hooks": hooks})); + let body = &response["result"]["structuredContent"]; + let warning = format!( + "warning: {hooks}:30: msg+ \"to avoid out-of-memory failures during building/testing\" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends" + ); + assert_eq!(body["hook_warnings"], json!([warning]), "{body}"); + assert_eq!(body["factor_warnings"], json!([]), "{body}"); + let text = body["text"].as_str().expect("text"); + assert!(text.ends_with(&format!("{warning}\n")), "{text}"); } diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index 42b1e33b..8af4ccf8 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -343,6 +343,53 @@ fn an_eb_hooks_py_row_the_model_cannot_follow_is_refused() { assert!(stdout.is_empty(), "{stdout}"); } +#[test] +fn a_dropped_msg_sum_in_eb_hooks_py_warns_at_its_line() { + let hooks = arg("hooks/msg-plus.py"); + let warning = format!( + "warning: {hooks}:30: msg+ \"to avoid out-of-memory failures during building/testing\" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends\n" + ); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", &hooks]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n{warning}") + ); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.starts_with(&format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n{warning}libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4" + )), + "{stdout}" + ); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); +} + +#[test] +fn an_eb_hooks_py_that_appends_with_plus_equals_does_not_warn() { + let hooks = arg("hooks/msg-plus-equals.py"); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", &hooks]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n") + ); +} + #[test] fn node_sizes_compare_the_hook_with_the_memory_rule() { let (code, stdout, stderr) = run(&[ @@ -542,6 +589,6 @@ fn the_how_to_shows_the_real_output() { "{command}" ); } - assert_eq!(oom_blocks, 9); + assert_eq!(oom_blocks, 10); assert_eq!(rss_blocks, 1); } diff --git a/tests/oom_profile_fixtures/hooks/msg-plus-equals.py b/tests/oom_profile_fixtures/hooks/msg-plus-equals.py new file mode 100644 index 00000000..6da7e6d6 --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/msg-plus-equals.py @@ -0,0 +1,28 @@ +# msg-plus.py with the log message appended by msg += "...". +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} + + +def post_ready_hook(self, *args, **kwargs): + new_parallel = 8 + msg = "limiting parallelism to %s for %s " + msg += "to avoid out-of-memory failures during building/testing" + print(msg % (new_parallel, self.name)) diff --git a/tests/oom_profile_fixtures/hooks/msg-plus.py b/tests/oom_profile_fixtures/hooks/msg-plus.py new file mode 100644 index 00000000..afe7292e --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/msg-plus.py @@ -0,0 +1,31 @@ +# integers.py with a post_ready_hook that ends its log message the way +# software-layer-scripts eb_hooks.py does since 8132905f: msg+ "..." on a +# line of its own. That builds the longer string and drops it, so the +# second half of the message never prints. msg += "..." appends. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} + + +def post_ready_hook(self, *args, **kwargs): + new_parallel = 8 + msg = "limiting parallelism to %s for %s " + msg+ "to avoid out-of-memory failures during building/testing" + print(msg % (new_parallel, self.name)) From 65274c205e42145984b2f41bd589da1b26fdfac6 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:16:07 +0000 Subject: [PATCH 3/7] feat(stack): show the memory cap next to the core cap in stack oom The memory cap of a package on a node is the level the hook leaves, the core cap, capped at the number of compiles of mib_per_job MiB that fit in the node's memory: `min(core cap, memory // mib_per_job)`. The `stack oom --node` command printed the hook's level and limit_by_memory in place of the hook, `max(1, min(cores, memory // mib_per_job))`, but not the hook's level with a memory ceiling on top. For a package with an mib_per_job, each `--node` line now carries "memory cap N (min(current, memory_mib // mib_per_job))" right after the current clause. The cap is not raised to 1: 0 is printed with "not one N MiB job fits", because one compile then needs more than the node has. Flooring the bytes to MiB first does not change the quotient, so the cap is also `memory_bytes // (mib_per_job * 2**20)`; a unit test checks that identity. Packages with no number print as before, so the bundled table, where libxc 7.0 is unmeasured, gives the same report. The `--mib-per-job FILE` option is a visible alias of `--job-memory FILE`, the table the cap reads. The Model Context Protocol tool keeps job_memory. --- CHANGELOG.md | 8 ++++ docs/orgmode/howto/oom-profile.org | 44 +++++++++++++++++++-- docs/orgmode/reference/cli.org | 16 +++++--- llms-full.txt | 60 +++++++++++++++++++++++++---- skills/eessi-extend/SKILL.md | 4 +- src/main.rs | 14 +++++-- src/mcp.rs | 2 +- src/oom_profile.rs | 60 +++++++++++++++++++++++++++-- tests/mcp_v1.rs | 20 ++++++++++ tests/oom_profile.rs | 61 +++++++++++++++++++++++++++++- 10 files changed, 262 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7428ff7..3e3e1bda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,14 @@ All notable changes to this project are documented here. the file, the line, and the function; `msg += "..."` is not reported, and the parallel levels do not change. `eb_stack_oom` returns these as `hook_warnings`. +- `stack oom --node` prints the memory cap beside the level the hook + leaves, for a package with an `mib_per_job`: + `min(current, memory_mib // mib_per_job)`, the same quotient as + `memory_bytes // (mib_per_job * 2**20)`. It is not raised to 1, so + `memory cap 0 (...), not one N MiB job fits` says one compile needs + more than the node has. `--mib-per-job FILE` is a visible alias of + `--job-memory FILE`. The `memory` column, `limit_by_memory` in place of + the hook, is unchanged. ### Documentation diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index a32c1f03..73c6b03f 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -363,6 +363,43 @@ that name. The footer names the rows that are unmeasured. libxc 7.0 is the one the table calls out; the other names are absent from the table, which is the same as unmeasured. +With a number in the table, the line also prints the memory cap: the +level the hook leaves, with a memory ceiling on top, +~min(current, memory_mib // mib_per_job)~. The memory is a whole number +of MiB, so the quotient is the same as +~memory_bytes // (mib_per_job * 2**20)~. ~--mib-per-job FILE~ is the +same flag as ~--job-memory FILE~. The fixture +~job-memory-fixture.toml~ gives libxc 8192 MiB, which is arithmetic and +not a measurement. With ~hooks/integers.py~ the report is libxc alone: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --mib-per-job tests/oom_profile_fixtures/job-memory-fixture.toml \ + --node 16:32768:x86_64/amd/zen4 \ + --node 16:131072:x86_64/amd/zen4 \ + --node 16:4096:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192))) +node 16 cores, 131072 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 8 (min(8, 131072 // 8192)); memory 16 (max(1, min(16, 131072 // 8192))) +node 16 cores, 4096 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192))) +job memory: every package in the table has an mib_per_job +#+end_example + +On 32 GiB the memory allows four 8192 MiB jobs and the hook allows 8, +so the memory cap is 4. On 128 GiB the memory allows 16, and the cap +stays at the hook's 8; ~limit_by_memory~ in place of the hook would +give 16. On 4 GiB not one job fits. ~limit_by_memory~ still says 1, +because it never goes below one job. The memory cap is 0: one compile +already needs more than the node has. + * Measure the worst compile ~mib_per_job~ comes from a compiler launcher, not from a guess. @@ -435,9 +472,10 @@ then. under [[*Where the numbers come from][where the numbers come from]]. Of the rest of the file it checks only for a statement that drops a sum. Other mistakes in the hooks are not found. -- ~limit_by_memory~ does not replace the estimate. The estimate follows - the hooks that are in ~eb_hooks.py~. The ~--node~ report is the - comparison. A package with no measured ~mib_per_job~ stays +- ~limit_by_memory~ and the memory cap do not replace the estimate. The + estimate follows the hooks that are in ~eb_hooks.py~. The ~--node~ + report is the comparison, and the warnings for an easystack do not + print the cap. A package with no measured ~mib_per_job~ stays unmeasured. - The MCP tool ~eb_stack_oom~ returns the same lines and each warning as JSON. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 7bc47bcf..2407c125 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -1143,14 +1143,20 @@ already installed. ~--arch~ repeats and takes a target or a prefix such as ~aarch64~. No ~--arch~ checks every node in the table; a target with no node is reported and skipped. ~--profile~ replaces ~data/build-memory.toml~. ~--resources~ replaces -~data/eessi-bot-resources.toml~. ~--job-memory~ replaces -~data/job-memory.toml~, the per-package ~mib_per_job~ table. A row with -no number is unmeasured. libxc 7.0 is unmeasured; the peak RSS of its -worst compile has not been measured, and the table does not guess. +~data/eessi-bot-resources.toml~. ~--job-memory~, also spelled +~--mib-per-job~, replaces ~data/job-memory.toml~, the per-package +~mib_per_job~ table. A row with no number is unmeasured. libxc 7.0 is +unmeasured; the peak RSS of its worst compile has not been measured, +and the table does not guess. ~--node~ repeats. Each value is ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~, and memory is MiB. It prints the parallelism the current hook leaves beside ~limit_by_memory(parallel, mib_per_job)~, -which is ~max(1, min(parallel, memory_mib // mib_per_job))~. The memory +which is ~max(1, min(parallel, memory_mib // mib_per_job))~. For a +package with a number, the memory cap +~min(current, memory_mib // mib_per_job)~ follows the hook's level: the +core cap with a memory ceiling on top. It is not raised to 1, so +~memory cap 0~ with ~not one N MiB job fits~ says one compile needs more +than the node's memory. The memory a hook would read on the build machine is the smallest cgroup v2 ~memory.max~ on the process's cgroup path, else ~MemTotal~. diff --git a/llms-full.txt b/llms-full.txt index ab6f488e..ba9dff75 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4956,6 +4956,43 @@ that name. The footer names the rows that are unmeasured. libxc 7.0 is the one the table calls out; the other names are absent from the table, which is the same as unmeasured. +With a number in the table, the line also prints the memory cap: the +level the hook leaves, with a memory ceiling on top, +`min(current, memory_mib // mib_per_job)`. The memory is a whole number +of MiB, so the quotient is the same as +`memory_bytes // (mib_per_job * 2**20)`. `--mib-per-job FILE` is the +same flag as `--job-memory FILE`. The fixture +`job-memory-fixture.toml` gives libxc 8192 MiB, which is arithmetic and +not a measurement. With `hooks/integers.py` the report is libxc alone: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --mib-per-job tests/oom_profile_fixtures/job-memory-fixture.toml \ + --node 16:32768:x86_64/amd/zen4 \ + --node 16:131072:x86_64/amd/zen4 \ + --node 16:4096:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192))) +node 16 cores, 131072 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 8 (min(8, 131072 // 8192)); memory 16 (max(1, min(16, 131072 // 8192))) +node 16 cores, 4096 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192))) +job memory: every package in the table has an mib_per_job +``` + +On 32 GiB the memory allows four 8192 MiB jobs and the hook allows 8, +so the memory cap is 4. On 128 GiB the memory allows 16, and the cap +stays at the hook's 8; `limit_by_memory` in place of the hook would +give 16. On 4 GiB not one job fits. `limit_by_memory` still says 1, +because it never goes below one job. The memory cap is 0: one compile +already needs more than the node has. + # Measure the worst compile `mib_per_job` comes from a compiler launcher, not from a guess. @@ -5028,9 +5065,10 @@ then. under [[*Where the numbers come from][where the numbers come from]]. Of the rest of the file it checks only for a statement that drops a sum. Other mistakes in the hooks are not found. -- `limit_by_memory` does not replace the estimate. The estimate follows - the hooks that are in `eb_hooks.py`. The `--node` report is the - comparison. A package with no measured `mib_per_job` stays +- `limit_by_memory` and the memory cap do not replace the estimate. The + estimate follows the hooks that are in `eb_hooks.py`. The `--node` + report is the comparison, and the warnings for an easystack do not + print the cap. A package with no measured `mib_per_job` stays unmeasured. - The MCP tool `eb_stack_oom` returns the same lines and each warning as JSON. @@ -7048,14 +7086,20 @@ already installed. `--arch` repeats and takes a target or a prefix such as `aarch64`. No `--arch` checks every node in the table; a target with no node is reported and skipped. `--profile` replaces `data/build-memory.toml`. `--resources` replaces -`data/eessi-bot-resources.toml`. `--job-memory` replaces -`data/job-memory.toml`, the per-package `mib_per_job` table. A row with -no number is unmeasured. libxc 7.0 is unmeasured; the peak RSS of its -worst compile has not been measured, and the table does not guess. +`data/eessi-bot-resources.toml`. `--job-memory`, also spelled +`--mib-per-job`, replaces `data/job-memory.toml`, the per-package +`mib_per_job` table. A row with no number is unmeasured. libxc 7.0 is +unmeasured; the peak RSS of its worst compile has not been measured, +and the table does not guess. `--node` repeats. Each value is `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET`, and memory is MiB. It prints the parallelism the current hook leaves beside `limit_by_memory(parallel, mib_per_job)`, -which is `max(1, min(parallel, memory_mib // mib_per_job))`. The memory +which is `max(1, min(parallel, memory_mib // mib_per_job))`. For a +package with a number, the memory cap +`min(current, memory_mib // mib_per_job)` follows the hook's level: the +core cap with a memory ceiling on top. It is not raised to 1, so +`memory cap 0` with `not one N MiB job fits` says one compile needs more +than the node's memory. The memory a hook would read on the build machine is the smallest cgroup v2 `memory.max` on the process's cgroup path, else `MemTotal`. diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index 9feeef57..c3fe2d74 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -348,7 +348,9 @@ the message never prints. That is upstream. Do not patch it from here. `--hooks` prints it as a warning with the line and `post_ready_hook`. `eb-stack stack oom --node 16:32768:x86_64/amd/zen4` compares the hook -with `limit_by_memory`. `data/job-memory.toml` holds `mib_per_job`. +with `limit_by_memory`. `data/job-memory.toml` holds `mib_per_job`; +`--mib-per-job FILE` replaces it, and a package with a number gets the +memory cap `min(current, memory // mib_per_job)` beside the hook's level. libxc is unmeasured. Measure a compile with `scripts/compile_rss.py` under `-O2 -ftree-vectorize -march= -fno-math-errno`, and do not invent the number. The how-to is `docs/orgmode/howto/oom-profile.org`. diff --git a/src/main.rs b/src/main.rs index fbff7810..e19e3c94 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1544,7 +1544,9 @@ enum StackCommand { /// `//` then returns a float (`16 // 2.5` is `6.0`). `--hooks` reads /// the limits from an `eb_hooks.py` and also warns on a statement such /// as `msg+ "..."` that drops its sum. `--node` prints the hook - /// parallelism beside `limit_by_memory` for a core count and a memory. + /// parallelism beside `limit_by_memory` for a core count and a memory, + /// and with an `mib_per_job` the memory cap + /// `min(current, memory // mib_per_job)`. /// The command does not fail on a warning. Pass `--easystack`, /// `--node`, `--hooks`, or a mix. #[command( @@ -1575,13 +1577,17 @@ enum StackCommand { #[arg(long)] resources: Option, /// Node size `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET`. - /// Repeatable. Memory is MiB. Prints the hook parallelism and + /// Repeatable. Memory is MiB. Prints the hook parallelism, its + /// memory cap when the package has an `mib_per_job`, and /// `limit_by_memory` for each size. With no `--easystack`, this /// report is the whole output. #[arg(long)] node: Vec, - /// Replace the bundled per-package `mib_per_job` table. - #[arg(long)] + /// Replace the bundled per-package `mib_per_job` table, + /// `data/job-memory.toml`. For a package with a number, `--node` + /// prints the memory cap `min(current, memory // mib_per_job)` + /// beside the hook's level. + #[arg(long, visible_alias = "mib-per-job")] job_memory: Option, /// EESSI `eb_hooks.py` to read `PARALLELISM_LIMITS` from, in place /// of the hook rows of the bot resource table. The nodes and diff --git a/src/mcp.rs b/src/mcp.rs index 4e859057..1f3bfe55 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -540,7 +540,7 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_oom", - "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows; a statement such as msg+ \"...\" that drops its sum is returned in hook_warnings. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory. Pass easystack, node, or hooks. A warning is not an error.", + "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows; a statement such as msg+ \"...\" that drops its sum is returned in hook_warnings. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory; for a package with an mib_per_job in job_memory (the CLI's --job-memory or --mib-per-job) it also prints the memory cap min(current, memory // mib_per_job), where 0 means not one job fits. Pass easystack, node, or hooks. A warning is not an error.", &[], &[ ("easystack", "array"), diff --git a/src/oom_profile.rs b/src/oom_profile.rs index 7a2c7b17..530a0278 100644 --- a/src/oom_profile.rs +++ b/src/oom_profile.rs @@ -466,6 +466,22 @@ pub fn limit_by_memory(parallel: u32, mib_per_job: u32, memory_mib: u64) -> u32 parallel.min(by_memory).max(1) } +/// The core cap with a memory ceiling on top: +/// `min(core_cap, memory_mib // mib_per_job)`. +/// +/// `core_cap` is the level the hook leaves. Unlike [`limit_by_memory`] +/// the result is not raised to 1: 0 says one job needs more memory than +/// the node has. Flooring the MiB first does not change the quotient, +/// so this is also `memory_bytes // (mib_per_job * 2**20)`. A zero +/// `mib_per_job` is not a budget; the table parser refuses it, and this +/// returns `core_cap`. +pub fn memory_cap(core_cap: u32, mib_per_job: u32, memory_mib: u64) -> u32 { + let Some(jobs) = memory_mib.checked_div(u64::from(mib_per_job)) else { + return core_cap; + }; + core_cap.min(u32::try_from(jobs).unwrap_or(u32::MAX)) +} + impl LimitOp { fn apply(self, cores: u32) -> u32 { match self { @@ -1370,6 +1386,16 @@ fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32 (got.level, text) } +fn memory_cap_clause(core_cap: u32, mib_per_job: u32, node: &NodeSize) -> String { + let cap = memory_cap(core_cap, mib_per_job, node.memory_mib); + let formula = format!("min({core_cap}, {} // {mib_per_job})", node.memory_mib); + if cap == 0 { + format!("memory cap 0 ({formula}), not one {mib_per_job} MiB job fits") + } else { + format!("memory cap {cap} ({formula})") + } +} + fn memory_clause(mib_per_job: Option, node: &NodeSize, max_parallel: u32) -> String { let Some(mib_per_job) = mib_per_job else { return "memory unmeasured".to_string(); @@ -1435,7 +1461,8 @@ fn job_memory_footer(jobs: &JobMemoryTable) -> String { /// target. The hook column is what `post_ready_hook` would leave, including /// EasyBuild's `max-parallel`. The memory column is /// `limit_by_memory(cores, mib_per_job)` when the table has a number, and -/// `unmeasured` when it does not. No number is invented. +/// `unmeasured` when it does not. With a number, the hook column is +/// followed by its [`memory_cap`]. No number is invented. pub fn render_parallelism( resources: &BotResources, jobs: &JobMemoryTable, @@ -1458,8 +1485,11 @@ pub fn render_parallelism( for name in &names { let (level, clause) = current_clause(resources, name, node); let mib = jobs.get(name).and_then(|package| package.mib_per_job); + let cap = mib + .map(|mib| format!("; {}", memory_cap_clause(level, mib, node))) + .unwrap_or_default(); out.push_str(&format!( - " {name}: current {level} ({clause}); {}\n", + " {name}: current {level} ({clause}){cap}; {}\n", memory_clause(mib, node, resources.max_parallel) )); } @@ -1988,6 +2018,30 @@ source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6420263 assert_eq!(limit_by_memory(48, 100, 30_016), 48); } + #[test] + fn the_memory_cap_is_not_raised_to_one_and_matches_the_byte_quotient() { + assert_eq!(memory_cap(8, 8192, 32_768), 4); + assert_eq!(memory_cap(8, 8192, 131_072), 8); + assert_eq!(memory_cap(8, 8192, 4096), 0); + assert_eq!(memory_cap(6, 8192, 30_016), 3); + assert_eq!(memory_cap(8, 0, 32_768), 8); + const BYTES_PER_MIB: u64 = 1 << 20; + let gib = 1024 * BYTES_PER_MIB; + for bytes in [ + 0, + BYTES_PER_MIB - 1, + 32 * gib, + 32 * gib + 12_345, + 30_016 * BYTES_PER_MIB + BYTES_PER_MIB - 1, + ] { + for mib_per_job in [1, 5, 6554, 8192] { + let by_bytes = bytes / (u64::from(mib_per_job) * BYTES_PER_MIB); + let by_mib = memory_cap(u32::MAX, mib_per_job, bytes / BYTES_PER_MIB); + assert_eq!(u64::from(by_mib), by_bytes, "{bytes} // {mib_per_job} MiB"); + } + } + } + #[test] fn a_non_integer_divide_warns_and_uses_the_numeric_floor() { let text = r#" @@ -2140,7 +2194,7 @@ note = "fixture for the report arithmetic, not a measurement of libxc" let one = render_parallelism(&resources, &fixture, &nodes[..1]); assert!( one.contains( - " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory 4 (max(1, min(16, 32768 // 8192)))\n" + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192)))\n" ), "{one}" ); diff --git a/tests/mcp_v1.rs b/tests/mcp_v1.rs index 4cf9a18f..3fbbfc58 100644 --- a/tests/mcp_v1.rs +++ b/tests/mcp_v1.rs @@ -3834,3 +3834,23 @@ fn mcp_stack_oom_returns_the_dropped_msg_sum() { let text = body["text"].as_str().expect("text"); assert!(text.ends_with(&format!("{warning}\n")), "{text}"); } + +#[test] +fn mcp_stack_oom_prints_the_memory_cap_beside_the_core_cap() { + let job_memory = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/oom_profile_fixtures/job-memory-fixture.toml") + .display() + .to_string(); + let response = call( + 1, + "eb_stack_oom", + json!({"node": ["16:4096:x86_64/amd/zen4"], "job_memory": job_memory}), + ); + let text = response["result"]["structuredContent"]["text"] + .as_str() + .expect("text"); + assert!( + text.contains(" libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192)))\n"), + "{text}" + ); +} diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index 8af4ccf8..a3d9ccc7 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -446,7 +446,7 @@ fn node_sizes_compare_the_hook_with_the_memory_rule() { assert_eq!(code, 0, "{stderr}"); assert!( stdout.contains( - " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory 4 (max(1, min(16, 32768 // 8192)))\n" + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192)))\n" ), "{stdout}" ); @@ -457,6 +457,63 @@ fn node_sizes_compare_the_hook_with_the_memory_rule() { ); } +#[test] +fn the_memory_cap_is_the_core_cap_under_the_memory_ceiling() { + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--mib-per-job", + &arg("job-memory-fixture.toml"), + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "16:131072:x86_64/amd/zen4", + "--node", + "16:4096:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + let libxc: Vec<&str> = stdout + .lines() + .filter(|line| line.starts_with(" libxc: ")) + .collect(); + assert_eq!( + libxc, + [ + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192)))", + // The hook already holds libxc below what 128 GiB allows. + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 8 (min(8, 131072 // 8192)); memory 16 (max(1, min(16, 131072 // 8192)))", + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192)))", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory cap 3 (min(6, 30016 // 8192)); memory 3 (max(1, min(48, 30016 // 8192)))", + ] + ); + // A name the table does not measure gets no cap. + assert!( + stdout.contains( + " LLVM: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n" + ), + "{stdout}" + ); + assert_eq!(stdout.matches("memory cap").count(), 4, "{stdout}"); + + let (_, by_long_name, _) = run(&[ + "stack", + "oom", + "--job-memory", + &arg("job-memory-fixture.toml"), + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "16:131072:x86_64/amd/zen4", + "--node", + "16:4096:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(by_long_name, stdout); +} + #[test] fn a_node_size_without_cores_or_memory_is_rejected() { let (code, _, stderr) = run(&["stack", "oom", "--node", "16"]); @@ -589,6 +646,6 @@ fn the_how_to_shows_the_real_output() { "{command}" ); } - assert_eq!(oom_blocks, 10); + assert_eq!(oom_blocks, 11); assert_eq!(rss_blocks, 1); } From be17e9dcbf9c1a32a51a6de7c79f2d4dc45fe580 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:33:23 +0000 Subject: [PATCH 4/7] feat(stack): model EasyBuild `--parallel` against `eb_hooks.py` The post_ready_hook in software-layer-scripts eb_hooks.py does not divide the level EasyBuild chose. It calls `det_parallelism()` with no arguments, which counts the cores. Then the hook applies its PARALLELISM_LIMITS op to that count. It keeps the result only when it is below EasyBuild's level. EasyBuild 5.4.2 set_parallel uses `--parallel` in place of the core count when it is set, capped at `min(maxparallel, --max-parallel)`. The `stack oom` command modelled only the bot's case, where neither is set, so the difference never showed. The `stack oom --parallel N` and `--max-parallel N` options now give EasyBuild's level for the job: `--parallel` (which may be above the cores) or the cores, capped at max-parallel and maxparallel. The hook op still applies to the node's cores. On a 16-core zen4 node with `--parallel 12`, libxc gets `16 // 2 = 8`, not `12 // 2`. With `--parallel 4`, the hook's 8 is not lower and libxc stays at 4. The `--node` lines name the setting that decided (`then --parallel 4`, `then max-parallel 16`). The node line names `--parallel`. In a warning, the parallel line names `--parallel` and the cores the hook divided. It also says when the hook's result was not lower. A value of 0 for either flag is a usage error. The `eb_stack_oom` tool takes parallel and max_parallel. Without them the output is unchanged. At software-layer-scripts 43df6f4 the hook reads EESSI_SOFTWARE_SUBDIR, the software name, EasyBuild's level and `det_parallelism()`. No environment variable there caps PARALLELISM_LIMITS, so none is modelled. --- CHANGELOG.md | 11 ++ docs/orgmode/howto/oom-profile.org | 65 ++++++++- docs/orgmode/reference/cli.org | 17 ++- llms-full.txt | 82 ++++++++++- skills/eessi-extend/SKILL.md | 5 +- src/main.rs | 16 +++ src/mcp.rs | 12 +- src/oom_profile.rs | 224 ++++++++++++++++++++++++----- src/site_commands.rs | 2 + tests/mcp_v1.rs | 50 +++++++ tests/oom_profile.rs | 147 ++++++++++++++++++- 11 files changed, 581 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e3e1bda..315579c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,17 @@ All notable changes to this project are documented here. more than the node has. `--mib-per-job FILE` is a visible alias of `--job-memory FILE`. The `memory` column, `limit_by_memory` in place of the hook, is unchanged. +- `stack oom --parallel N` and `--max-parallel N` model a job that sets + EasyBuild's `--parallel` or `--max-parallel`; the bot sets neither. + EasyBuild uses `--parallel` in place of the core count, capped at + max-parallel and `maxparallel`. The EESSI `post_ready_hook` still + applies its op to the node's cores (`det_parallelism()` with no + arguments) and keeps the result only when it is lower: with + `--parallel 12` on 16 cores libxc gets 16 // 2 = 8, and with + `--parallel 4` it stays at 4. The `--node` lines and a warning's + `parallel` line name the setting that decided, and say when the + hook's result was not lower. 0 for either flag is status 2. + `eb_stack_oom` takes `parallel` and `max_parallel`. ### Documentation diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index 73c6b03f..63eec7ea 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -168,9 +168,9 @@ or bot configuration. also holds EasyBuild's default maximum parallel level of 16 and the ~PARALLELISM_LIMITS~ table from the EESSI ~eb_hooks.py~. -The parallel level is the smallest of these: the node's cores, -EasyBuild's maximum, the hook limit applied to the cores, and a -top-level ~maxparallel~ in the recipe. The hook ops are the ones in +The parallel level is the smallest of these: the node's cores, or +~--parallel~ in their place; EasyBuild's maximum; the hook limit applied +to the cores; and a top-level ~maxparallel~ in the recipe. The hook ops are the ones in ~eb_hooks.py~. ~divide_by_factor(parallel, factor)~ is ~max(1, parallel // factor)~. ~set_maximum(parallel, n)~ is ~min(parallel, n)~. A software name uses its limit for the CPU target, @@ -289,6 +289,65 @@ The warning names the function and the line. It does not change a parallel level, and the command still exits 0. ~msg += "..."~, as in ~hooks/msg-plus-equals.py~, appends and is not reported. +* EasyBuild's --parallel and the hook + +~post_ready_hook~ does not divide the level EasyBuild chose. It calls +~det_parallelism()~ with no arguments, which counts the cores the job +can use, applies the op to that count, and keeps the result only when +it is below EasyBuild's level. EasyBuild's level comes from +~set_parallel~: ~--parallel~ (~EASYBUILD_PARALLEL~) in place of the +core count when it is set, capped at the easyconfig's ~maxparallel~ and +at ~--max-parallel~ (~EASYBUILD_MAX_PARALLEL~, 16 by default). The bot +sets neither. ~stack oom --parallel N~ and ~--max-parallel N~ model a +job that does: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 12 \ + --node 16:32768:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 12 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +#+end_example + +libxc gets 8, which is 16 // 2. Half of ~--parallel 12~ would be 6. +With ~--parallel 4~ the hook's 8 is not lower, so it does not apply: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 4 \ + --node 16:32768:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 4 + libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +#+end_example + +A divide factor is a fraction of the node, not of ~--parallel~: +~--parallel~ above the hook's result does not change it, and +~--parallel~ below it wins. ~--parallel~ also replaces the core count in +EasyBuild, so with ~--parallel 32 --max-parallel 64~ a build with no +hook row runs at 32 on a 16-core node. With an easystack, the +~parallel~ line of a warning names ~--parallel~ and the cores the hook +divided, and says when the hook's result was not lower. + +At software-layer-scripts 43df6f4 the hook reads ~EESSI_SOFTWARE_SUBDIR~, +the software name, EasyBuild's level, and ~det_parallelism()~. No +environment variable there caps or overrides ~PARALLELISM_LIMITS~. +EESSI-extend's ~EESSI_EASYBUILD_HOOKS_OVERRIDE~ replaces the whole hooks +file; ~--hooks~ reads such a file. + * A memory limit instead of a core fraction A core fraction does not travel. Half of 16 cores and half of 64 cores diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 2407c125..280bd89c 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -17,7 +17,7 @@ reachability, toolchain family, and the claim ladder are defined in | 1 | The work was attempted and failed: a missing or unreadable file, a document that does not parse, a check that found errors, a network fetch that failed, a doctor probe, or a campaign whose build failed | | 2 | The command line was rejected before that work: a missing or unknown flag (clap), or a flag value or combination the command does not accept | -A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails ~recipe lint~ or ~recipe check~ is status 1: the files were read and the check failed. ~stack from-commit~ is status 1 when any pin is ~released~, and status 0 when every pin is ~newer~ or ~unknown~ or the files name no pin. A file name with no ~eb-X.Y.Z~, and no ~--easybuild-version~, is status 2. ~stack arch~ is status 1 when an entry is unsupported or flagged, or when ~--probe-sources~ reports ~unreachable~ or ~arch-missing~ sources. ~stack files~ is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are ~no_url~ or ~unchecked~. ~stack generation~ is status 1 on any finding, and status 2 when an easystack path names no EESSI version and ~--eessi-version~ is absent. ~stack known-failures~ is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with ~--strict~ it is status 1 when a warning remains, and status 0 when every hit is remedied. ~stack oom~ is status 0 when it warns: the warnings are advice, not a failed check. A ~--node~ value that is not ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~ is status 2, and so is a command with none of ~--easystack~, ~--node~, and ~--hooks~. A ~--hooks~ file that is not valid Python, has no module-level ~PARALLELISM_LIMITS~, or has a row the model cannot follow is status 1, and the message names the line. ~stack bot-classify~ is status 0 for every class, and status 2 when a path is a directory. ~stack batch~ is status 1 when the plan includes, or depends on, an entry the architecture cannot build. ~stack provided~ is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets ~--hooks~, or belongs in a ~--family-lock~. A group-restricted warning is status 0 unless ~--strict~. Status 2 is when neither ~--installed~ nor ~--baseline-lock~ is set. ~stack shard~ is status 1 when a build-list line cannot be resolved, and writes nothing; ~--shards 0~ and a list with no lines are status 2. ~stack shadow~ is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. +A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails ~recipe lint~ or ~recipe check~ is status 1: the files were read and the check failed. ~stack from-commit~ is status 1 when any pin is ~released~, and status 0 when every pin is ~newer~ or ~unknown~ or the files name no pin. A file name with no ~eb-X.Y.Z~, and no ~--easybuild-version~, is status 2. ~stack arch~ is status 1 when an entry is unsupported or flagged, or when ~--probe-sources~ reports ~unreachable~ or ~arch-missing~ sources. ~stack files~ is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are ~no_url~ or ~unchecked~. ~stack generation~ is status 1 on any finding, and status 2 when an easystack path names no EESSI version and ~--eessi-version~ is absent. ~stack known-failures~ is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with ~--strict~ it is status 1 when a warning remains, and status 0 when every hit is remedied. ~stack oom~ is status 0 when it warns: the warnings are advice, not a failed check. A ~--node~ value that is not ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~ is status 2, and so is a command with none of ~--easystack~, ~--node~, and ~--hooks~, or a ~--parallel~ or ~--max-parallel~ of 0. A ~--hooks~ file that is not valid Python, has no module-level ~PARALLELISM_LIMITS~, or has a row the model cannot follow is status 1, and the message names the line. ~stack bot-classify~ is status 0 for every class, and status 2 when a path is a directory. ~stack batch~ is status 1 when the plan includes, or depends on, an entry the architecture cannot build. ~stack provided~ is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets ~--hooks~, or belongs in a ~--family-lock~. A group-restricted warning is status 0 unless ~--strict~. Status 2 is when neither ~--installed~ nor ~--baseline-lock~ is set. ~stack shard~ is status 1 when a build-list line cannot be resolved, and writes nothing; ~--shards 0~ and a list with no lines are status 2. ~stack shadow~ is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. ~--help~ for every command has an example and a description of every flag. A default that the command applies is printed in that help. A directory that receives several artifacts is ~--out-dir~. A single output file is ~--out~ (~--lock-out~ for the lock ~stack solve~ writes). ~--installed~, ~--easyconfigs~, ~--filter-deps~, and ~--generated-at~ use those names on every command that takes them. @@ -1160,6 +1160,18 @@ than the node's memory. The memory a hook would read on the build machine is the smallest cgroup v2 ~memory.max~ on the process's cgroup path, else ~MemTotal~. +~--parallel N~ and ~--max-parallel N~ are EasyBuild's settings for the +job, as ~EASYBUILD_PARALLEL~ and ~EASYBUILD_MAX_PARALLEL~ would set them; +the bot sets neither. EasyBuild uses ~--parallel~ in place of the core +count, capped at ~--max-parallel~ (in place of the table's +~max_parallel~) and at the easyconfig's ~maxparallel~. The hook applies +its op to the node's cores, not to either setting, and keeps the result +only when it is lower. ~--node~ lines then end the hook formula with +~then --parallel N~ or ~then max-parallel N~ when that setting decided, +and the node line names ~--parallel~. A warning's ~parallel~ line names +~--parallel~ and the cores the hook divided, and says when the hook's +result was not below EasyBuild's level. 0 for either flag is status 2. + ~--hooks~ reads ~PARALLELISM_LIMITS~ from an EESSI ~eb_hooks.py~ in place of the hook rows of ~data/eessi-bot-resources.toml~. The nodes and EasyBuild's ~max-parallel~ still come from the table. The file is @@ -1819,7 +1831,8 @@ needs one ~easystack~ or ~pr~. It returns the same report as stdout, and ~hits~, one object per line. Warnings are not a tool error unless ~strict~ is true, when a remaining warning is. ~eb_stack_oom~ takes ~easystack~, ~arch~, ~easyconfigs~, -~installed~, ~profile~, ~resources~, ~node~, ~job_memory~, and ~hooks~. +~installed~, ~profile~, ~resources~, ~node~, ~job_memory~, ~hooks~, +~parallel~, and ~max_parallel~. It needs an easystack, a node, or hooks. It returns the ~stack oom~ text, one object per warning, ~factor_warnings~ for a divide factor that is not an integer, and ~hook_warnings~ for a dropped sum in the diff --git a/llms-full.txt b/llms-full.txt index ba9dff75..8f76d241 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4761,9 +4761,9 @@ or bot configuration. also holds EasyBuild's default maximum parallel level of 16 and the `PARALLELISM_LIMITS` table from the EESSI `eb_hooks.py`. -The parallel level is the smallest of these: the node's cores, -EasyBuild's maximum, the hook limit applied to the cores, and a -top-level `maxparallel` in the recipe. The hook ops are the ones in +The parallel level is the smallest of these: the node's cores, or +`--parallel` in their place; EasyBuild's maximum; the hook limit applied +to the cores; and a top-level `maxparallel` in the recipe. The hook ops are the ones in `eb_hooks.py`. `divide_by_factor(parallel, factor)` is `max(1, parallel // factor)`. `set_maximum(parallel, n)` is `min(parallel, n)`. A software name uses its limit for the CPU target, @@ -4882,6 +4882,65 @@ The warning names the function and the line. It does not change a parallel level, and the command still exits 0. `msg += "..."`, as in `hooks/msg-plus-equals.py`, appends and is not reported. +# EasyBuild's --parallel and the hook + +`post_ready_hook` does not divide the level EasyBuild chose. It calls +`det_parallelism()` with no arguments, which counts the cores the job +can use, applies the op to that count, and keeps the result only when +it is below EasyBuild's level. EasyBuild's level comes from +`set_parallel`: `--parallel` (`EASYBUILD_PARALLEL`) in place of the +core count when it is set, capped at the easyconfig's `maxparallel` and +at `--max-parallel` (`EASYBUILD_MAX_PARALLEL`, 16 by default). The bot +sets neither. `stack oom --parallel N` and `--max-parallel N` model a +job that does: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 12 \ + --node 16:32768:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 12 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +``` + +libxc gets 8, which is 16 // 2. Half of `--parallel 12` would be 6. +With `--parallel 4` the hook's 8 is not lower, so it does not apply: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 4 \ + --node 16:32768:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 4 + libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +``` + +A divide factor is a fraction of the node, not of `--parallel`: +`--parallel` above the hook's result does not change it, and +`--parallel` below it wins. `--parallel` also replaces the core count in +EasyBuild, so with `--parallel 32 --max-parallel 64` a build with no +hook row runs at 32 on a 16-core node. With an easystack, the +`parallel` line of a warning names `--parallel` and the cores the hook +divided, and says when the hook's result was not lower. + +At software-layer-scripts 43df6f4 the hook reads `EESSI_SOFTWARE_SUBDIR`, +the software name, EasyBuild's level, and `det_parallelism()`. No +environment variable there caps or overrides `PARALLELISM_LIMITS`. +EESSI-extend's `EESSI_EASYBUILD_HOOKS_OVERRIDE` replaces the whole hooks +file; `--hooks` reads such a file. + # A memory limit instead of a core fraction A core fraction does not travel. Half of 16 cores and half of 64 cores @@ -5960,7 +6019,7 @@ reachability, toolchain family, and the claim ladder are defined in | 1 | The work was attempted and failed: a missing or unreadable file, a document that does not parse, a check that found errors, a network fetch that failed, a doctor probe, or a campaign whose build failed | | 2 | The command line was rejected before that work: a missing or unknown flag (clap), or a flag value or combination the command does not accept | -A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails `recipe lint` or `recipe check` is status 1: the files were read and the check failed. `stack from-commit` is status 1 when any pin is `released`, and status 0 when every pin is `newer` or `unknown` or the files name no pin. A file name with no `eb-X.Y.Z`, and no `--easybuild-version`, is status 2. `stack arch` is status 1 when an entry is unsupported or flagged, or when `--probe-sources` reports `unreachable` or `arch-missing` sources. `stack files` is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are `no_url` or `unchecked`. `stack generation` is status 1 on any finding, and status 2 when an easystack path names no EESSI version and `--eessi-version` is absent. `stack known-failures` is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with `--strict` it is status 1 when a warning remains, and status 0 when every hit is remedied. `stack oom` is status 0 when it warns: the warnings are advice, not a failed check. A `--node` value that is not `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET` is status 2, and so is a command with none of `--easystack`, `--node`, and `--hooks`. A `--hooks` file that is not valid Python, has no module-level `PARALLELISM_LIMITS`, or has a row the model cannot follow is status 1, and the message names the line. `stack bot-classify` is status 0 for every class, and status 2 when a path is a directory. `stack batch` is status 1 when the plan includes, or depends on, an entry the architecture cannot build. `stack provided` is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets `--hooks`, or belongs in a `--family-lock`. A group-restricted warning is status 0 unless `--strict`. Status 2 is when neither `--installed` nor `--baseline-lock` is set. `stack shard` is status 1 when a build-list line cannot be resolved, and writes nothing; `--shards 0` and a list with no lines are status 2. `stack shadow` is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. +A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails `recipe lint` or `recipe check` is status 1: the files were read and the check failed. `stack from-commit` is status 1 when any pin is `released`, and status 0 when every pin is `newer` or `unknown` or the files name no pin. A file name with no `eb-X.Y.Z`, and no `--easybuild-version`, is status 2. `stack arch` is status 1 when an entry is unsupported or flagged, or when `--probe-sources` reports `unreachable` or `arch-missing` sources. `stack files` is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are `no_url` or `unchecked`. `stack generation` is status 1 on any finding, and status 2 when an easystack path names no EESSI version and `--eessi-version` is absent. `stack known-failures` is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with `--strict` it is status 1 when a warning remains, and status 0 when every hit is remedied. `stack oom` is status 0 when it warns: the warnings are advice, not a failed check. A `--node` value that is not `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET` is status 2, and so is a command with none of `--easystack`, `--node`, and `--hooks`, or a `--parallel` or `--max-parallel` of 0. A `--hooks` file that is not valid Python, has no module-level `PARALLELISM_LIMITS`, or has a row the model cannot follow is status 1, and the message names the line. `stack bot-classify` is status 0 for every class, and status 2 when a path is a directory. `stack batch` is status 1 when the plan includes, or depends on, an entry the architecture cannot build. `stack provided` is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets `--hooks`, or belongs in a `--family-lock`. A group-restricted warning is status 0 unless `--strict`. Status 2 is when neither `--installed` nor `--baseline-lock` is set. `stack shard` is status 1 when a build-list line cannot be resolved, and writes nothing; `--shards 0` and a list with no lines are status 2. `stack shadow` is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. `--help` for every command has an example and a description of every flag. A default that the command applies is printed in that help. A directory that receives several artifacts is `--out-dir`. A single output file is `--out` (`--lock-out` for the lock `stack solve` writes). `--installed`, `--easyconfigs`, `--filter-deps`, and `--generated-at` use those names on every command that takes them. @@ -7103,6 +7162,18 @@ than the node's memory. The memory a hook would read on the build machine is the smallest cgroup v2 `memory.max` on the process's cgroup path, else `MemTotal`. +`--parallel N` and `--max-parallel N` are EasyBuild's settings for the +job, as `EASYBUILD_PARALLEL` and `EASYBUILD_MAX_PARALLEL` would set them; +the bot sets neither. EasyBuild uses `--parallel` in place of the core +count, capped at `--max-parallel` (in place of the table's +`max_parallel`) and at the easyconfig's `maxparallel`. The hook applies +its op to the node's cores, not to either setting, and keeps the result +only when it is lower. `--node` lines then end the hook formula with +`then --parallel N` or `then max-parallel N` when that setting decided, +and the node line names `--parallel`. A warning's `parallel` line names +`--parallel` and the cores the hook divided, and says when the hook's +result was not below EasyBuild's level. 0 for either flag is status 2. + `--hooks` reads `PARALLELISM_LIMITS` from an EESSI `eb_hooks.py` in place of the hook rows of `data/eessi-bot-resources.toml`. The nodes and EasyBuild's `max-parallel` still come from the table. The file is @@ -7762,7 +7833,8 @@ needs one `easystack` or `pr`. It returns the same report as stdout, and `hits`, one object per line. Warnings are not a tool error unless `strict` is true, when a remaining warning is. `eb_stack_oom` takes `easystack`, `arch`, `easyconfigs`, -`installed`, `profile`, `resources`, `node`, `job_memory`, and `hooks`. +`installed`, `profile`, `resources`, `node`, `job_memory`, `hooks`, +`parallel`, and `max_parallel`. It needs an easystack, a node, or hooks. It returns the `stack oom` text, one object per warning, `factor_warnings` for a divide factor that is not an integer, and `hook_warnings` for a dropped sum in the diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index c3fe2d74..773087e6 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -339,7 +339,10 @@ the change in the pull request yourself. The command exits 0. The hook ops are `divide_by_factor` (`max(1, parallel // factor)`) and `set_maximum` (`min(parallel, n)`). A CPU-target key overrides `*`. -A64FX divides every other build by 4. A divide factor that is not an +A64FX divides every other build by 4. The op applies to the node's +cores, not to EasyBuild's `--parallel`, and wins only when lower; +`--parallel N` and `--max-parallel N` on `stack oom` show a job that +sets them. A divide factor that is not an integer is a warning: `16 // 2.5` is the float `6.0`. Use an integer. `--hooks software-layer-scripts/eb_hooks.py` reads `PARALLELISM_LIMITS` from the file instead of the bundled copy, and the warning names the diff --git a/src/main.rs b/src/main.rs index e19e3c94..98b67c84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1597,6 +1597,18 @@ enum StackCommand { /// a name and drops the sum, such as `msg+ "..."`, is a warning. #[arg(long)] hooks: Option, + /// EasyBuild's `--parallel` for the job, as `EASYBUILD_PARALLEL` + /// would set it. EasyBuild then uses it in place of the cores, + /// capped at max-parallel. The hook still applies its op to the + /// node's cores and keeps the result only when it is lower. The bot + /// sets none. + #[arg(long)] + parallel: Option, + /// EasyBuild's `--max-parallel` for the job, in place of + /// `max_parallel` in the bot resource table, EasyBuild's default. + /// The hook does not read it. + #[arg(long)] + max_parallel: Option, }, /// Say what an EESSI bot job's FAILURE was: a real failure, ERROR-only /// noise, an empty tarball, or infra. @@ -3282,6 +3294,8 @@ fn run_stack(command: StackCommand) -> Result<()> { node, job_memory, hooks, + parallel, + max_parallel, } => { let report = eb_stack::oom_profile::run_oom_check(&eb_stack::oom_profile::OomCheckRequest { @@ -3294,6 +3308,8 @@ fn run_stack(command: StackCommand) -> Result<()> { nodes: node, job_memory, hooks, + parallel, + max_parallel, })?; print!("{}", report.stdout); Ok(()) diff --git a/src/mcp.rs b/src/mcp.rs index 1f3bfe55..8a7397d6 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -540,7 +540,7 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_oom", - "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows; a statement such as msg+ \"...\" that drops its sum is returned in hook_warnings. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory; for a package with an mib_per_job in job_memory (the CLI's --job-memory or --mib-per-job) it also prints the memory cap min(current, memory // mib_per_job), where 0 means not one job fits. Pass easystack, node, or hooks. A warning is not an error.", + "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows; a statement such as msg+ \"...\" that drops its sum is returned in hook_warnings. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory; for a package with an mib_per_job in job_memory (the CLI's --job-memory or --mib-per-job) it also prints the memory cap min(current, memory // mib_per_job), where 0 means not one job fits. parallel and max_parallel are EasyBuild's --parallel and --max-parallel for the job; the hook applies its op to the node's cores, not to either, and keeps the result only when it is lower. Pass easystack, node, or hooks. A warning is not an error.", &[], &[ ("easystack", "array"), @@ -552,6 +552,8 @@ fn tool_catalog() -> Vec { ("node", "array"), ("job_memory", "string"), ("hooks", "string"), + ("parallel", "integer"), + ("max_parallel", "integer"), ], ), tool_with_optional( @@ -3418,6 +3420,8 @@ fn stack_oom_tool(arguments: &Value) -> Result { nodes: string_array(arguments, "node")?, job_memory: optional_path(arguments, "job_memory"), hooks: optional_path(arguments, "hooks"), + parallel: optional_u32(arguments, "parallel")?, + max_parallel: optional_u32(arguments, "max_parallel")?, }) .map_err(workflow_error)?; Ok(json!({ @@ -3939,6 +3943,12 @@ fn optional_u64(arguments: &Value, name: &str) -> Result, String> { } } +fn optional_u32(arguments: &Value, name: &str) -> Result, String> { + optional_u64(arguments, name)? + .map(|value| u32::try_from(value).map_err(|_| format!("argument {name} is too large"))) + .transpose() +} + fn u64_array(arguments: &Value, name: &str) -> Result, String> { match arguments.get(name) { None => Ok(Vec::new()), diff --git a/src/oom_profile.rs b/src/oom_profile.rs index 530a0278..e7ba9c42 100644 --- a/src/oom_profile.rs +++ b/src/oom_profile.rs @@ -547,6 +547,9 @@ pub struct HookLimit { pub struct BotResources { /// EasyBuild's default `max-parallel`. pub max_parallel: u32, + /// EasyBuild's `--parallel` for the job. `None`, which is what the bot + /// passes, lets EasyBuild count the cores. + pub parallel: Option, /// Node types, in file order. pub partitions: Vec, /// Hook limits, in file order. @@ -624,7 +627,13 @@ pub struct Parallel { pub level: u32, /// CPUs the job holds. pub cores: u32, - /// The hook limit that applied. + /// EasyBuild's `--parallel`, when the job sets one. + pub requested: Option, + /// EasyBuild's `max-parallel`, when it lowered `--parallel` or the + /// cores. + pub max_parallel: Option, + /// The hook limit that matched. It applies only when its result is + /// below the level EasyBuild chose. pub hook: Option, /// `maxparallel` in the easyconfig. pub recipe: Option, @@ -714,6 +723,7 @@ impl BotResources { } Ok(Self { max_parallel: raw.easybuild.max_parallel, + parallel: None, partitions, limits, hooks_source: raw.hooks.source, @@ -784,9 +794,27 @@ impl BotResources { lines } + /// The level EasyBuild's `set_parallel` gives a build before the hook. + /// + /// `--parallel` replaces the core count, so it may be above the cores. + /// `max-parallel` caps either. The second value is that cap when it + /// lowered the level. + fn easybuild_level(&self, cores: u32) -> (u32, Option) { + let asked = self.parallel.unwrap_or(cores); + if self.max_parallel < asked { + (self.max_parallel, Some(self.max_parallel)) + } else { + (asked, None) + } + } + /// The parallel level `name` builds with on `partition`. + /// + /// `post_ready_hook` applies its op to the cores it detects, not to + /// `--parallel` or `max-parallel`, and keeps the result only when it + /// is below the level EasyBuild chose. pub fn parallel(&self, name: &str, partition: &Partition, recipe: Option) -> Parallel { - let mut level = partition.cores.min(self.max_parallel); + let (mut level, max_parallel) = self.easybuild_level(partition.cores); if let Some(recipe) = recipe { level = level.min(recipe.max(1)); } @@ -797,6 +825,8 @@ impl BotResources { Parallel { level, cores: partition.cores, + requested: self.parallel, + max_parallel, hook, recipe, } @@ -956,28 +986,53 @@ fn estimate_text(entry: &MemoryEntry, parallel: u32, peak: f64) -> String { } fn parallel_reason(name: &str, parallel: &Parallel) -> String { - let mut parts = vec![format!("{} cores", parallel.cores)]; + let cores = parallel.cores; + let mut parts = vec![match parallel.requested { + Some(requested) => format!("--parallel {requested} on {cores} cores"), + None => format!("{cores} cores"), + }]; + if let Some(max_parallel) = parallel.max_parallel { + parts.push(format!("max-parallel {max_parallel}")); + } if let Some(recipe) = parallel.recipe { parts.push(format!("maxparallel = {recipe} in the easyconfig")); } - match parallel.hook { - Some(LimitOp::Divide(factor)) => { + let Some(op) = parallel.hook else { + parts.push(format!("no eb_hooks.py limit for {name}")); + return parts.join(", "); + }; + let mut hook = match op { + LimitOp::Divide(factor) => { let shown = format_factor(factor); - let divided = divide_by_factor(parallel.cores, factor); - if divided.float_result { - parts.push(format!( - "eb_hooks.py divides {name} by {shown} ({} // {shown} is {:.1}, a float)", - parallel.cores, divided.level as f64 - )); + let divided = divide_by_factor(cores, factor); + let mut text = if divided.float_result { + format!( + "eb_hooks.py divides {name} by {shown} ({cores} // {shown} is {:.1}, a float)", + divided.level as f64 + ) } else { - parts.push(format!("eb_hooks.py divides {name} by {shown}")); + format!("eb_hooks.py divides {name} by {shown}") + }; + if let Some(requested) = parallel.requested { + text.push_str(&format!( + " on the {cores} cores, not on --parallel {requested}" + )); } + text } - Some(LimitOp::Maximum(value)) => { - parts.push(format!("eb_hooks.py caps {name} at {value}")); + LimitOp::Maximum(value) => format!("eb_hooks.py caps {name} at {value}"), + }; + let hooked = op.apply(cores); + if hooked > parallel.level { + if op != LimitOp::Maximum(hooked) { + hook.push_str(&format!(", which gives {hooked}")); } - None => parts.push(format!("no eb_hooks.py limit for {name}")), + hook.push_str(&format!( + ", not below {}, so it does not apply", + parallel.level + )); } + parts.push(hook); parts.join(", ") } @@ -1335,9 +1390,25 @@ fn node_partition(node: &NodeSize) -> Partition { } } +/// EasyBuild's level before the hook on `cores`, and the setting that +/// decided it when that is not the cores. +fn easybuild_bound(resources: &BotResources, cores: u32) -> (u32, Option) { + let (level, capped) = resources.easybuild_level(cores); + let bound = match (capped, resources.parallel) { + (Some(max_parallel), _) => Some(format!("max-parallel {max_parallel}")), + (None, Some(requested)) => Some(format!("--parallel {requested}")), + (None, None) => None, + }; + (level, bound) +} + fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32, String) { let partition = node_partition(node); let got = resources.parallel(name, &partition, None); + let (_, bound) = easybuild_bound(resources, node.cores); + let then = bound + .clone() + .unwrap_or_else(|| format!("{} cores", node.cores)); let key = resources .matching_limit(name, &partition.target) .map(|limit| limit.target.as_str()); @@ -1353,14 +1424,11 @@ fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32 if divided.level != got.level { if divided.float_result { text.push_str(&format!( - " = {:.1}, a float, then max-parallel {}", - divided.level as f64, resources.max_parallel + " = {:.1}, a float, then {then}", + divided.level as f64 )); } else { - text.push_str(&format!( - " = {}, then max-parallel {}", - divided.level, resources.max_parallel - )); + text.push_str(&format!(" = {}, then {then}", divided.level)); } } else if divided.float_result { text.push_str(&format!(" = {:.1}, a float", divided.level as f64)); @@ -1371,17 +1439,14 @@ fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32 let raw = set_maximum(node.cores, value); let mut text = format!("set_maximum{on}, min({}, {value})", node.cores); if raw != got.level { - text.push_str(&format!( - " = {raw}, then max-parallel {}", - resources.max_parallel - )); + text.push_str(&format!(" = {raw}, then {then}")); } text } - None if node.cores > resources.max_parallel => { - format!("no hook limit, max-parallel {}", resources.max_parallel) - } - None => "no hook limit".to_string(), + None => match bound { + Some(bound) => format!("no hook limit, {bound}"), + None => "no hook limit".to_string(), + }, }; (got.level, text) } @@ -1396,19 +1461,20 @@ fn memory_cap_clause(core_cap: u32, mib_per_job: u32, node: &NodeSize) -> String } } -fn memory_clause(mib_per_job: Option, node: &NodeSize, max_parallel: u32) -> String { +fn memory_clause(mib_per_job: Option, node: &NodeSize, resources: &BotResources) -> String { let Some(mib_per_job) = mib_per_job else { return "memory unmeasured".to_string(); }; let raw = limit_by_memory(node.cores, mib_per_job, node.memory_mib); - let curr = node.cores.min(max_parallel).max(1); - let effective = if raw < curr { raw } else { curr }; + let (curr, bound) = easybuild_bound(resources, node.cores); + let effective = raw.min(curr); let formula = format!( "max(1, min({}, {} // {mib_per_job}))", node.cores, node.memory_mib ); if effective != raw { - format!("memory {effective} ({formula} = {raw}, then max-parallel {max_parallel})") + let bound = bound.unwrap_or_else(|| format!("{} cores", node.cores)); + format!("memory {effective} ({formula} = {raw}, then {bound})") } else { format!("memory {effective} ({formula})") } @@ -1459,7 +1525,7 @@ fn job_memory_footer(jobs: &JobMemoryTable) -> String { /// /// Each node is a core count, a memory in MiB, and an optional CPU /// target. The hook column is what `post_ready_hook` would leave, including -/// EasyBuild's `max-parallel`. The memory column is +/// EasyBuild's `--parallel` and `max-parallel`. The memory column is /// `limit_by_memory(cores, mib_per_job)` when the table has a number, and /// `unmeasured` when it does not. With a number, the hook column is /// followed by its [`memory_cap`]. No number is invented. @@ -1471,14 +1537,18 @@ pub fn render_parallelism( let mut out = String::from("parallelism under the current eb_hooks.py limit and under limit_by_memory\n"); let names = report_names(resources, jobs); + let requested = resources + .parallel + .map(|parallel| format!(", --parallel {parallel}")) + .unwrap_or_default(); for node in nodes { match &node.target { Some(target) => out.push_str(&format!( - "node {} cores, {} MiB, {target}\n", + "node {} cores, {} MiB, {target}{requested}\n", node.cores, node.memory_mib )), None => out.push_str(&format!( - "node {} cores, {} MiB\n", + "node {} cores, {} MiB{requested}\n", node.cores, node.memory_mib )), } @@ -1490,7 +1560,7 @@ pub fn render_parallelism( .unwrap_or_default(); out.push_str(&format!( " {name}: current {level} ({clause}){cap}; {}\n", - memory_clause(mib, node, resources.max_parallel) + memory_clause(mib, node, resources) )); } let sentinel = "\u{0}"; @@ -1533,6 +1603,10 @@ pub struct OomCheckRequest { /// An EESSI `eb_hooks.py` whose `PARALLELISM_LIMITS` replace the hook /// rows of the bot resource table. pub hooks: Option, + /// EasyBuild's `--parallel` for the job. `None` counts the cores. + pub parallel: Option, + /// EasyBuild's `--max-parallel` for the job, in place of the table's. + pub max_parallel: Option, } /// What `stack oom` prints. @@ -1568,10 +1642,22 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { for text in &request.nodes { nodes.push(parse_node_size(text).map_err(usage)?); } + for (flag, value) in [ + ("--parallel", request.parallel), + ("--max-parallel", request.max_parallel), + ] { + if value == Some(0) { + return Err(usage(format!("{flag} must be a positive integer"))); + } + } let mut resources = match &request.resources { Some(path) => BotResources::load(path)?, None => BotResources::bundled()?, }; + resources.parallel = request.parallel; + if let Some(max_parallel) = request.max_parallel { + resources.max_parallel = max_parallel; + } let jobs = match &request.job_memory { Some(path) => JobMemoryTable::load(path)?, None => JobMemoryTable::bundled()?, @@ -1922,6 +2008,70 @@ mod tests { assert_eq!(resources.parallel("libxc", zen4, Some(4)).level, 4); } + #[test] + fn the_hook_applies_to_the_cores_not_to_easybuild_parallel() { + let mut resources = BotResources::bundled().unwrap(); + let zen4 = zen4(&resources).clone(); + let a64fx = resources + .partitions + .iter() + .find(|p| p.target == "aarch64/a64fx") + .unwrap() + .clone(); + + resources.parallel = Some(12); + let libxc = resources.parallel("libxc", &zen4, None); + // 16 // 2, not 12 // 2. + assert_eq!(libxc.level, 8); + assert_eq!( + parallel_reason("libxc", &libxc), + "--parallel 12 on 16 cores, eb_hooks.py divides libxc by 2 on the 16 cores, not on --parallel 12" + ); + assert_eq!(resources.parallel("zlib", &zen4, None).level, 12); + + resources.parallel = Some(4); + let libxc = resources.parallel("libxc", &zen4, None); + assert_eq!(libxc.level, 4); + assert_eq!( + parallel_reason("libxc", &libxc), + "--parallel 4 on 16 cores, eb_hooks.py divides libxc by 2 on the 16 cores, not on --parallel 4, which gives 8, not below 4, so it does not apply" + ); + let rocblas = resources.parallel("rocBLAS", &zen4, None); + assert_eq!( + parallel_reason("rocBLAS", &rocblas), + "--parallel 4 on 16 cores, eb_hooks.py caps rocBLAS at 8, not below 4, so it does not apply" + ); + + // EasyBuild takes --parallel as given, above the cores, up to + // max-parallel. The hook still starts from the cores. + resources.parallel = Some(32); + resources.max_parallel = 64; + assert_eq!(resources.parallel("zlib", &zen4, None).level, 32); + assert_eq!(resources.parallel("libxc", &zen4, None).level, 8); + resources.max_parallel = 16; + let zlib = resources.parallel("zlib", &zen4, None); + assert_eq!(zlib.level, 16); + assert_eq!( + parallel_reason("zlib", &zlib), + "--parallel 32 on 16 cores, max-parallel 16, no eb_hooks.py limit for zlib" + ); + + resources.parallel = None; + let llvm = resources.parallel("LLVM", &a64fx, None); + assert_eq!(llvm.level, 16); + assert_eq!( + parallel_reason("LLVM", &llvm), + "48 cores, max-parallel 16, eb_hooks.py divides LLVM by 2, which gives 24, not below 16, so it does not apply" + ); + resources.max_parallel = 48; + let llvm = resources.parallel("LLVM", &a64fx, None); + assert_eq!(llvm.level, 24); + assert_eq!( + parallel_reason("LLVM", &llvm), + "48 cores, eb_hooks.py divides LLVM by 2" + ); + } + #[test] fn version_toolchain_and_target_families_have_boundaries() { assert!(in_family("7.0.0", "7.0", &['.', '-'])); diff --git a/src/site_commands.rs b/src/site_commands.rs index 61d4e199..26a47d3c 100644 --- a/src/site_commands.rs +++ b/src/site_commands.rs @@ -1717,6 +1717,8 @@ pub fn run_stack_batch(args: &StackBatchRequest) -> Result { nodes: Vec::new(), job_memory: None, hooks: None, + parallel: None, + max_parallel: None, })?; let oom = oom_report.warnings; let factor_warnings = oom_report.factor_warnings; diff --git a/tests/mcp_v1.rs b/tests/mcp_v1.rs index 3fbbfc58..4bd8ff21 100644 --- a/tests/mcp_v1.rs +++ b/tests/mcp_v1.rs @@ -3854,3 +3854,53 @@ fn mcp_stack_oom_prints_the_memory_cap_beside_the_core_cap() { "{text}" ); } + +#[test] +fn mcp_stack_oom_takes_easybuild_parallel_and_max_parallel() { + let response = call( + 1, + "eb_stack_oom", + json!({"node": ["16:32768:x86_64/amd/zen4"], "parallel": 4}), + ); + assert_eq!(response["result"]["isError"], false, "{response}"); + let text = response["result"]["structuredContent"]["text"] + .as_str() + .expect("text"); + assert!( + text.contains("node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 4\n"), + "{text}" + ); + assert!( + text.contains(" libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured\n"), + "{text}" + ); + + let response = call( + 2, + "eb_stack_oom", + json!({"node": ["48:30016:aarch64/a64fx"], "max_parallel": 48}), + ); + let text = response["result"]["structuredContent"]["text"] + .as_str() + .expect("text"); + assert!( + text.contains( + " LLVM: current 24 (divide_by_factor on *, max(1, 48 // 2)); memory unmeasured\n" + ), + "{text}" + ); + + let response = call( + 3, + "eb_stack_oom", + json!({"node": ["16:32768"], "parallel": 0}), + ); + assert_eq!(response["result"]["isError"], true, "{response}"); + let message = response["result"]["content"][0]["text"] + .as_str() + .unwrap_or_default(); + assert!( + message.contains("--parallel must be a positive integer"), + "{message}" + ); +} diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index a3d9ccc7..917a1579 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -514,6 +514,151 @@ fn the_memory_cap_is_the_core_cap_under_the_memory_ceiling() { assert_eq!(by_long_name, stdout); } +#[test] +fn the_hook_divides_the_node_cores_not_easybuild_parallel() { + let hooks = arg("hooks/integers.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--hooks", + &hooks, + "--parallel", + "12", + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + // 16 // 2 is 8; 12 // 2 would have been 6. + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + parallelism under the current eb_hooks.py limit and under limit_by_memory\n\ + node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 12\n \ + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n\ + node 48 cores, 30016 MiB, aarch64/a64fx, --parallel 12\n \ + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n \ + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured\n\ + job memory: libxc 7.0 unmeasured, no mib_per_job\n" + ) + ); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--hooks", + &hooks, + "--parallel", + "4", + "--node", + "16:32768:x86_64/amd/zen4", + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains( + " libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured\n" + ), + "{stdout}" + ); + + // EasyBuild uses --parallel as given, above the cores; a build with no + // hook row runs at 32 on 16 cores once max-parallel allows it. + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--parallel", + "32", + "--max-parallel", + "64", + "--node", + "16:32768:x86_64/amd/zen4", + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains(" Qt5: current 32 (no hook limit, --parallel 32); memory unmeasured\n"), + "{stdout}" + ); + assert!( + stdout.contains( + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n" + ), + "{stdout}" + ); + + let (_, default, _) = run(&["stack", "oom", "--node", "48:30016:aarch64/a64fx"]); + assert!( + default.contains(" LLVM: current 16 (divide_by_factor on *, max(1, 48 // 2) = 24, then max-parallel 16); memory unmeasured\n"), + "{default}" + ); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--max-parallel", + "48", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains( + " LLVM: current 24 (divide_by_factor on *, max(1, 48 // 2)); memory unmeasured\n" + ), + "{stdout}" + ); +} + +#[test] +fn easybuild_parallel_12_still_leaves_libxc_at_8_on_zen4() { + let easystack = arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--parallel", + "12", + "--easystack", + &easystack, + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains(" parallel 8: --parallel 12 on 16 cores, eb_hooks.py divides libxc by 2 on the 16 cores, not on --parallel 12\n"), + "{stdout}" + ); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--parallel", + "4", + "--easystack", + &easystack, + ]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!(stdout, "oom=0 fits=2 builds=5 nodes=2\n"); +} + +#[test] +fn a_zero_easybuild_parallel_level_is_rejected() { + for flag in ["--parallel", "--max-parallel"] { + let (code, stdout, stderr) = run(&["stack", "oom", flag, "0", "--node", "16:32768"]); + assert_eq!(code, 2, "{stdout}\n{stderr}"); + assert!( + stderr.contains(&format!("{flag} must be a positive integer")), + "{stderr}" + ); + assert!(stdout.is_empty(), "{stdout}"); + } +} + #[test] fn a_node_size_without_cores_or_memory_is_rejected() { let (code, _, stderr) = run(&["stack", "oom", "--node", "16"]); @@ -646,6 +791,6 @@ fn the_how_to_shows_the_real_output() { "{command}" ); } - assert_eq!(oom_blocks, 11); + assert_eq!(oom_blocks, 13); assert_eq!(rss_blocks, 1); } From e45fcdb2aa5111e0700b956a1c01a96418168888 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:44:08 +0000 Subject: [PATCH 5/7] docs(stack): check two forms of the libxc row with `--hooks` The fixture `hooks/set-maximum-8.py` is `hooks/integers.py` with libxc's '*' row a cap of 8, `(set_maximum, 8)`, in place of `(divide_by_factor, 2)`: the form proposed in the review of software-layer-scripts pull request 337. With it, `stack oom --hooks` gives libxc 8 on the 16-core zen4 node, the same 8 as `16 // 2`, and pull request 1772 still warns on the 32 GiB zen4 node; only the parallel line changes, to "eb_hooks.py caps libxc at 8". A cap does not scale with the node, so the two rows differ on 64 cores (8 against 16 after max-parallel) and on 12 (8 against 6). The fixture `hooks/divide-8.py` has a reduction of 8, `(divide_by_factor, 8)`, as asked in the review of 337 (issuecomment-6599147327): `max(1, 16 // 8)` is 2, and pull request 1772 fits on both zen4 nodes. The how-to gets a section with the three commands. The eessi-extend skill says to check a scripts commit with `--hooks` before a software-layer pull request pins it in `bot/software_layer_scripts_commit`. The changelog has a Documentation entry. --- CHANGELOG.md | 9 ++ docs/orgmode/howto/oom-profile.org | 84 ++++++++++++ llms-full.txt | 84 ++++++++++++ skills/eessi-extend/SKILL.md | 6 + tests/oom_profile.rs | 128 +++++++++++++++++- tests/oom_profile_fixtures/hooks/divide-8.py | 24 ++++ .../hooks/set-maximum-8.py | 23 ++++ 7 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 tests/oom_profile_fixtures/hooks/divide-8.py create mode 100644 tests/oom_profile_fixtures/hooks/set-maximum-8.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 315579c9..aea9451c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,15 @@ All notable changes to this project are documented here. - How to read the reproduction scoreboard and hold it to a ratchet file. - How to report a site-layer recipe that shadows a module the EESSI base stack already provides. +- The out-of-memory how-to checks two forms of libxc's row with + `--hooks`: a cap of 8, `(set_maximum, 8)`, the form proposed in the + review of software-layer-scripts #337, and a reduction of 8, + `(divide_by_factor, 8)`, as asked in the review of #337 + (issuecomment-6599147327). On the bot's 16-core nodes the cap gives 8, + the same as dividing by 2, and pull request 1772 still warns on the + 32 GiB zen4 node; the reduction gives 2 and fits. A cap does not scale + with the node: it gives 8 on 64 cores and on 12. The fixtures are + `hooks/set-maximum-8.py` and `hooks/divide-8.py`. ## 0.4.0 - 2026-11-01 diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index 63eec7ea..65c60851 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -459,6 +459,90 @@ give 16. On 4 GiB not one job fits. ~limit_by_memory~ still says 1, because it never goes below one job. The memory cap is 0: one compile already needs more than the node has. +* A cap of 8 or a reduction of 8 for libxc + +Two forms of libxc's ~'*'~ row in ~PARALLELISM_LIMITS~ have come up in +the review of software-layer-scripts pull request 337. Each fixture is +~hooks/integers.py~ with that row in place of ~(divide_by_factor, 2)~, +and the A64FX row stays at ~(set_maximum, 6)~: + +- ~hooks/set-maximum-8.py~ has a cap of 8, ~(set_maximum, 8)~, the form + proposed in review. +- ~hooks/divide-8.py~ has a reduction of 8, ~(divide_by_factor, 8)~, as + asked in the review of 337 (issuecomment-6599147327). + +Compare node sizes under the cap of 8: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --node 16:32768:x86_64/amd/zen4 \ + --node 48:30016:aarch64/a64fx \ + --node 64:262144:x86_64/amd/zen4 \ + --node 12:24576:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(16, 8)); memory unmeasured +node 48 cores, 30016 MiB, aarch64/a64fx + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured +node 64 cores, 262144 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(64, 8)); memory unmeasured +node 12 cores, 24576 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(12, 8)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +#+end_example + +With ~hooks/integers.py~ the same four nodes give libxc 8, 6, 16 and 6. +A cap is a number of jobs, so it does not grow with the node: on 64 +cores it gives 8, where ~64 // 2~ is 32 and max-parallel holds that at +16. It does not shrink either: on 12 cores it gives 8, where ~12 // 2~ +is 6. The two rows agree on 16 and 17 cores. Every node in the bot +resource table has 16 cores except A64FX, which keeps its own row, so +on the bot the cap leaves libxc where it is. Pull request 1772 still +warns on the 32 GiB zen4 node: + +#+begin_src bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) + parallel 8: 16 cores, eb_hooks.py caps libxc at 8 + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 + remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 +#+end_example + +The estimate is the one in the first example. Only the ~parallel~ line +changes, and it names the cap in place of the divide. The reduction of +8 in ~hooks/divide-8.py~ leaves 2 cores on these nodes: + +#+begin_src bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-8.py +oom=0 fits=2 builds=5 nodes=2 +#+end_example + +~max(1, 16 // 8)~ is 2, so libxc fits on both zen4 nodes. On 64 cores +the same row gives 8. + * Measure the worst compile ~mib_per_job~ comes from a compiler launcher, not from a guess. diff --git a/llms-full.txt b/llms-full.txt index 8f76d241..d1e5d00f 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -5052,6 +5052,90 @@ give 16. On 4 GiB not one job fits. `limit_by_memory` still says 1, because it never goes below one job. The memory cap is 0: one compile already needs more than the node has. +# A cap of 8 or a reduction of 8 for libxc + +Two forms of libxc's `'*'` row in `PARALLELISM_LIMITS` have come up in +the review of software-layer-scripts pull request 337. Each fixture is +`hooks/integers.py` with that row in place of `(divide_by_factor, 2)`, +and the A64FX row stays at `(set_maximum, 6)`: + +- `hooks/set-maximum-8.py` has a cap of 8, `(set_maximum, 8)`, the form + proposed in review. +- `hooks/divide-8.py` has a reduction of 8, `(divide_by_factor, 8)`, as + asked in the review of 337 (issuecomment-6599147327). + +Compare node sizes under the cap of 8: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --node 16:32768:x86_64/amd/zen4 \ + --node 48:30016:aarch64/a64fx \ + --node 64:262144:x86_64/amd/zen4 \ + --node 12:24576:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(16, 8)); memory unmeasured +node 48 cores, 30016 MiB, aarch64/a64fx + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured +node 64 cores, 262144 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(64, 8)); memory unmeasured +node 12 cores, 24576 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(12, 8)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +``` + +With `hooks/integers.py` the same four nodes give libxc 8, 6, 16 and 6. +A cap is a number of jobs, so it does not grow with the node: on 64 +cores it gives 8, where `64 // 2` is 32 and max-parallel holds that at +16. It does not shrink either: on 12 cores it gives 8, where `12 // 2` +is 6. The two rows agree on 16 and 17 cores. Every node in the bot +resource table has 16 cores except A64FX, which keeps its own row, so +on the bot the cap leaves libxc where it is. Pull request 1772 still +warns on the 32 GiB zen4 node: + +```bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) + parallel 8: 16 cores, eb_hooks.py caps libxc at 8 + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 + remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 +``` + +The estimate is the one in the first example. Only the `parallel` line +changes, and it names the cap in place of the divide. The reduction of +8 in `hooks/divide-8.py` leaves 2 cores on these nodes: + +```bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-8.py +oom=0 fits=2 builds=5 nodes=2 +``` + +`max(1, 16 // 8)` is 2, so libxc fits on both zen4 nodes. On 64 cores +the same row gives 8. + # Measure the worst compile `mib_per_job` comes from a compiler launcher, not from a guess. diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index 773087e6..55677671 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -349,6 +349,12 @@ from the file instead of the bundled copy, and the warning names the line. `eb_hooks.py` also has a `msg+` where the log line should use `+=`; half the message never prints. That is upstream. Do not patch it from here. `--hooks` prints it as a warning with the line and `post_ready_hook`. +A cap does not scale with the node: on the bot's 16-core nodes +`(set_maximum, 8)` and `(divide_by_factor, 2)` both leave 8, and 2 cores +of 16 is `(divide_by_factor, 8)`. Before a software-layer pull request +points `bot/software_layer_scripts_commit` at a scripts change, run +`stack oom --hooks` on that commit's `eb_hooks.py` with `--node` for the +bot node, and check that the level is the one asked for. `eb-stack stack oom --node 16:32768:x86_64/amd/zen4` compares the hook with `limit_by_memory`. `data/job-memory.toml` holds `mib_per_job`; diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index 917a1579..9539846a 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -659,6 +659,132 @@ fn a_zero_easybuild_parallel_level_is_rejected() { } } +#[test] +fn a_libxc_cap_of_8_is_the_divide_by_2_on_16_cores() { + let nodes = [ + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + "--node", + "64:262144:x86_64/amd/zen4", + "--node", + "12:24576:x86_64/amd/zen4", + ]; + let hooks = arg("hooks/set-maximum-8.py"); + let (code, stdout, stderr) = run(&[&["stack", "oom", "--hooks", &hooks][..], &nodes].concat()); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + parallelism under the current eb_hooks.py limit and under limit_by_memory\n\ + node 16 cores, 32768 MiB, x86_64/amd/zen4\n \ + libxc: current 8 (set_maximum on *, min(16, 8)); memory unmeasured\n\ + node 48 cores, 30016 MiB, aarch64/a64fx\n \ + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n \ + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured\n\ + node 64 cores, 262144 MiB, x86_64/amd/zen4\n \ + libxc: current 8 (set_maximum on *, min(64, 8)); memory unmeasured\n\ + node 12 cores, 24576 MiB, x86_64/amd/zen4\n \ + libxc: current 8 (set_maximum on *, min(12, 8)); memory unmeasured\n\ + job memory: libxc 7.0 unmeasured, no mib_per_job\n" + ) + ); + + // The same nodes under the divide by 2 that the scripts ship. + let (code, divide, stderr) = run(&[ + &["stack", "oom", "--hooks", &arg("hooks/integers.py")][..], + &nodes, + ] + .concat()); + assert_eq!(code, 0, "{stderr}"); + let libxc: Vec<&str> = divide + .lines() + .filter(|line| line.starts_with(" libxc: ")) + .collect(); + assert_eq!( + libxc, + [ + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured", + " libxc: current 16 (divide_by_factor on *, max(1, 64 // 2) = 32, then max-parallel 16); memory unmeasured", + " libxc: current 6 (divide_by_factor on *, max(1, 12 // 2)); memory unmeasured", + ] + ); +} + +#[test] +fn libxc_from_pr_1772_still_warns_on_zen4_under_a_cap_of_8() { + let hooks = arg("hooks/set-maximum-8.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.starts_with(&format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772)\n \ + parallel 8: 16 cores, eb_hooks.py caps libxc at 8\n" + )), + "{stdout}" + ); + assert!(!stdout.contains("warning:"), "{stdout}"); + // The 180 GiB zen4 node fits at 8 under either row. + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); +} + +#[test] +fn a_reduction_of_8_leaves_libxc_2_cores() { + let hooks = arg("hooks/divide-8.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--hooks", + &hooks, + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + for line in [ + " libxc: current 2 (divide_by_factor on *, max(1, 16 // 8)); memory unmeasured\n", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n", + ] { + assert!(stdout.contains(line), "{stdout}"); + } + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + oom=0 fits=2 builds=5 nodes=2\n" + ) + ); +} + #[test] fn a_node_size_without_cores_or_memory_is_rejected() { let (code, _, stderr) = run(&["stack", "oom", "--node", "16"]); @@ -791,6 +917,6 @@ fn the_how_to_shows_the_real_output() { "{command}" ); } - assert_eq!(oom_blocks, 13); + assert_eq!(oom_blocks, 16); assert_eq!(rss_blocks, 1); } diff --git a/tests/oom_profile_fixtures/hooks/divide-8.py b/tests/oom_profile_fixtures/hooks/divide-8.py new file mode 100644 index 00000000..64741285 --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/divide-8.py @@ -0,0 +1,24 @@ +# integers.py with libxc's '*' row a reduction of 8, (divide_by_factor, 8), +# as asked in the review of EESSI/software-layer-scripts#337 +# (issuecomment-6599147327): 2 cores on a 16-core build node. The A64FX +# row stays at 6. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 8), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} diff --git a/tests/oom_profile_fixtures/hooks/set-maximum-8.py b/tests/oom_profile_fixtures/hooks/set-maximum-8.py new file mode 100644 index 00000000..72a7d08d --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/set-maximum-8.py @@ -0,0 +1,23 @@ +# integers.py with libxc's '*' row a cap of 8, (set_maximum, 8), in place +# of the divide by 2: the form proposed in the review of +# EESSI/software-layer-scripts#337. The A64FX row stays at 6. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (set_maximum, 8), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} From b83b945a2d4178b8c0716cda433e1679c6f398a4 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:44:38 +0000 Subject: [PATCH 6/7] docs(stack): link the first `--node` examples in the oom how-to The `--hooks` and `--parallel` sections of the out-of-memory how-to run `--node` before "A memory limit instead of a core fraction" says what it prints. The `--hooks` section links to that section, and the `--parallel` section names the `CORES:MEMORY_MIB:TARGET` form before its first example. --- docs/orgmode/howto/oom-profile.org | 8 +++++--- llms-full.txt | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index 65c60851..e673c17a 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -262,8 +262,9 @@ hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers. #+end_example The libxc and A64FX rows of ~integers.py~ are the ones in the bot -resource table, so ~--node~ prints the same parallel levels with and -without it. For the real hooks, pass the file from a software-layer-scripts +resource table, so ~--node~ (see +[[*A memory limit instead of a core fraction][below]]) prints the same +parallel levels with and without it. For the real hooks, pass the file from a software-layer-scripts checkout, ~--hooks software-layer-scripts/eb_hooks.py~. A key is a string or a module-level string constant such as @@ -299,7 +300,8 @@ it is below EasyBuild's level. EasyBuild's level comes from core count when it is set, capped at the easyconfig's ~maxparallel~ and at ~--max-parallel~ (~EASYBUILD_MAX_PARALLEL~, 16 by default). The bot sets neither. ~stack oom --parallel N~ and ~--max-parallel N~ model a -job that does: +job that does. ~--node CORES:MEMORY_MIB:TARGET~ prints the levels for +one node size: #+begin_src bash eb-stack stack oom \ diff --git a/llms-full.txt b/llms-full.txt index d1e5d00f..47880f64 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4855,8 +4855,9 @@ hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers. ``` The libxc and A64FX rows of `integers.py` are the ones in the bot -resource table, so `--node` prints the same parallel levels with and -without it. For the real hooks, pass the file from a software-layer-scripts +resource table, so `--node` (see +[[*A memory limit instead of a core fraction][below]]) prints the same +parallel levels with and without it. For the real hooks, pass the file from a software-layer-scripts checkout, `--hooks software-layer-scripts/eb_hooks.py`. A key is a string or a module-level string constant such as @@ -4892,7 +4893,8 @@ it is below EasyBuild's level. EasyBuild's level comes from core count when it is set, capped at the easyconfig's `maxparallel` and at `--max-parallel` (`EASYBUILD_MAX_PARALLEL`, 16 by default). The bot sets neither. `stack oom --parallel N` and `--max-parallel N` model a -job that does: +job that does. `--node CORES:MEMORY_MIB:TARGET` prints the levels for +one node size: ```bash eb-stack stack oom \ From e435f29f23aa3374e2a2033130885ed0e28756f0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 3 Nov 2026 10:51:16 +0000 Subject: [PATCH 7/7] fix(stack): record the libxc job that ran out at make -j 5 on zen4 Bot job 221273 of software-layer pull request 1772 built libxc-7.0.0-GCC-15.2.0.eb on the 32 GiB Amazon Web Services zen4 node with `bot/software_layer_scripts_commit` set to a software-layer-scripts commit that divides libxc's parallelism by 3. Since `16 // 3` is 5, the log shows `make -j 5`, and cc1 was killed (issuecomment-6599075713). The table in `data/build-memory.toml` did not have that job, and with 4.0 GiB per process it predicted that 5 fits (20 GiB of 32). The observation is added, and the zen4 figure becomes the lower bound that explains it: 32 GiB over 5 is 6.4 GiB per process. The product `5 x 6.4` is exactly 32.0 in f64, so the table still reproduces the job. The two jobs at 8 ran out under both figures. A libxc warning on that node now reads `8 x 6.4 GiB = 51.2 GiB` and cites software-layer-scripts 337 as well. The maxparallel remedy is 4 in place of 7. The why line names 8 and then 5 compilers. The remedy line says the divider of 3 ran out too. It names a reduction of 8, 2 cores, from the review of 337 (issuecomment-6599147327). The review says to "really crank it" and "see if that works". The divider of 2.5 gives 6 processes or 38.4 GiB and no longer fits, so the float examples in the how-to and the tests now also warn about memory. The eessi-extend skill, the changelog, and the examples in the CLI reference and how-to follow. --- CHANGELOG.md | 13 ++++++ data/build-memory.toml | 26 +++++++++--- docs/orgmode/howto/oom-profile.org | 52 ++++++++++++++--------- docs/orgmode/reference/cli.org | 14 +++---- llms-full.txt | 66 ++++++++++++++++++------------ skills/eessi-extend/SKILL.md | 9 ++-- src/oom_profile.rs | 2 +- tests/oom_profile.rs | 44 ++++++++++++++------ 8 files changed, 151 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aea9451c..db0fbe24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,19 @@ All notable changes to this project are documented here. hook's result was not lower. 0 for either flag is status 2. `eb_stack_oom` takes `parallel` and `max_parallel`. +### Changed + +- `data/build-memory.toml` records software-layer #1772 bot job 221273: + libxc 7.0.0 with GCC 15.2.0 ran out of the 32 GiB zen4 job at + `make -j 5`, under a libxc divider of 3 in the software-layer-scripts + commit the job used. The zen4 figure is now a lower bound of 6.4 GiB + per process, 32 over 5, in place of 4.0, 32 over 8. A libxc warning + on that node reads `8 x 6.4 GiB = 51.2 GiB` and suggests + `maxparallel = 4` in place of 7, a divider of 2.5 (6 processes, + 38.4 GiB) no longer fits, and the remedy names a reduction of 8, as + asked in the review of software-layer-scripts #337 + (issuecomment-6599147327). + ### Documentation - How to split one Jenkins build list into shards that build at the diff --git a/data/build-memory.toml b/data/build-memory.toml index ef5a061f..8dcdd7b7 100644 --- a/data/build-memory.toml +++ b/data/build-memory.toml @@ -28,18 +28,24 @@ name = "libxc" versions = ["7.0"] toolchains = ["GCC-15.2"] targets = ["x86_64/amd/zen4"] -per_process_gib = 4.0 +per_process_gib = 6.4 bound = "lower" -reason = "cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4" -remedy = "the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster" +reason = "cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job" +remedy = "a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster" sources = [ "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6420263033", "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6440377764", + "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6599075713", + "https://github.com/EESSI/software-layer-scripts/pull/337#issuecomment-6599147327", ] -# 4.0 GiB is 32 GiB over the 8 compilers the hook leaves of 16 cores. -# aarch64/generic built the same easyconfig at 8 on 29.8 GiB in the same -# pull request, so the entry is scoped to zen4. +# 6.4 GiB is 32 GiB over the 5 compilers of job 221273, which ran +# make -j 5 under a libxc divider of 3 (16 // 3); the two jobs at 8 ran +# out as well. The review of #337 suspects that a single compile is the +# bottleneck. These jobs cannot tell that apart from 6.4 GiB each, so +# the figure stays a lower bound. aarch64/generic built the same +# easyconfig at 8 on 29.8 GiB in the same pull request, so the entry is +# scoped to zen4. [[entry.observation]] easyconfig = "libxc-7.0.0-GCC-15.2.0.eb" target = "x86_64/amd/zen4" @@ -56,6 +62,14 @@ memory_gib = 32.0 outcome = "oom" source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6440377764" +[[entry.observation]] +easyconfig = "libxc-7.0.0-GCC-15.2.0.eb" +target = "x86_64/amd/zen4" +parallel = 5 +memory_gib = 32.0 +outcome = "oom" +source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6599075713" + [[entry]] name = "libxc" versions = ["7.0"] diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index e673c17a..966e86ee 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -10,8 +10,8 @@ The command never edits a recipe or a hook, and it always exits 0. Why you'd use it: a bot job that runs out of memory fails late and quietly. Software-layer pull request 1772 added ~libxc-7.0.0-GCC-15.2.0.eb~. The build was killed for lack of memory on -zen4 twice, at 8 processes on a 32 GiB job, before anyone changed the -parallel level. The same thing happened to Highway in #1692, to +zen4 twice at 8 processes on a 32 GiB job, and again at 5 after a hook +change. The same thing happened to Highway in #1692, to hipBLASLt and rocBLAS in #1549, and to PETSc in #1608. Run the check before you open the pull request or ask the bot to build. @@ -23,12 +23,12 @@ eb-stack stack oom \ The command prints: #+begin_example -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=14 builds=5 nodes=15 #+end_example @@ -218,13 +218,20 @@ eb-stack stack oom \ #+begin_example warning: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. -oom=0 fits=1 builds=5 nodes=1 +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: no other x86_64/amd/zen4 node in the bot resource table has room for it + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=0 builds=5 nodes=1 #+end_example -The warning is the float. The estimate then uses 6, so 6 times 4.0 GiB -is 24 GiB and the 32 GiB zen4 job fits. With the integer divider of 2 -the same build is the warning in the first example. The command still -exits 0. +The first warning is the float. The estimate then uses 6, and 6 times +6.4 GiB is 38.4 GiB, more than the 32 GiB zen4 job. The divider of 3 +ran out at 5 in bot job 221273 of #1772, so 6 does not fit either. The +fixture has only the AWS zen4 node, so no other node has room. The +command still exits 0. * Read the limits from eb_hooks.py @@ -245,13 +252,20 @@ eb-stack stack oom \ #+begin_example hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-2.5.py warning: tests/oom_profile_fixtures/hooks/divide-2.5.py:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. -oom=0 fits=2 builds=5 nodes=2 +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 #+end_example The first line says how many rows were read, and from which file. The -warning names the line of the ~(divide_by_factor, 2.5)~ tuple. The nodes -still come from the bot resource table, so both zen4 nodes are checked. -With every factor an integer there is no warning: +first warning names the line of the ~(divide_by_factor, 2.5)~ tuple. +The nodes still come from the bot resource table, so both zen4 nodes are +checked, and the 180 GiB node has room for 6. With every factor an +integer there is no ~warning:~ line: #+begin_src bash eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/integers.py @@ -517,12 +531,12 @@ eb-stack stack oom \ #+begin_example hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py caps libxc at 8 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=1 builds=5 nodes=2 #+end_example diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 280bd89c..f0c858c2 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -1102,9 +1102,9 @@ EESSI bot node, at the parallel level the bot job gives it, and warn where the estimate reaches the job's memory. Each warning names the remedies the maintainers have used. Nothing is changed. -Why you'd use it: software-layer pull request 1772 lost two zen4 bot -runs to a killed ~cc1~ while building libxc. The check names that build -before the first run. +Why you'd use it: software-layer pull request 1772 lost three zen4 bot +runs to a killed ~cc1~ while building libxc, two at 8 processes and one +at 5. The check names that build before the first run. #+begin_src bash eb-stack stack oom \ @@ -1113,12 +1113,12 @@ eb-stack stack oom \ #+end_src #+begin_example -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=1 builds=5 nodes=2 #+end_example diff --git a/llms-full.txt b/llms-full.txt index 47880f64..72c6afea 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4603,8 +4603,8 @@ The command never edits a recipe or a hook, and it always exits 0. Why you'd use it: a bot job that runs out of memory fails late and quietly. Software-layer pull request 1772 added `libxc-7.0.0-GCC-15.2.0.eb`. The build was killed for lack of memory on -zen4 twice, at 8 processes on a 32 GiB job, before anyone changed the -parallel level. The same thing happened to Highway in #1692, to +zen4 twice at 8 processes on a 32 GiB job, and again at 5 after a hook +change. The same thing happened to Highway in #1692, to hipBLASLt and rocBLAS in #1549, and to PETSc in #1608. Run the check before you open the pull request or ask the bot to build. @@ -4616,12 +4616,12 @@ eb-stack stack oom \ The command prints: ``` -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=14 builds=5 nodes=15 ``` @@ -4811,13 +4811,20 @@ eb-stack stack oom \ ``` warning: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. -oom=0 fits=1 builds=5 nodes=1 +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: no other x86_64/amd/zen4 node in the bot resource table has room for it + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=0 builds=5 nodes=1 ``` -The warning is the float. The estimate then uses 6, so 6 times 4.0 GiB -is 24 GiB and the 32 GiB zen4 job fits. With the integer divider of 2 -the same build is the warning in the first example. The command still -exits 0. +The first warning is the float. The estimate then uses 6, and 6 times +6.4 GiB is 38.4 GiB, more than the 32 GiB zen4 job. The divider of 3 +ran out at 5 in bot job 221273 of #1772, so 6 does not fit either. The +fixture has only the AWS zen4 node, so no other node has room. The +command still exits 0. # Read the limits from eb_hooks.py @@ -4838,13 +4845,20 @@ eb-stack stack oom \ ``` hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-2.5.py warning: tests/oom_profile_fixtures/hooks/divide-2.5.py:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. -oom=0 fits=2 builds=5 nodes=2 +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 ``` The first line says how many rows were read, and from which file. The -warning names the line of the `(divide_by_factor, 2.5)` tuple. The nodes -still come from the bot resource table, so both zen4 nodes are checked. -With every factor an integer there is no warning: +first warning names the line of the `(divide_by_factor, 2.5)` tuple. +The nodes still come from the bot resource table, so both zen4 nodes are +checked, and the 180 GiB node has room for 6. With every factor an +integer there is no `warning:` line: ```bash eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/integers.py @@ -5110,12 +5124,12 @@ eb-stack stack oom \ ``` hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py caps libxc at 8 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=1 builds=5 nodes=2 ``` @@ -7190,9 +7204,9 @@ EESSI bot node, at the parallel level the bot job gives it, and warn where the estimate reaches the job's memory. Each warning names the remedies the maintainers have used. Nothing is changed. -Why you'd use it: software-layer pull request 1772 lost two zen4 bot -runs to a killed `cc1` while building libxc. The check names that build -before the first run. +Why you'd use it: software-layer pull request 1772 lost three zen4 bot +runs to a killed `cc1` while building libxc, two at 8 processes and one +at 5. The check names that build before the first run. ```bash eb-stack stack oom \ @@ -7201,12 +7215,12 @@ eb-stack stack oom \ ``` ``` -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=1 builds=5 nodes=2 ``` diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index 55677671..bb0008c9 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -332,10 +332,11 @@ eb-stack stack oom \ A warning names the entry, the target and node, the parallel level and the estimate against the job's memory. The libxc build in software-layer -#1772 was killed twice on zen4 at 8 processes on 32 GiB. The remedies are -text only: a `maxparallel` in the recipe, a limit in `PARALLELISM_LIMITS` -in software-layer-scripts `eb_hooks.py`, or a node with more memory. Put -the change in the pull request yourself. The command exits 0. +#1772 was killed on zen4 twice at 8 processes on 32 GiB, and once at 5. +The remedies are text only: a `maxparallel` in the recipe, a limit in +`PARALLELISM_LIMITS` in software-layer-scripts `eb_hooks.py`, or a node +with more memory. Put the change in the pull request yourself. The +command exits 0. The hook ops are `divide_by_factor` (`max(1, parallel // factor)`) and `set_maximum` (`min(parallel, n)`). A CPU-target key overrides `*`. diff --git a/src/oom_profile.rs b/src/oom_profile.rs index e7ba9c42..b0c311f8 100644 --- a/src/oom_profile.rs +++ b/src/oom_profile.rs @@ -2128,7 +2128,7 @@ source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6420263 .iter() .find(|e| e.name == "libxc" && !e.toolchains.is_empty()) .unwrap(); - assert_eq!(fitting_parallel(entry, 8, 32.0), Some(7)); + assert_eq!(fitting_parallel(entry, 8, 32.0), Some(4)); let hip = profile .entries() .iter() diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index 9539846a..475f85f9 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -1,7 +1,8 @@ //! Out-of-memory warnings for a plan, from EESSI software-layer bot jobs. //! //! Each easystack is the lines a real pull request added. The libxc build -//! in pull request 1772 was killed twice on zen4. Highway in 1692 ran out on +//! in pull request 1772 was killed on zen4 twice at 8 and once at 5. +//! Highway in 1692 ran out on //! aarch64/generic. hipBLASLt for rocBLAS in 1549 needed about 90 GiB. The //! PETSc test step in 1608 ran out on zen5 before the hooks halved it. @@ -35,21 +36,24 @@ fn libxc_from_pr_1772_warns_on_zen4_only() { let easystack = arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"); let (code, stdout, stderr) = run(&["stack", "oom", "--easystack", &easystack]); assert_eq!(code, 0, "{stdout}\n{stderr}"); - let first = "libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772)"; + let first = "libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337)"; assert!(stdout.contains(first), "{stdout}"); assert!( stdout.contains(" parallel 8: 16 cores, eb_hooks.py divides libxc by 2\n"), "{stdout}" ); assert!( - stdout.contains(" remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many\n"), + stdout.contains(" remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many\n"), "{stdout}" ); assert!( stdout.contains(" remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90)\n"), "{stdout}" ); - assert!(stdout.contains("divider of 2.5 or 3"), "{stdout}"); + assert!( + stdout.contains(" remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster\n"), + "{stdout}" + ); // aarch64/generic built the same easyconfig at 8 cores in that pull request. assert!(!stdout.contains("on aarch64/generic"), "{stdout}"); assert_eq!( @@ -238,9 +242,20 @@ fn a_non_integer_divide_warns_and_the_estimate_uses_six() { ), "{stdout}" ); - assert!(!stdout.contains("may run out of memory"), "{stdout}"); assert!( - stdout.ends_with("oom=0 fits=1 builds=5 nodes=1\n"), + stdout.contains( + "libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB" + ), + "{stdout}" + ); + assert!( + stdout.contains( + " parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float)\n" + ), + "{stdout}" + ); + assert!( + stdout.ends_with("oom=1 fits=0 builds=5 nodes=1\n"), "{stdout}" ); } @@ -259,13 +274,18 @@ fn an_eb_hooks_py_with_a_float_divider_warns_at_its_line() { &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), ]); assert_eq!(code, 0, "{stdout}\n{stderr}"); - assert_eq!( - stdout, - format!( + assert!( + stdout.starts_with(&format!( "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ warning: {hooks}:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor.\n\ - oom=0 fits=2 builds=5 nodes=2\n" - ) + libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB" + )), + "{stdout}" + ); + assert_eq!(stdout.matches("warning:").count(), 1, "{stdout}"); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" ); } @@ -731,7 +751,7 @@ fn libxc_from_pr_1772_still_warns_on_zen4_under_a_cap_of_8() { assert!( stdout.starts_with(&format!( "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ - libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772)\n \ + libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337)\n \ parallel 8: 16 cores, eb_hooks.py caps libxc at 8\n" )), "{stdout}"