diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b1898e..db0fbe24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,53 @@ All notable changes to this project are documented here. records the closures, the shard weights and a SHA-256 of the inputs. A line that cannot be resolved exits 1 and writes nothing. `eb_stack_shard` is the same command over MCP. +- `stack oom --hooks FILE` reads `PARALLELISM_LIMITS` from an EESSI + `eb_hooks.py` in place of the hook rows of + `data/eessi-bot-resources.toml`. The file is parsed as Python and + never run. A key may be a module-level string constant such as + `CPU_TARGET_A64FX`, and a repeated key keeps its last value. A + `divide_by_factor` that is not an integer is a warning that names the + file and line (`16 // 2.5` is `6.0`, a float). An op other than + `divide_by_factor` or `set_maximum`, a `**` unpacking, or a key that + is not a string exits 1 with the line. `eb_stack_oom` takes `hooks`. +- `stack oom --hooks FILE` warns on a statement that adds to a name and + drops the sum, such as `msg+ "..."` in the EESSI `post_ready_hook`: + the second half of its log message never prints. The warning names + the file, the line, and the function; `msg += "..."` is not reported, + and the parallel levels do not change. `eb_stack_oom` returns these + as `hook_warnings`. +- `stack oom --node` prints the memory cap beside the level the hook + leaves, for a package with an `mib_per_job`: + `min(current, memory_mib // mib_per_job)`, the same quotient as + `memory_bytes // (mib_per_job * 2**20)`. It is not raised to 1, so + `memory cap 0 (...), not one N MiB job fits` says one compile needs + more than the node has. `--mib-per-job FILE` is a visible alias of + `--job-memory FILE`. The `memory` column, `limit_by_memory` in place of + the hook, is unchanged. +- `stack oom --parallel N` and `--max-parallel N` model a job that sets + EasyBuild's `--parallel` or `--max-parallel`; the bot sets neither. + EasyBuild uses `--parallel` in place of the core count, capped at + max-parallel and `maxparallel`. The EESSI `post_ready_hook` still + applies its op to the node's cores (`det_parallelism()` with no + arguments) and keeps the result only when it is lower: with + `--parallel 12` on 16 cores libxc gets 16 // 2 = 8, and with + `--parallel 4` it stays at 4. The `--node` lines and a warning's + `parallel` line name the setting that decided, and say when the + hook's result was not lower. 0 for either flag is status 2. + `eb_stack_oom` takes `parallel` and `max_parallel`. + +### Changed + +- `data/build-memory.toml` records software-layer #1772 bot job 221273: + libxc 7.0.0 with GCC 15.2.0 ran out of the 32 GiB zen4 job at + `make -j 5`, under a libxc divider of 3 in the software-layer-scripts + commit the job used. The zen4 figure is now a lower bound of 6.4 GiB + per process, 32 over 5, in place of 4.0, 32 over 8. A libxc warning + on that node reads `8 x 6.4 GiB = 51.2 GiB` and suggests + `maxparallel = 4` in place of 7, a divider of 2.5 (6 processes, + 38.4 GiB) no longer fits, and the remedy names a reduction of 8, as + asked in the review of software-layer-scripts #337 + (issuecomment-6599147327). ### Documentation @@ -90,6 +137,15 @@ All notable changes to this project are documented here. - How to read the reproduction scoreboard and hold it to a ratchet file. - How to report a site-layer recipe that shadows a module the EESSI base stack already provides. +- The out-of-memory how-to checks two forms of libxc's row with + `--hooks`: a cap of 8, `(set_maximum, 8)`, the form proposed in the + review of software-layer-scripts #337, and a reduction of 8, + `(divide_by_factor, 8)`, as asked in the review of #337 + (issuecomment-6599147327). On the bot's 16-core nodes the cap gives 8, + the same as dividing by 2, and pull request 1772 still warns on the + 32 GiB zen4 node; the reduction gives 2 and fits. A cap does not scale + with the node: it gives 8 on 64 cores and on 12. The fixtures are + `hooks/set-maximum-8.py` and `hooks/divide-8.py`. ## 0.4.0 - 2026-11-01 diff --git a/data/build-memory.toml b/data/build-memory.toml index ef5a061f..8dcdd7b7 100644 --- a/data/build-memory.toml +++ b/data/build-memory.toml @@ -28,18 +28,24 @@ name = "libxc" versions = ["7.0"] toolchains = ["GCC-15.2"] targets = ["x86_64/amd/zen4"] -per_process_gib = 4.0 +per_process_gib = 6.4 bound = "lower" -reason = "cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4" -remedy = "the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster" +reason = "cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job" +remedy = "a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster" sources = [ "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6420263033", "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6440377764", + "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6599075713", + "https://github.com/EESSI/software-layer-scripts/pull/337#issuecomment-6599147327", ] -# 4.0 GiB is 32 GiB over the 8 compilers the hook leaves of 16 cores. -# aarch64/generic built the same easyconfig at 8 on 29.8 GiB in the same -# pull request, so the entry is scoped to zen4. +# 6.4 GiB is 32 GiB over the 5 compilers of job 221273, which ran +# make -j 5 under a libxc divider of 3 (16 // 3); the two jobs at 8 ran +# out as well. The review of #337 suspects that a single compile is the +# bottleneck. These jobs cannot tell that apart from 6.4 GiB each, so +# the figure stays a lower bound. aarch64/generic built the same +# easyconfig at 8 on 29.8 GiB in the same pull request, so the entry is +# scoped to zen4. [[entry.observation]] easyconfig = "libxc-7.0.0-GCC-15.2.0.eb" target = "x86_64/amd/zen4" @@ -56,6 +62,14 @@ memory_gib = 32.0 outcome = "oom" source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6440377764" +[[entry.observation]] +easyconfig = "libxc-7.0.0-GCC-15.2.0.eb" +target = "x86_64/amd/zen4" +parallel = 5 +memory_gib = 32.0 +outcome = "oom" +source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6599075713" + [[entry]] name = "libxc" versions = ["7.0"] diff --git a/docs/orgmode/howto/oom-profile.org b/docs/orgmode/howto/oom-profile.org index 4cda00a6..966e86ee 100644 --- a/docs/orgmode/howto/oom-profile.org +++ b/docs/orgmode/howto/oom-profile.org @@ -10,8 +10,8 @@ The command never edits a recipe or a hook, and it always exits 0. Why you'd use it: a bot job that runs out of memory fails late and quietly. Software-layer pull request 1772 added ~libxc-7.0.0-GCC-15.2.0.eb~. The build was killed for lack of memory on -zen4 twice, at 8 processes on a 32 GiB job, before anyone changed the -parallel level. The same thing happened to Highway in #1692, to +zen4 twice at 8 processes on a 32 GiB job, and again at 5 after a hook +change. The same thing happened to Highway in #1692, to hipBLASLt and rocBLAS in #1549, and to PETSc in #1608. Run the check before you open the pull request or ask the bot to build. @@ -23,12 +23,12 @@ eb-stack stack oom \ The command prints: #+begin_example -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=14 builds=5 nodes=15 #+end_example @@ -112,9 +112,11 @@ bigger node. The parallel level comes from the hook limits in ~data/eessi-bot-resources.toml~. Pass ~--resources~ to try other limits -or other nodes. Pull request 1608 ran out of memory in the PETSc test -step on zen5 before software-layer-scripts #298 halved PETSc. With the -current hooks, PETSc fits: +or other nodes, or ~--hooks~ to read the limits from an ~eb_hooks.py~ +(see [[*Read the limits from eb_hooks.py][below]]). Pull request 1608 +ran out of memory in the PETSc test step on zen5 before +software-layer-scripts #298 halved PETSc. With the current hooks, PETSc +fits: #+begin_src bash eb-stack stack oom \ @@ -166,9 +168,9 @@ or bot configuration. also holds EasyBuild's default maximum parallel level of 16 and the ~PARALLELISM_LIMITS~ table from the EESSI ~eb_hooks.py~. -The parallel level is the smallest of these: the node's cores, -EasyBuild's maximum, the hook limit applied to the cores, and a -top-level ~maxparallel~ in the recipe. The hook ops are the ones in +The parallel level is the smallest of these: the node's cores, or +~--parallel~ in their place; EasyBuild's maximum; the hook limit applied +to the cores; and a top-level ~maxparallel~ in the recipe. The hook ops are the ones in ~eb_hooks.py~. ~divide_by_factor(parallel, factor)~ is ~max(1, parallel // factor)~. ~set_maximum(parallel, n)~ is ~min(parallel, n)~. A software name uses its limit for the CPU target, @@ -196,7 +198,8 @@ msg+ "to avoid out-of-memory failures during building/testing" The second statement adds nothing. ~msg + "..."~ is an expression, and the result is thrown away. The operator that appends is ~+=~. The line that prints stops after the CPU target. This is upstream ~eb_hooks.py~. -eb-stack does not patch it. +eb-stack does not patch it. ~--hooks~ reports it with its line (see +[[*Read the limits from eb_hooks.py][below]]). * A divide factor that is not an integer @@ -215,13 +218,151 @@ eb-stack stack oom \ #+begin_example warning: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. -oom=0 fits=1 builds=5 nodes=1 +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: no other x86_64/amd/zen4 node in the bot resource table has room for it + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=0 builds=5 nodes=1 #+end_example -The warning is the float. The estimate then uses 6, so 6 times 4.0 GiB -is 24 GiB and the 32 GiB zen4 job fits. With the integer divider of 2 -the same build is the warning in the first example. The command still -exits 0. +The first warning is the float. The estimate then uses 6, and 6 times +6.4 GiB is 38.4 GiB, more than the 32 GiB zen4 job. The divider of 3 +ran out at 5 in bot job 221273 of #1772, so 6 does not fit either. The +fixture has only the AWS zen4 node, so no other node has room. The +command still exits 0. + +* Read the limits from eb_hooks.py + +The hook rows in ~data/eessi-bot-resources.toml~ are a copy of +~PARALLELISM_LIMITS~. ~--hooks~ reads that dict from an ~eb_hooks.py~ +instead, so a hook change can be checked before it is merged. The file +is parsed as Python, and nothing in it runs. The fixture +~hooks/divide-2.5.py~ has the layout of the real file and the libxc +divider of 2.5: + +#+begin_src bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-2.5.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-2.5.py +warning: tests/oom_profile_fixtures/hooks/divide-2.5.py:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 +#+end_example + +The first line says how many rows were read, and from which file. The +first warning names the line of the ~(divide_by_factor, 2.5)~ tuple. +The nodes still come from the bot resource table, so both zen4 nodes are +checked, and the 180 GiB node has room for 6. With every factor an +integer there is no ~warning:~ line: + +#+begin_src bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/integers.py +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +#+end_example + +The libxc and A64FX rows of ~integers.py~ are the ones in the bot +resource table, so ~--node~ (see +[[*A memory limit instead of a core fraction][below]]) prints the same +parallel levels with and without it. For the real hooks, pass the file from a software-layer-scripts +checkout, ~--hooks software-layer-scripts/eb_hooks.py~. + +A key is a string or a module-level string constant such as +~CPU_TARGET_A64FX~. A key the dict repeats keeps its last value, as in +Python. An op other than ~divide_by_factor~ or ~set_maximum~, a ~**~ +unpacking, or a key that is not a string stops the command with the +file and the line: the parallel levels would be wrong without that row. + +The same file is checked for a statement that adds to a name and drops +the sum, such as the ~msg+~ in ~post_ready_hook~. The fixture +~hooks/msg-plus.py~ has that statement: + +#+begin_src bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/msg-plus.py +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/msg-plus.py +warning: tests/oom_profile_fixtures/hooks/msg-plus.py:30: msg+ "to avoid out-of-memory failures during building/testing" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends +#+end_example + +The warning names the function and the line. It does not change a +parallel level, and the command still exits 0. ~msg += "..."~, as in +~hooks/msg-plus-equals.py~, appends and is not reported. + +* EasyBuild's --parallel and the hook + +~post_ready_hook~ does not divide the level EasyBuild chose. It calls +~det_parallelism()~ with no arguments, which counts the cores the job +can use, applies the op to that count, and keeps the result only when +it is below EasyBuild's level. EasyBuild's level comes from +~set_parallel~: ~--parallel~ (~EASYBUILD_PARALLEL~) in place of the +core count when it is set, capped at the easyconfig's ~maxparallel~ and +at ~--max-parallel~ (~EASYBUILD_MAX_PARALLEL~, 16 by default). The bot +sets neither. ~stack oom --parallel N~ and ~--max-parallel N~ model a +job that does. ~--node CORES:MEMORY_MIB:TARGET~ prints the levels for +one node size: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 12 \ + --node 16:32768:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 12 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +#+end_example + +libxc gets 8, which is 16 // 2. Half of ~--parallel 12~ would be 6. +With ~--parallel 4~ the hook's 8 is not lower, so it does not apply: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 4 \ + --node 16:32768:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 4 + libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +#+end_example + +A divide factor is a fraction of the node, not of ~--parallel~: +~--parallel~ above the hook's result does not change it, and +~--parallel~ below it wins. ~--parallel~ also replaces the core count in +EasyBuild, so with ~--parallel 32 --max-parallel 64~ a build with no +hook row runs at 32 on a 16-core node. With an easystack, the +~parallel~ line of a warning names ~--parallel~ and the cores the hook +divided, and says when the hook's result was not lower. + +At software-layer-scripts 43df6f4 the hook reads ~EESSI_SOFTWARE_SUBDIR~, +the software name, EasyBuild's level, and ~det_parallelism()~. No +environment variable there caps or overrides ~PARALLELISM_LIMITS~. +EESSI-extend's ~EESSI_EASYBUILD_HOOKS_OVERRIDE~ replaces the whole hooks +file; ~--hooks~ reads such a file. * A memory limit instead of a core fraction @@ -297,6 +438,127 @@ that name. The footer names the rows that are unmeasured. libxc 7.0 is the one the table calls out; the other names are absent from the table, which is the same as unmeasured. +With a number in the table, the line also prints the memory cap: the +level the hook leaves, with a memory ceiling on top, +~min(current, memory_mib // mib_per_job)~. The memory is a whole number +of MiB, so the quotient is the same as +~memory_bytes // (mib_per_job * 2**20)~. ~--mib-per-job FILE~ is the +same flag as ~--job-memory FILE~. The fixture +~job-memory-fixture.toml~ gives libxc 8192 MiB, which is arithmetic and +not a measurement. With ~hooks/integers.py~ the report is libxc alone: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --mib-per-job tests/oom_profile_fixtures/job-memory-fixture.toml \ + --node 16:32768:x86_64/amd/zen4 \ + --node 16:131072:x86_64/amd/zen4 \ + --node 16:4096:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192))) +node 16 cores, 131072 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 8 (min(8, 131072 // 8192)); memory 16 (max(1, min(16, 131072 // 8192))) +node 16 cores, 4096 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192))) +job memory: every package in the table has an mib_per_job +#+end_example + +On 32 GiB the memory allows four 8192 MiB jobs and the hook allows 8, +so the memory cap is 4. On 128 GiB the memory allows 16, and the cap +stays at the hook's 8; ~limit_by_memory~ in place of the hook would +give 16. On 4 GiB not one job fits. ~limit_by_memory~ still says 1, +because it never goes below one job. The memory cap is 0: one compile +already needs more than the node has. + +* A cap of 8 or a reduction of 8 for libxc + +Two forms of libxc's ~'*'~ row in ~PARALLELISM_LIMITS~ have come up in +the review of software-layer-scripts pull request 337. Each fixture is +~hooks/integers.py~ with that row in place of ~(divide_by_factor, 2)~, +and the A64FX row stays at ~(set_maximum, 6)~: + +- ~hooks/set-maximum-8.py~ has a cap of 8, ~(set_maximum, 8)~, the form + proposed in review. +- ~hooks/divide-8.py~ has a reduction of 8, ~(divide_by_factor, 8)~, as + asked in the review of 337 (issuecomment-6599147327). + +Compare node sizes under the cap of 8: + +#+begin_src bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --node 16:32768:x86_64/amd/zen4 \ + --node 48:30016:aarch64/a64fx \ + --node 64:262144:x86_64/amd/zen4 \ + --node 12:24576:x86_64/amd/zen4 +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(16, 8)); memory unmeasured +node 48 cores, 30016 MiB, aarch64/a64fx + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured +node 64 cores, 262144 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(64, 8)); memory unmeasured +node 12 cores, 24576 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(12, 8)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +#+end_example + +With ~hooks/integers.py~ the same four nodes give libxc 8, 6, 16 and 6. +A cap is a number of jobs, so it does not grow with the node: on 64 +cores it gives 8, where ~64 // 2~ is 32 and max-parallel holds that at +16. It does not shrink either: on 12 cores it gives 8, where ~12 // 2~ +is 6. The two rows agree on 16 and 17 cores. Every node in the bot +resource table has 16 cores except A64FX, which keeps its own row, so +on the bot the cap leaves libxc where it is. Pull request 1772 still +warns on the 32 GiB zen4 node: + +#+begin_src bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 8: 16 cores, eb_hooks.py caps libxc at 8 + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 +#+end_example + +The estimate is the one in the first example. Only the ~parallel~ line +changes, and it names the cap in place of the divide. The reduction of +8 in ~hooks/divide-8.py~ leaves 2 cores on these nodes: + +#+begin_src bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +#+end_src + +#+begin_example +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-8.py +oom=0 fits=2 builds=5 nodes=2 +#+end_example + +~max(1, 16 // 8)~ is 2, so libxc fits on both zen4 nodes. On 64 cores +the same row gives 8. + * Measure the worst compile ~mib_per_job~ comes from a compiler launcher, not from a guess. @@ -364,9 +626,15 @@ then. steps, test steps that run with a different parallel level, or memory that other jobs on the node use. It does not model ~ulimit -u~, which ~det_parallelism~ also consults. -- ~limit_by_memory~ does not replace the estimate. The estimate follows - the hooks that are in ~eb_hooks.py~. The ~--node~ report is the - comparison. A package with no measured ~mib_per_job~ stays +- ~--hooks~ reads the ~PARALLELISM_LIMITS~ literal. It does not run + ~post_ready_hook~; the rules that pick a row are the ones described + under [[*Where the numbers come from][where the numbers come from]]. + Of the rest of the file it checks only for a statement that drops a + sum. Other mistakes in the hooks are not found. +- ~limit_by_memory~ and the memory cap do not replace the estimate. The + estimate follows the hooks that are in ~eb_hooks.py~. The ~--node~ + report is the comparison, and the warnings for an easystack do not + print the cap. A package with no measured ~mib_per_job~ stays unmeasured. - The MCP tool ~eb_stack_oom~ returns the same lines and each warning as JSON. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 6bca18bf..f0c858c2 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -17,7 +17,7 @@ reachability, toolchain family, and the claim ladder are defined in | 1 | The work was attempted and failed: a missing or unreadable file, a document that does not parse, a check that found errors, a network fetch that failed, a doctor probe, or a campaign whose build failed | | 2 | The command line was rejected before that work: a missing or unknown flag (clap), or a flag value or combination the command does not accept | -A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails ~recipe lint~ or ~recipe check~ is status 1: the files were read and the check failed. ~stack from-commit~ is status 1 when any pin is ~released~, and status 0 when every pin is ~newer~ or ~unknown~ or the files name no pin. A file name with no ~eb-X.Y.Z~, and no ~--easybuild-version~, is status 2. ~stack arch~ is status 1 when an entry is unsupported or flagged, or when ~--probe-sources~ reports ~unreachable~ or ~arch-missing~ sources. ~stack files~ is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are ~no_url~ or ~unchecked~. ~stack generation~ is status 1 on any finding, and status 2 when an easystack path names no EESSI version and ~--eessi-version~ is absent. ~stack known-failures~ is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with ~--strict~ it is status 1 when a warning remains, and status 0 when every hit is remedied. ~stack oom~ is status 0 when it warns: the warnings are advice, not a failed check. A ~--node~ value that is not ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~ is status 2, and so is a command with neither ~--easystack~ nor ~--node~. ~stack bot-classify~ is status 0 for every class, and status 2 when a path is a directory. ~stack batch~ is status 1 when the plan includes, or depends on, an entry the architecture cannot build. ~stack provided~ is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets ~--hooks~, or belongs in a ~--family-lock~. A group-restricted warning is status 0 unless ~--strict~. Status 2 is when neither ~--installed~ nor ~--baseline-lock~ is set. ~stack shard~ is status 1 when a build-list line cannot be resolved, and writes nothing; ~--shards 0~ and a list with no lines are status 2. ~stack shadow~ is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. +A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails ~recipe lint~ or ~recipe check~ is status 1: the files were read and the check failed. ~stack from-commit~ is status 1 when any pin is ~released~, and status 0 when every pin is ~newer~ or ~unknown~ or the files name no pin. A file name with no ~eb-X.Y.Z~, and no ~--easybuild-version~, is status 2. ~stack arch~ is status 1 when an entry is unsupported or flagged, or when ~--probe-sources~ reports ~unreachable~ or ~arch-missing~ sources. ~stack files~ is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are ~no_url~ or ~unchecked~. ~stack generation~ is status 1 on any finding, and status 2 when an easystack path names no EESSI version and ~--eessi-version~ is absent. ~stack known-failures~ is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with ~--strict~ it is status 1 when a warning remains, and status 0 when every hit is remedied. ~stack oom~ is status 0 when it warns: the warnings are advice, not a failed check. A ~--node~ value that is not ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~ is status 2, and so is a command with none of ~--easystack~, ~--node~, and ~--hooks~, or a ~--parallel~ or ~--max-parallel~ of 0. A ~--hooks~ file that is not valid Python, has no module-level ~PARALLELISM_LIMITS~, or has a row the model cannot follow is status 1, and the message names the line. ~stack bot-classify~ is status 0 for every class, and status 2 when a path is a directory. ~stack batch~ is status 1 when the plan includes, or depends on, an entry the architecture cannot build. ~stack provided~ is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets ~--hooks~, or belongs in a ~--family-lock~. A group-restricted warning is status 0 unless ~--strict~. Status 2 is when neither ~--installed~ nor ~--baseline-lock~ is set. ~stack shard~ is status 1 when a build-list line cannot be resolved, and writes nothing; ~--shards 0~ and a list with no lines are status 2. ~stack shadow~ is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. ~--help~ for every command has an example and a description of every flag. A default that the command applies is printed in that help. A directory that receives several artifacts is ~--out-dir~. A single output file is ~--out~ (~--lock-out~ for the lock ~stack solve~ writes). ~--installed~, ~--easyconfigs~, ~--filter-deps~, and ~--generated-at~ use those names on every command that takes them. @@ -1102,9 +1102,9 @@ EESSI bot node, at the parallel level the bot job gives it, and warn where the estimate reaches the job's memory. Each warning names the remedies the maintainers have used. Nothing is changed. -Why you'd use it: software-layer pull request 1772 lost two zen4 bot -runs to a killed ~cc1~ while building libxc. The check names that build -before the first run. +Why you'd use it: software-layer pull request 1772 lost three zen4 bot +runs to a killed ~cc1~ while building libxc, two at 8 processes and one +at 5. The check names that build before the first run. #+begin_src bash eb-stack stack oom \ @@ -1113,12 +1113,12 @@ eb-stack stack oom \ #+end_src #+begin_example -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=1 builds=5 nodes=2 #+end_example @@ -1143,17 +1143,50 @@ already installed. ~--arch~ repeats and takes a target or a prefix such as ~aarch64~. No ~--arch~ checks every node in the table; a target with no node is reported and skipped. ~--profile~ replaces ~data/build-memory.toml~. ~--resources~ replaces -~data/eessi-bot-resources.toml~. ~--job-memory~ replaces -~data/job-memory.toml~, the per-package ~mib_per_job~ table. A row with -no number is unmeasured. libxc 7.0 is unmeasured; the peak RSS of its -worst compile has not been measured, and the table does not guess. +~data/eessi-bot-resources.toml~. ~--job-memory~, also spelled +~--mib-per-job~, replaces ~data/job-memory.toml~, the per-package +~mib_per_job~ table. A row with no number is unmeasured. libxc 7.0 is +unmeasured; the peak RSS of its worst compile has not been measured, +and the table does not guess. ~--node~ repeats. Each value is ~CORES:MEMORY_MIB~ or ~CORES:MEMORY_MIB:TARGET~, and memory is MiB. It prints the parallelism the current hook leaves beside ~limit_by_memory(parallel, mib_per_job)~, -which is ~max(1, min(parallel, memory_mib // mib_per_job))~. The memory +which is ~max(1, min(parallel, memory_mib // mib_per_job))~. For a +package with a number, the memory cap +~min(current, memory_mib // mib_per_job)~ follows the hook's level: the +core cap with a memory ceiling on top. It is not raised to 1, so +~memory cap 0~ with ~not one N MiB job fits~ says one compile needs more +than the node's memory. The memory a hook would read on the build machine is the smallest cgroup v2 -~memory.max~ on the process's cgroup path, else ~MemTotal~. Pass -~--easystack~, ~--node~, or both. The command exits 0 when it warns. See +~memory.max~ on the process's cgroup path, else ~MemTotal~. + +~--parallel N~ and ~--max-parallel N~ are EasyBuild's settings for the +job, as ~EASYBUILD_PARALLEL~ and ~EASYBUILD_MAX_PARALLEL~ would set them; +the bot sets neither. EasyBuild uses ~--parallel~ in place of the core +count, capped at ~--max-parallel~ (in place of the table's +~max_parallel~) and at the easyconfig's ~maxparallel~. The hook applies +its op to the node's cores, not to either setting, and keeps the result +only when it is lower. ~--node~ lines then end the hook formula with +~then --parallel N~ or ~then max-parallel N~ when that setting decided, +and the node line names ~--parallel~. A warning's ~parallel~ line names +~--parallel~ and the cores the hook divided, and says when the hook's +result was not below EasyBuild's level. 0 for either flag is status 2. + +~--hooks~ reads ~PARALLELISM_LIMITS~ from an EESSI ~eb_hooks.py~ in +place of the hook rows of ~data/eessi-bot-resources.toml~. The nodes +and EasyBuild's ~max-parallel~ still come from the table. The file is +parsed, not run. A key is a string or a module-level string constant +such as ~CPU_TARGET_A64FX~, and a key the dict repeats keeps its last +value, as in Python. An op other than ~divide_by_factor~ or +~set_maximum~, a ~**~ unpacking, or a key that is not a string is +refused with its line: the parallel levels would be wrong without that +row. The output starts with ~hooks: N PARALLELISM_LIMITS rows from +FILE~, and a ~divide~ that is not an integer is a warning that names the +file and the line. A statement that adds to a name and drops the sum, +such as ~msg+ "..."~ in ~post_ready_hook~, is a warning that names the +file, the line, and the function; ~msg += "..."~ is not. Pass +~--easystack~, ~--node~, ~--hooks~, or a mix. +The command exits 0 when it warns. See [[file:../howto/oom-profile.org][warn before a bot job runs out of memory]]. ** ~stack bot-classify~ @@ -1798,10 +1831,12 @@ needs one ~easystack~ or ~pr~. It returns the same report as stdout, and ~hits~, one object per line. Warnings are not a tool error unless ~strict~ is true, when a remaining warning is. ~eb_stack_oom~ takes ~easystack~, ~arch~, ~easyconfigs~, -~installed~, ~profile~, ~resources~, ~node~, and ~job_memory~. It needs -an easystack or a node. It returns the ~stack oom~ text, one object per -warning, and ~factor_warnings~ for a divide factor that is not an -integer. A warning is not a tool error. +~installed~, ~profile~, ~resources~, ~node~, ~job_memory~, ~hooks~, +~parallel~, and ~max_parallel~. +It needs an easystack, a node, or hooks. It returns the ~stack oom~ +text, one object per warning, ~factor_warnings~ for a divide factor +that is not an integer, and ~hook_warnings~ for a dropped sum in the +~hooks~ file. A warning is not a tool error. ~eb_stack_bot_classify~ requires ~files~. It returns ~text~, the ~stack bot-classify~ stdout, ~jobs~, one object per job, and ~counts~. ~eb_stack_batch~ takes ~arch~, ~unsupported~, ~known_failures~, diff --git a/llms-full.txt b/llms-full.txt index e55505f4..72c6afea 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4603,8 +4603,8 @@ The command never edits a recipe or a hook, and it always exits 0. Why you'd use it: a bot job that runs out of memory fails late and quietly. Software-layer pull request 1772 added `libxc-7.0.0-GCC-15.2.0.eb`. The build was killed for lack of memory on -zen4 twice, at 8 processes on a 32 GiB job, before anyone changed the -parallel level. The same thing happened to Highway in #1692, to +zen4 twice at 8 processes on a 32 GiB job, and again at 5 after a hook +change. The same thing happened to Highway in #1692, to hipBLASLt and rocBLAS in #1549, and to PETSc in #1608. Run the check before you open the pull request or ask the bot to build. @@ -4616,12 +4616,12 @@ eb-stack stack oom \ The command prints: ``` -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=14 builds=5 nodes=15 ``` @@ -4705,9 +4705,11 @@ bigger node. The parallel level comes from the hook limits in `data/eessi-bot-resources.toml`. Pass `--resources` to try other limits -or other nodes. Pull request 1608 ran out of memory in the PETSc test -step on zen5 before software-layer-scripts #298 halved PETSc. With the -current hooks, PETSc fits: +or other nodes, or `--hooks` to read the limits from an `eb_hooks.py` +(see [[*Read the limits from eb_hooks.py][below]]). Pull request 1608 +ran out of memory in the PETSc test step on zen5 before +software-layer-scripts #298 halved PETSc. With the current hooks, PETSc +fits: ```bash eb-stack stack oom \ @@ -4759,9 +4761,9 @@ or bot configuration. also holds EasyBuild's default maximum parallel level of 16 and the `PARALLELISM_LIMITS` table from the EESSI `eb_hooks.py`. -The parallel level is the smallest of these: the node's cores, -EasyBuild's maximum, the hook limit applied to the cores, and a -top-level `maxparallel` in the recipe. The hook ops are the ones in +The parallel level is the smallest of these: the node's cores, or +`--parallel` in their place; EasyBuild's maximum; the hook limit applied +to the cores; and a top-level `maxparallel` in the recipe. The hook ops are the ones in `eb_hooks.py`. `divide_by_factor(parallel, factor)` is `max(1, parallel // factor)`. `set_maximum(parallel, n)` is `min(parallel, n)`. A software name uses its limit for the CPU target, @@ -4789,7 +4791,8 @@ msg+ "to avoid out-of-memory failures during building/testing" The second statement adds nothing. `msg + "..."` is an expression, and the result is thrown away. The operator that appends is `+=`. The line that prints stops after the CPU target. This is upstream `eb_hooks.py`. -eb-stack does not patch it. +eb-stack does not patch it. `--hooks` reports it with its line (see +[[*Read the limits from eb_hooks.py][below]]). # A divide factor that is not an integer @@ -4808,13 +4811,151 @@ eb-stack stack oom \ ``` warning: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. -oom=0 fits=1 builds=5 nodes=1 +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: no other x86_64/amd/zen4 node in the bot resource table has room for it + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=0 builds=5 nodes=1 ``` -The warning is the float. The estimate then uses 6, so 6 times 4.0 GiB -is 24 GiB and the 32 GiB zen4 job fits. With the integer divider of 2 -the same build is the warning in the first example. The command still -exits 0. +The first warning is the float. The estimate then uses 6, and 6 times +6.4 GiB is 38.4 GiB, more than the 32 GiB zen4 job. The divider of 3 +ran out at 5 in bot job 221273 of #1772, so 6 does not fit either. The +fixture has only the AWS zen4 node, so no other node has room. The +command still exits 0. + +# Read the limits from eb_hooks.py + +The hook rows in `data/eessi-bot-resources.toml` are a copy of +`PARALLELISM_LIMITS`. `--hooks` reads that dict from an `eb_hooks.py` +instead, so a hook change can be checked before it is merged. The file +is parsed as Python, and nothing in it runs. The fixture +`hooks/divide-2.5.py` has the layout of the real file and the libxc +divider of 2.5: + +```bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-2.5.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-2.5.py +warning: tests/oom_profile_fixtures/hooks/divide-2.5.py:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor. +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float) + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 +``` + +The first line says how many rows were read, and from which file. The +first warning names the line of the `(divide_by_factor, 2.5)` tuple. +The nodes still come from the bot resource table, so both zen4 nodes are +checked, and the 180 GiB node has room for 6. With every factor an +integer there is no `warning:` line: + +```bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/integers.py +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +``` + +The libxc and A64FX rows of `integers.py` are the ones in the bot +resource table, so `--node` (see +[[*A memory limit instead of a core fraction][below]]) prints the same +parallel levels with and without it. For the real hooks, pass the file from a software-layer-scripts +checkout, `--hooks software-layer-scripts/eb_hooks.py`. + +A key is a string or a module-level string constant such as +`CPU_TARGET_A64FX`. A key the dict repeats keeps its last value, as in +Python. An op other than `divide_by_factor` or `set_maximum`, a `**` +unpacking, or a key that is not a string stops the command with the +file and the line: the parallel levels would be wrong without that row. + +The same file is checked for a statement that adds to a name and drops +the sum, such as the `msg+` in `post_ready_hook`. The fixture +`hooks/msg-plus.py` has that statement: + +```bash +eb-stack stack oom --hooks tests/oom_profile_fixtures/hooks/msg-plus.py +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/msg-plus.py +warning: tests/oom_profile_fixtures/hooks/msg-plus.py:30: msg+ "to avoid out-of-memory failures during building/testing" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends +``` + +The warning names the function and the line. It does not change a +parallel level, and the command still exits 0. `msg += "..."`, as in +`hooks/msg-plus-equals.py`, appends and is not reported. + +# EasyBuild's --parallel and the hook + +`post_ready_hook` does not divide the level EasyBuild chose. It calls +`det_parallelism()` with no arguments, which counts the cores the job +can use, applies the op to that count, and keeps the result only when +it is below EasyBuild's level. EasyBuild's level comes from +`set_parallel`: `--parallel` (`EASYBUILD_PARALLEL`) in place of the +core count when it is set, capped at the easyconfig's `maxparallel` and +at `--max-parallel` (`EASYBUILD_MAX_PARALLEL`, 16 by default). The bot +sets neither. `stack oom --parallel N` and `--max-parallel N` model a +job that does. `--node CORES:MEMORY_MIB:TARGET` prints the levels for +one node size: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 12 \ + --node 16:32768:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 12 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +``` + +libxc gets 8, which is 16 // 2. Half of `--parallel 12` would be 6. +With `--parallel 4` the hook's 8 is not lower, so it does not apply: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --parallel 4 \ + --node 16:32768:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 4 + libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +``` + +A divide factor is a fraction of the node, not of `--parallel`: +`--parallel` above the hook's result does not change it, and +`--parallel` below it wins. `--parallel` also replaces the core count in +EasyBuild, so with `--parallel 32 --max-parallel 64` a build with no +hook row runs at 32 on a 16-core node. With an easystack, the +`parallel` line of a warning names `--parallel` and the cores the hook +divided, and says when the hook's result was not lower. + +At software-layer-scripts 43df6f4 the hook reads `EESSI_SOFTWARE_SUBDIR`, +the software name, EasyBuild's level, and `det_parallelism()`. No +environment variable there caps or overrides `PARALLELISM_LIMITS`. +EESSI-extend's `EESSI_EASYBUILD_HOOKS_OVERRIDE` replaces the whole hooks +file; `--hooks` reads such a file. # A memory limit instead of a core fraction @@ -4890,6 +5031,127 @@ that name. The footer names the rows that are unmeasured. libxc 7.0 is the one the table calls out; the other names are absent from the table, which is the same as unmeasured. +With a number in the table, the line also prints the memory cap: the +level the hook leaves, with a memory ceiling on top, +`min(current, memory_mib // mib_per_job)`. The memory is a whole number +of MiB, so the quotient is the same as +`memory_bytes // (mib_per_job * 2**20)`. `--mib-per-job FILE` is the +same flag as `--job-memory FILE`. The fixture +`job-memory-fixture.toml` gives libxc 8192 MiB, which is arithmetic and +not a measurement. With `hooks/integers.py` the report is libxc alone: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/integers.py \ + --mib-per-job tests/oom_profile_fixtures/job-memory-fixture.toml \ + --node 16:32768:x86_64/amd/zen4 \ + --node 16:131072:x86_64/amd/zen4 \ + --node 16:4096:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/integers.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192))) +node 16 cores, 131072 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 8 (min(8, 131072 // 8192)); memory 16 (max(1, min(16, 131072 // 8192))) +node 16 cores, 4096 MiB, x86_64/amd/zen4 + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192))) +job memory: every package in the table has an mib_per_job +``` + +On 32 GiB the memory allows four 8192 MiB jobs and the hook allows 8, +so the memory cap is 4. On 128 GiB the memory allows 16, and the cap +stays at the hook's 8; `limit_by_memory` in place of the hook would +give 16. On 4 GiB not one job fits. `limit_by_memory` still says 1, +because it never goes below one job. The memory cap is 0: one compile +already needs more than the node has. + +# A cap of 8 or a reduction of 8 for libxc + +Two forms of libxc's `'*'` row in `PARALLELISM_LIMITS` have come up in +the review of software-layer-scripts pull request 337. Each fixture is +`hooks/integers.py` with that row in place of `(divide_by_factor, 2)`, +and the A64FX row stays at `(set_maximum, 6)`: + +- `hooks/set-maximum-8.py` has a cap of 8, `(set_maximum, 8)`, the form + proposed in review. +- `hooks/divide-8.py` has a reduction of 8, `(divide_by_factor, 8)`, as + asked in the review of 337 (issuecomment-6599147327). + +Compare node sizes under the cap of 8: + +```bash +eb-stack stack oom \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --node 16:32768:x86_64/amd/zen4 \ + --node 48:30016:aarch64/a64fx \ + --node 64:262144:x86_64/amd/zen4 \ + --node 12:24576:x86_64/amd/zen4 +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +parallelism under the current eb_hooks.py limit and under limit_by_memory +node 16 cores, 32768 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(16, 8)); memory unmeasured +node 48 cores, 30016 MiB, aarch64/a64fx + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured +node 64 cores, 262144 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(64, 8)); memory unmeasured +node 12 cores, 24576 MiB, x86_64/amd/zen4 + libxc: current 8 (set_maximum on *, min(12, 8)); memory unmeasured +job memory: libxc 7.0 unmeasured, no mib_per_job +``` + +With `hooks/integers.py` the same four nodes give libxc 8, 6, 16 and 6. +A cap is a number of jobs, so it does not grow with the node: on 64 +cores it gives 8, where `64 // 2` is 32 and max-parallel holds that at +16. It does not shrink either: on 12 cores it gives 8, where `12 // 2` +is 6. The two rows agree on 16 and 17 cores. Every node in the bot +resource table has 16 cores except A64FX, which keeps its own row, so +on the bot the cap leaves libxc where it is. Pull request 1772 still +warns on the 32 GiB zen4 node: + +```bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/set-maximum-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/set-maximum-8.py +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) + parallel 8: 16 cores, eb_hooks.py caps libxc at 8 + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many + remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster +oom=1 fits=1 builds=5 nodes=2 +``` + +The estimate is the one in the first example. Only the `parallel` line +changes, and it names the cap in place of the divide. The reduction of +8 in `hooks/divide-8.py` leaves 2 cores on these nodes: + +```bash +eb-stack stack oom \ + --arch x86_64/amd/zen4 \ + --hooks tests/oom_profile_fixtures/hooks/divide-8.py \ + --easystack tests/oom_profile_fixtures/easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml +``` + +``` +hooks: 3 PARALLELISM_LIMITS rows from tests/oom_profile_fixtures/hooks/divide-8.py +oom=0 fits=2 builds=5 nodes=2 +``` + +`max(1, 16 // 8)` is 2, so libxc fits on both zen4 nodes. On 64 cores +the same row gives 8. + # Measure the worst compile `mib_per_job` comes from a compiler launcher, not from a guess. @@ -4957,9 +5219,15 @@ then. steps, test steps that run with a different parallel level, or memory that other jobs on the node use. It does not model `ulimit -u`, which `det_parallelism` also consults. -- `limit_by_memory` does not replace the estimate. The estimate follows - the hooks that are in `eb_hooks.py`. The `--node` report is the - comparison. A package with no measured `mib_per_job` stays +- `--hooks` reads the `PARALLELISM_LIMITS` literal. It does not run + `post_ready_hook`; the rules that pick a row are the ones described + under [[*Where the numbers come from][where the numbers come from]]. + Of the rest of the file it checks only for a statement that drops a + sum. Other mistakes in the hooks are not found. +- `limit_by_memory` and the memory cap do not replace the estimate. The + estimate follows the hooks that are in `eb_hooks.py`. The `--node` + report is the comparison, and the warnings for an easystack do not + print the cap. A package with no measured `mib_per_job` stays unmeasured. - The MCP tool `eb_stack_oom` returns the same lines and each warning as JSON. @@ -5851,7 +6119,7 @@ reachability, toolchain family, and the claim ladder are defined in | 1 | The work was attempted and failed: a missing or unreadable file, a document that does not parse, a check that found errors, a network fetch that failed, a doctor probe, or a campaign whose build failed | | 2 | The command line was rejected before that work: a missing or unknown flag (clap), or a flag value or combination the command does not accept | -A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails `recipe lint` or `recipe check` is status 1: the files were read and the check failed. `stack from-commit` is status 1 when any pin is `released`, and status 0 when every pin is `newer` or `unknown` or the files name no pin. A file name with no `eb-X.Y.Z`, and no `--easybuild-version`, is status 2. `stack arch` is status 1 when an entry is unsupported or flagged, or when `--probe-sources` reports `unreachable` or `arch-missing` sources. `stack files` is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are `no_url` or `unchecked`. `stack generation` is status 1 on any finding, and status 2 when an easystack path names no EESSI version and `--eessi-version` is absent. `stack known-failures` is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with `--strict` it is status 1 when a warning remains, and status 0 when every hit is remedied. `stack oom` is status 0 when it warns: the warnings are advice, not a failed check. A `--node` value that is not `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET` is status 2, and so is a command with neither `--easystack` nor `--node`. `stack bot-classify` is status 0 for every class, and status 2 when a path is a directory. `stack batch` is status 1 when the plan includes, or depends on, an entry the architecture cannot build. `stack provided` is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets `--hooks`, or belongs in a `--family-lock`. A group-restricted warning is status 0 unless `--strict`. Status 2 is when neither `--installed` nor `--baseline-lock` is set. `stack shard` is status 1 when a build-list line cannot be resolved, and writes nothing; `--shards 0` and a list with no lines are status 2. `stack shadow` is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. +A rejected value names the flag and the value. A missing file names the flag and the path. Status 2 is the same status clap already uses. A recipe that fails `recipe lint` or `recipe check` is status 1: the files were read and the check failed. `stack from-commit` is status 1 when any pin is `released`, and status 0 when every pin is `newer` or `unknown` or the files name no pin. A file name with no `eb-X.Y.Z`, and no `--easybuild-version`, is status 2. `stack arch` is status 1 when an entry is unsupported or flagged, or when `--probe-sources` reports `unreachable` or `arch-missing` sources. `stack files` is status 1 when a patch or an easyconfig is missing, and status 0 when the only lines are `no_url` or `unchecked`. `stack generation` is status 1 on any finding, and status 2 when an easystack path names no EESSI version and `--eessi-version` is absent. `stack known-failures` is status 0 when it prints warnings: they are advice for the pull request, not a refusal; with `--strict` it is status 1 when a warning remains, and status 0 when every hit is remedied. `stack oom` is status 0 when it warns: the warnings are advice, not a failed check. A `--node` value that is not `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET` is status 2, and so is a command with none of `--easystack`, `--node`, and `--hooks`, or a `--parallel` or `--max-parallel` of 0. A `--hooks` file that is not valid Python, has no module-level `PARALLELISM_LIMITS`, or has a row the model cannot follow is status 1, and the message names the line. `stack bot-classify` is status 0 for every class, and status 2 when a path is a directory. `stack batch` is status 1 when the plan includes, or depends on, an entry the architecture cannot build. `stack provided` is status 1 when a build-list line names software the baseline already provides, names no package, uses a toolchain other than 2026.1 or GCCcore-15.2.0 below it, sets `--hooks`, or belongs in a `--family-lock`. A group-restricted warning is status 0 unless `--strict`. Status 2 is when neither `--installed` nor `--baseline-lock` is set. `stack shard` is status 1 when a build-list line cannot be resolved, and writes nothing; `--shards 0` and a list with no lines are status 2. `stack shadow` is status 1 when a site-layer recipe shadows a base-stack module, or when a recipe does not parse, and status 0 when every recipe names software the base does not provide. `--help` for every command has an example and a description of every flag. A default that the command applies is printed in that help. A directory that receives several artifacts is `--out-dir`. A single output file is `--out` (`--lock-out` for the lock `stack solve` writes). `--installed`, `--easyconfigs`, `--filter-deps`, and `--generated-at` use those names on every command that takes them. @@ -6936,9 +7204,9 @@ EESSI bot node, at the parallel level the bot job gives it, and warn where the estimate reaches the job's memory. Each warning names the remedies the maintainers have used. Nothing is changed. -Why you'd use it: software-layer pull request 1772 lost two zen4 bot -runs to a killed `cc1` while building libxc. The check names that build -before the first run. +Why you'd use it: software-layer pull request 1772 lost three zen4 bot +runs to a killed `cc1` while building libxc, two at 8 processes and one +at 5. The check names that build before the first run. ```bash eb-stack stack oom \ @@ -6947,12 +7215,12 @@ eb-stack stack oom \ ``` ``` -libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772) +libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337) parallel 8: 16 cores, eb_hooks.py divides libxc by 2 - why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 eight compilers exceed 32 GiB on zen4 - remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many + why: cc1 on the generated mgga_x_*.c sources; with GCC 15.2.0 on zen4, 8 and then 5 compilers ran out of a 32 GiB job + remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90) - remedy: the maintainers asked for a libxc divider of 2.5 or 3 in eb_hooks.py, which is 6 or 5 cores on the AWS build cluster + remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster oom=1 fits=1 builds=5 nodes=2 ``` @@ -6977,17 +7245,50 @@ already installed. `--arch` repeats and takes a target or a prefix such as `aarch64`. No `--arch` checks every node in the table; a target with no node is reported and skipped. `--profile` replaces `data/build-memory.toml`. `--resources` replaces -`data/eessi-bot-resources.toml`. `--job-memory` replaces -`data/job-memory.toml`, the per-package `mib_per_job` table. A row with -no number is unmeasured. libxc 7.0 is unmeasured; the peak RSS of its -worst compile has not been measured, and the table does not guess. +`data/eessi-bot-resources.toml`. `--job-memory`, also spelled +`--mib-per-job`, replaces `data/job-memory.toml`, the per-package +`mib_per_job` table. A row with no number is unmeasured. libxc 7.0 is +unmeasured; the peak RSS of its worst compile has not been measured, +and the table does not guess. `--node` repeats. Each value is `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET`, and memory is MiB. It prints the parallelism the current hook leaves beside `limit_by_memory(parallel, mib_per_job)`, -which is `max(1, min(parallel, memory_mib // mib_per_job))`. The memory +which is `max(1, min(parallel, memory_mib // mib_per_job))`. For a +package with a number, the memory cap +`min(current, memory_mib // mib_per_job)` follows the hook's level: the +core cap with a memory ceiling on top. It is not raised to 1, so +`memory cap 0` with `not one N MiB job fits` says one compile needs more +than the node's memory. The memory a hook would read on the build machine is the smallest cgroup v2 -`memory.max` on the process's cgroup path, else `MemTotal`. Pass -`--easystack`, `--node`, or both. The command exits 0 when it warns. See +`memory.max` on the process's cgroup path, else `MemTotal`. + +`--parallel N` and `--max-parallel N` are EasyBuild's settings for the +job, as `EASYBUILD_PARALLEL` and `EASYBUILD_MAX_PARALLEL` would set them; +the bot sets neither. EasyBuild uses `--parallel` in place of the core +count, capped at `--max-parallel` (in place of the table's +`max_parallel`) and at the easyconfig's `maxparallel`. The hook applies +its op to the node's cores, not to either setting, and keeps the result +only when it is lower. `--node` lines then end the hook formula with +`then --parallel N` or `then max-parallel N` when that setting decided, +and the node line names `--parallel`. A warning's `parallel` line names +`--parallel` and the cores the hook divided, and says when the hook's +result was not below EasyBuild's level. 0 for either flag is status 2. + +`--hooks` reads `PARALLELISM_LIMITS` from an EESSI `eb_hooks.py` in +place of the hook rows of `data/eessi-bot-resources.toml`. The nodes +and EasyBuild's `max-parallel` still come from the table. The file is +parsed, not run. A key is a string or a module-level string constant +such as `CPU_TARGET_A64FX`, and a key the dict repeats keeps its last +value, as in Python. An op other than `divide_by_factor` or +`set_maximum`, a `**` unpacking, or a key that is not a string is +refused with its line: the parallel levels would be wrong without that +row. The output starts with ~hooks: N PARALLELISM_LIMITS rows from +FILE`, and a `divide~ that is not an integer is a warning that names the +file and the line. A statement that adds to a name and drops the sum, +such as `msg+ "..."` in `post_ready_hook`, is a warning that names the +file, the line, and the function; `msg += "..."` is not. Pass +`--easystack`, `--node`, `--hooks`, or a mix. +The command exits 0 when it warns. See [warn before a bot job runs out of memory](https://eb-stack.rgoswami.me/../howto/oom-profile.html.md). ## `stack bot-classify` @@ -7632,10 +7933,12 @@ needs one `easystack` or `pr`. It returns the same report as stdout, and `hits`, one object per line. Warnings are not a tool error unless `strict` is true, when a remaining warning is. `eb_stack_oom` takes `easystack`, `arch`, `easyconfigs`, -`installed`, `profile`, `resources`, `node`, and `job_memory`. It needs -an easystack or a node. It returns the `stack oom` text, one object per -warning, and `factor_warnings` for a divide factor that is not an -integer. A warning is not a tool error. +`installed`, `profile`, `resources`, `node`, `job_memory`, `hooks`, +`parallel`, and `max_parallel`. +It needs an easystack, a node, or hooks. It returns the `stack oom` +text, one object per warning, `factor_warnings` for a divide factor +that is not an integer, and `hook_warnings` for a dropped sum in the +`hooks` file. A warning is not a tool error. `eb_stack_bot_classify` requires `files`. It returns `text`, the `stack bot-classify` stdout, `jobs`, one object per job, and `counts`. `eb_stack_batch` takes `arch`, `unsupported`, `known_failures`, diff --git a/skills/eessi-extend/SKILL.md b/skills/eessi-extend/SKILL.md index d6f182fe..bb0008c9 100644 --- a/skills/eessi-extend/SKILL.md +++ b/skills/eessi-extend/SKILL.md @@ -332,20 +332,35 @@ eb-stack stack oom \ A warning names the entry, the target and node, the parallel level and the estimate against the job's memory. The libxc build in software-layer -#1772 was killed twice on zen4 at 8 processes on 32 GiB. The remedies are -text only: a `maxparallel` in the recipe, a limit in `PARALLELISM_LIMITS` -in software-layer-scripts `eb_hooks.py`, or a node with more memory. Put -the change in the pull request yourself. The command exits 0. +#1772 was killed on zen4 twice at 8 processes on 32 GiB, and once at 5. +The remedies are text only: a `maxparallel` in the recipe, a limit in +`PARALLELISM_LIMITS` in software-layer-scripts `eb_hooks.py`, or a node +with more memory. Put the change in the pull request yourself. The +command exits 0. The hook ops are `divide_by_factor` (`max(1, parallel // factor)`) and `set_maximum` (`min(parallel, n)`). A CPU-target key overrides `*`. -A64FX divides every other build by 4. A divide factor that is not an +A64FX divides every other build by 4. The op applies to the node's +cores, not to EasyBuild's `--parallel`, and wins only when lower; +`--parallel N` and `--max-parallel N` on `stack oom` show a job that +sets them. A divide factor that is not an integer is a warning: `16 // 2.5` is the float `6.0`. Use an integer. -`eb_hooks.py` also has a `msg+` where the log line should use `+=`; half +`--hooks software-layer-scripts/eb_hooks.py` reads `PARALLELISM_LIMITS` +from the file instead of the bundled copy, and the warning names the +line. `eb_hooks.py` also has a `msg+` where the log line should use `+=`; half the message never prints. That is upstream. Do not patch it from here. +`--hooks` prints it as a warning with the line and `post_ready_hook`. +A cap does not scale with the node: on the bot's 16-core nodes +`(set_maximum, 8)` and `(divide_by_factor, 2)` both leave 8, and 2 cores +of 16 is `(divide_by_factor, 8)`. Before a software-layer pull request +points `bot/software_layer_scripts_commit` at a scripts change, run +`stack oom --hooks` on that commit's `eb_hooks.py` with `--node` for the +bot node, and check that the level is the one asked for. `eb-stack stack oom --node 16:32768:x86_64/amd/zen4` compares the hook -with `limit_by_memory`. `data/job-memory.toml` holds `mib_per_job`. +with `limit_by_memory`. `data/job-memory.toml` holds `mib_per_job`; +`--mib-per-job FILE` replaces it, and a package with a number gets the +memory cap `min(current, memory // mib_per_job)` beside the hook's level. libxc is unmeasured. Measure a compile with `scripts/compile_rss.py` under `-O2 -ftree-vectorize -march= -fno-math-errno`, and do not invent the number. The how-to is `docs/orgmode/howto/oom-profile.org`. diff --git a/src/eb_hooks.rs b/src/eb_hooks.rs new file mode 100644 index 00000000..5b0bf27f --- /dev/null +++ b/src/eb_hooks.rs @@ -0,0 +1,607 @@ +//! `PARALLELISM_LIMITS` from an EESSI `eb_hooks.py`, read as Python source. +//! +//! software-layer-scripts keeps its per-package parallelism limits in a +//! module-level dict that `post_ready_hook` reads. A key whose value is an +//! `(op, argument)` tuple is a CPU target's own limit. A key whose value is +//! a dict is a software name, and that dict's keys are CPU targets or +//! `'*'`. A key is a string or a module-level string constant such as +//! `CPU_TARGET_A64FX`. The ops are `divide_by_factor` and `set_maximum`. +//! The file is parsed, never run. A row this reader cannot model is an +//! error that names its line, because the parallel levels would be wrong +//! without it. +//! +//! The same file is checked for a statement that adds to a name and drops +//! the result: `msg+ "..."` on a line of its own builds a longer string and +//! throws it away, where `msg += "..."` appends. + +use crate::oom_profile::{DivideFactor, HookLimit, LimitOp}; +use anyhow::{anyhow, bail, Result}; +use rustpython_parser::ast::{self, Ranged}; +use rustpython_parser::text_size::TextSize; +use rustpython_parser::Parse; +use std::collections::BTreeMap; + +/// The dict `post_ready_hook` looks a build up in. +pub const LIMITS_NAME: &str = "PARALLELISM_LIMITS"; + +/// `max(1, parallel // factor)` in `eb_hooks.py`. +const DIVIDE_OP: &str = "divide_by_factor"; + +/// `min(parallel, max_value)` in `eb_hooks.py`. +const MAXIMUM_OP: &str = "set_maximum"; + +/// What `stack oom` reads from one `eb_hooks.py`. +#[derive(Debug, Clone, PartialEq)] +pub struct HooksFile { + /// `PARALLELISM_LIMITS` rows. A key the dict repeats keeps its last + /// value, as Python does. + pub limits: Vec, + /// Statements that add to a name and drop the sum. + pub dropped_sums: Vec, +} + +/// `name + ...` as a statement of its own: the sum is computed and lost. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DroppedSum { + /// 1-based line of the statement. + pub line: usize, + /// The innermost `def` around the statement. `None` at module level. + pub function: Option, + /// The name left of `+`, `msg`. + pub name: String, + /// The statement's first source line, trimmed. + pub statement: String, +} + +impl DroppedSum { + /// The warning `stack oom` prints, with `file` as the location. + pub fn message(&self, file: &str) -> String { + let place = match &self.function { + Some(function) => format!("in {function}"), + None => "at module level".to_string(), + }; + let name = &self.name; + format!( + "warning: {file}:{}: {} {place} adds to {name} and drops the result, so {name} does not change; {name} += appends", + self.line, self.statement + ) + } +} + +/// Read `PARALLELISM_LIMITS` and the dropped sums from `text`. +/// +/// `file` names the file in errors and in each row's location. The last +/// module-level assignment to `PARALLELISM_LIMITS` is the one +/// `post_ready_hook` sees, and its keys resolve through the string +/// constants bound before it. +pub fn parse_hooks(text: &str, file: &str) -> Result { + let suite = ast::Suite::parse(text, file).map_err(|error| { + let offset = offset(error.offset).min(text.len()); + anyhow!( + "{file}:{}: not valid Python: {}", + line_at(text, offset), + error.error + ) + })?; + let (value, constants) = + find_limits(&suite).ok_or_else(|| anyhow!("{file}: no module-level {LIMITS_NAME}"))?; + let reader = Reader { + text, + file, + constants: &constants, + }; + let ast::Expr::Dict(dict) = value else { + bail!( + "{file}:{}: {LIMITS_NAME} is not a dict literal", + reader.line(value) + ); + }; + let limits = reader.limits(dict)?; + let mut dropped_sums = Vec::new(); + collect_dropped_sums(&suite, None, text, &mut dropped_sums); + Ok(HooksFile { + limits, + dropped_sums, + }) +} + +/// The last module-level `PARALLELISM_LIMITS = ...`, and the string +/// constants bound before it. +fn find_limits(suite: &[ast::Stmt]) -> Option<(&ast::Expr, BTreeMap)> { + let mut constants = BTreeMap::new(); + let mut found = None; + for statement in suite { + let (targets, value): (Vec<&ast::Expr>, &ast::Expr) = match statement { + ast::Stmt::Assign(assign) => (assign.targets.iter().collect(), &assign.value), + ast::Stmt::AnnAssign(assign) => match &assign.value { + Some(value) => (vec![&assign.target], value), + None => continue, + }, + _ => continue, + }; + for target in targets { + let ast::Expr::Name(name) = target else { + continue; + }; + if name.id.as_str() == LIMITS_NAME { + found = Some((value, constants.clone())); + } else if let Some(text) = string_literal(value) { + constants.insert(name.id.to_string(), text.to_string()); + } else { + constants.remove(name.id.as_str()); + } + } + } + found +} + +fn string_literal(expr: &ast::Expr) -> Option<&str> { + match expr { + ast::Expr::Constant(ast::ExprConstant { + value: ast::Constant::Str(text), + .. + }) => Some(text), + _ => None, + } +} + +enum Number { + Int(u32), + Float(f64), +} + +fn number(expr: &ast::Expr) -> Option { + let ast::Expr::Constant(constant) = expr else { + return None; + }; + match &constant.value { + ast::Constant::Int(value) => value.to_string().parse().ok().map(Number::Int), + ast::Constant::Float(value) => Some(Number::Float(*value)), + _ => None, + } +} + +struct Reader<'a> { + text: &'a str, + file: &'a str, + constants: &'a BTreeMap, +} + +impl Reader<'_> { + fn line(&self, node: &impl Ranged) -> usize { + line_at(self.text, offset(node.start())) + } + + fn source(&self, node: &impl Ranged) -> &str { + let range = node.range(); + self.text + .get(offset(range.start())..offset(range.end())) + .unwrap_or("") + } + + fn key(&self, key: &ast::Expr) -> Result { + if let Some(text) = string_literal(key) { + return Ok(text.to_string()); + } + let at = format!("{}:{}", self.file, self.line(key)); + match key { + ast::Expr::Name(name) => { + self.constants + .get(name.id.as_str()) + .cloned() + .ok_or_else(|| { + anyhow!( + "{at}: {LIMITS_NAME} key {} is not a module-level string constant", + name.id + ) + }) + } + other => bail!( + "{at}: {LIMITS_NAME} key {} is not a string", + self.source(other) + ), + } + } + + fn limits(&self, dict: &ast::ExprDict) -> Result> { + let mut rows: Vec = Vec::new(); + for (key, value) in dict.keys.iter().zip(&dict.values) { + let Some(key) = key else { + bail!( + "{}:{}: {LIMITS_NAME} unpacks a dict with **; only written-out rows are read", + self.file, + self.line(value) + ); + }; + let name = self.key(key)?; + rows.retain(|row| { + row.software.as_deref() != Some(name.as_str()) + && !(row.software.is_none() && row.target == name) + }); + match value { + ast::Expr::Tuple(tuple) => { + if name == "*" { + bail!( + "{}:{}: a {LIMITS_NAME} limit for every build needs a CPU target, not '*'", + self.file, + self.line(key) + ); + } + let op = self.op(tuple, "every build", &name)?; + rows.push(HookLimit { + software: None, + target: name, + op, + line: Some(self.line(tuple)), + }); + } + ast::Expr::Dict(inner) => { + if inner.keys.is_empty() { + bail!( + "{}:{}: {LIMITS_NAME} entry for {name} is empty, and post_ready_hook then skips even the CPU target's own limit", + self.file, + self.line(value) + ); + } + for (target, row) in inner.keys.iter().zip(&inner.values) { + let Some(target) = target else { + bail!( + "{}:{}: {LIMITS_NAME} entry for {name} unpacks a dict with **; only written-out rows are read", + self.file, + self.line(row) + ); + }; + let target = self.key(target)?; + let ast::Expr::Tuple(tuple) = row else { + bail!( + "{}:{}: {LIMITS_NAME} row for {name} on {target} is {}, not an (op, argument) tuple", + self.file, + self.line(row), + self.source(row) + ); + }; + let op = self.op(tuple, &name, &target)?; + rows.retain(|limit| { + !(limit.software.as_deref() == Some(name.as_str()) + && limit.target == target) + }); + rows.push(HookLimit { + software: Some(name.clone()), + target, + op, + line: Some(self.line(tuple)), + }); + } + } + other => bail!( + "{}:{}: {LIMITS_NAME} value for {name} is {}, neither an (op, argument) tuple nor a dict of them", + self.file, + self.line(other), + self.source(other) + ), + } + } + Ok(rows) + } + + fn op(&self, tuple: &ast::ExprTuple, who: &str, target: &str) -> Result { + let at = format!( + "{}:{}: {LIMITS_NAME} row for {who} on {target}", + self.file, + self.line(tuple) + ); + let [op, argument] = tuple.elts.as_slice() else { + bail!("{at} is {}, not (op, argument)", self.source(tuple)); + }; + let ast::Expr::Name(op) = op else { + bail!( + "{at} names its op as {}; the ops are {DIVIDE_OP} and {MAXIMUM_OP}", + self.source(op) + ); + }; + match (op.id.as_str(), number(argument)) { + (DIVIDE_OP, Some(Number::Int(factor))) if factor > 0 => { + Ok(LimitOp::Divide(DivideFactor::from_integer(factor))) + } + (DIVIDE_OP, Some(Number::Float(factor))) if factor.is_finite() && factor > 0.0 => { + Ok(LimitOp::Divide(DivideFactor::from_float(factor))) + } + (MAXIMUM_OP, Some(Number::Int(maximum))) if maximum > 0 => { + Ok(LimitOp::Maximum(maximum)) + } + (DIVIDE_OP, _) => bail!( + "{at} divides by {}; {DIVIDE_OP} needs a positive number", + self.source(argument) + ), + (MAXIMUM_OP, _) => bail!( + "{at} caps at {}; {MAXIMUM_OP} needs a positive integer", + self.source(argument) + ), + (other, _) => bail!("{at} uses {other}; the ops are {DIVIDE_OP} and {MAXIMUM_OP}"), + } + } +} + +/// Every `name + ...` expression statement, in source order. +fn collect_dropped_sums( + statements: &[ast::Stmt], + function: Option<&str>, + text: &str, + out: &mut Vec, +) { + for statement in statements { + match statement { + ast::Stmt::Expr(expr) => { + let ast::Expr::BinOp(sum) = expr.value.as_ref() else { + continue; + }; + let ast::Expr::Name(name) = sum.left.as_ref() else { + continue; + }; + if !matches!(sum.op, ast::Operator::Add) { + continue; + } + let range = expr.range(); + let source = text + .get(offset(range.start())..offset(range.end())) + .unwrap_or(""); + out.push(DroppedSum { + line: line_at(text, offset(range.start())), + function: function.map(str::to_string), + name: name.id.to_string(), + statement: source.lines().next().unwrap_or("").trim().to_string(), + }); + } + ast::Stmt::FunctionDef(def) => { + collect_dropped_sums(&def.body, Some(def.name.as_str()), text, out); + } + ast::Stmt::AsyncFunctionDef(def) => { + collect_dropped_sums(&def.body, Some(def.name.as_str()), text, out); + } + ast::Stmt::ClassDef(class) => collect_dropped_sums(&class.body, function, text, out), + ast::Stmt::For(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::AsyncFor(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::While(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::If(block) => { + collect_dropped_sums(&block.body, function, text, out); + collect_dropped_sums(&block.orelse, function, text, out); + } + ast::Stmt::With(block) => collect_dropped_sums(&block.body, function, text, out), + ast::Stmt::AsyncWith(block) => collect_dropped_sums(&block.body, function, text, out), + ast::Stmt::Match(block) => { + for case in &block.cases { + collect_dropped_sums(&case.body, function, text, out); + } + } + ast::Stmt::Try(block) => { + collect_dropped_sums(&block.body, function, text, out); + for handler in &block.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + collect_dropped_sums(&handler.body, function, text, out); + } + collect_dropped_sums(&block.orelse, function, text, out); + collect_dropped_sums(&block.finalbody, function, text, out); + } + ast::Stmt::TryStar(block) => { + collect_dropped_sums(&block.body, function, text, out); + for handler in &block.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + collect_dropped_sums(&handler.body, function, text, out); + } + collect_dropped_sums(&block.orelse, function, text, out); + collect_dropped_sums(&block.finalbody, function, text, out); + } + _ => {} + } + } +} + +fn offset(size: TextSize) -> usize { + u32::from(size) as usize +} + +fn line_at(text: &str, offset: usize) -> usize { + text.as_bytes()[..offset.min(text.len())] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits(text: &str) -> Vec { + parse_hooks(text, "eb_hooks.py") + .unwrap_or_else(|error| panic!("{error:#}")) + .limits + } + + fn refusal(text: &str) -> String { + format!("{:#}", parse_hooks(text, "eb_hooks.py").unwrap_err()) + } + + #[test] + fn rows_resolve_constants_and_keep_the_literal_kind() { + let text = "\ +CPU_TARGET_A64FX = 'aarch64/a64fx' + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2.5), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} +"; + let rows = limits(text); + assert_eq!(rows.len(), 3, "{rows:?}"); + assert_eq!(rows[0].software, None); + assert_eq!(rows[0].target, "aarch64/a64fx"); + assert_eq!(rows[0].line, Some(10)); + let LimitOp::Divide(global) = rows[0].op else { + panic!("{:?}", rows[0].op); + }; + assert!(global.integer_literal()); + assert_eq!(global.value(), 4.0); + assert_eq!(rows[1].software.as_deref(), Some("libxc")); + assert_eq!(rows[1].target, "*"); + assert_eq!(rows[1].line, Some(12)); + let LimitOp::Divide(factor) = rows[1].op else { + panic!("{:?}", rows[1].op); + }; + assert!(!factor.integer_literal()); + assert_eq!(factor.value(), 2.5); + assert_eq!(rows[2].target, "aarch64/a64fx"); + assert_eq!(rows[2].op, LimitOp::Maximum(6)); + } + + #[test] + fn a_repeated_key_keeps_the_last_value_as_python_does() { + let text = "\ +PARALLELISM_LIMITS = { + 'libxc': {'*': (divide_by_factor, 2)}, + 'LLVM': {'*': (divide_by_factor, 2), '*': (set_maximum, 8)}, + 'libxc': {'x86_64/amd/zen4': (set_maximum, 4)}, +} +"; + let rows = limits(text); + assert_eq!(rows.len(), 2, "{rows:?}"); + assert!(rows + .iter() + .any(|row| row.software.as_deref() == Some("LLVM") && row.op == LimitOp::Maximum(8))); + let libxc: Vec<&HookLimit> = rows + .iter() + .filter(|row| row.software.as_deref() == Some("libxc")) + .collect(); + assert_eq!(libxc.len(), 1, "{rows:?}"); + assert_eq!(libxc[0].target, "x86_64/amd/zen4"); + } + + #[test] + fn the_last_assignment_wins_and_uses_the_constants_bound_before_it() { + let text = "\ +TARGET = 'aarch64/a64fx' +PARALLELISM_LIMITS = {TARGET: (divide_by_factor, 4)} +TARGET = 'x86_64/amd/zen4' +PARALLELISM_LIMITS = {TARGET: (set_maximum, 8)} +"; + let rows = limits(text); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert_eq!(rows[0].target, "x86_64/amd/zen4"); + assert_eq!(rows[0].op, LimitOp::Maximum(8)); + assert_eq!(rows[0].line, Some(4)); + } + + #[test] + fn rows_the_model_cannot_follow_are_refused_with_their_line() { + let cases = [ + ("x = (\n", "eb_hooks.py:2: not valid Python"), + ("LIMITS = {}\n", "eb_hooks.py: no module-level PARALLELISM_LIMITS"), + ( + "PARALLELISM_LIMITS = dict()\n", + "eb_hooks.py:1: PARALLELISM_LIMITS is not a dict literal", + ), + ( + "PARALLELISM_LIMITS = {\n 'libxc': {'*': (limit_by_memory, 6400)},\n}\n", + "eb_hooks.py:2: PARALLELISM_LIMITS row for libxc on * uses limit_by_memory; the ops are divide_by_factor and set_maximum", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (set_maximum, 8.0)}}\n", + "row for libxc on * caps at 8.0; set_maximum needs a positive integer", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (divide_by_factor, 0)}}\n", + "row for libxc on * divides by 0; divide_by_factor needs a positive number", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (divide_by_factor, -2)}}\n", + "row for libxc on * divides by -2; divide_by_factor needs a positive number", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {UNKNOWN: (set_maximum, 8)}}\n", + "PARALLELISM_LIMITS key UNKNOWN is not a module-level string constant", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {'*': (set_maximum, 8, 1)}}\n", + "row for libxc on * is (set_maximum, 8, 1), not (op, argument)", + ), + ( + "PARALLELISM_LIMITS = {'libxc': {}}\n", + "PARALLELISM_LIMITS entry for libxc is empty", + ), + ( + "PARALLELISM_LIMITS = {'*': (divide_by_factor, 2)}\n", + "needs a CPU target, not '*'", + ), + ( + "PARALLELISM_LIMITS = {**OTHER}\n", + "PARALLELISM_LIMITS unpacks a dict with **", + ), + ]; + for (text, expected) in cases { + let error = refusal(text); + assert!(error.contains(expected), "{text:?}: {error}"); + } + } + + #[test] + fn a_sum_statement_is_reported_and_an_augmented_assignment_is_not() { + let text = "\ +def post_ready_hook(self): + msg = 'limiting parallelism to %s ' + msg+ 'to avoid out-of-memory failures' + note = 'a' + note += 'b' + if self: + try: + pass + except OSError: + note + 'c' + return msg + +total = 1 +total + 2 +'''A docstring is an expression statement too, and is not a sum.''' +PARALLELISM_LIMITS = {} +"; + let hooks = parse_hooks(text, "hooks.py").unwrap(); + assert!(hooks.limits.is_empty()); + let found: Vec<(usize, Option<&str>, &str)> = hooks + .dropped_sums + .iter() + .map(|sum| (sum.line, sum.function.as_deref(), sum.name.as_str())) + .collect(); + assert_eq!( + found, + [ + (3, Some("post_ready_hook"), "msg"), + (10, Some("post_ready_hook"), "note"), + (14, None, "total"), + ] + ); + assert_eq!( + hooks.dropped_sums[0].message("hooks.py"), + "warning: hooks.py:3: msg+ 'to avoid out-of-memory failures' in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends" + ); + assert!(hooks.dropped_sums[2] + .message("hooks.py") + .contains("total + 2 at module level adds to total")); + } +} diff --git a/src/lib.rs b/src/lib.rs index f41a5da0..76b2e237 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod easystack; pub mod eb_easyblock; mod eb_emit; pub mod eb_engine_export; +pub mod eb_hooks; pub mod eb_maintainer; pub mod eb_module; pub mod eb_parse; diff --git a/src/main.rs b/src/main.rs index 4e824c69..98b67c84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1541,12 +1541,16 @@ enum StackCommand { /// the easyconfig. An estimate that reaches the job's memory in /// `data/eessi-bot-resources.toml` is a warning with remedies. A /// `divide_by_factor` that is not an integer is a warning too: Python's - /// `//` then returns a float (`16 // 2.5` is `6.0`). `--node` prints - /// the hook parallelism beside `limit_by_memory` for a core count and - /// a memory. The command does not fail on a warning. Pass `--easystack` - /// or `--node` (or both). + /// `//` then returns a float (`16 // 2.5` is `6.0`). `--hooks` reads + /// the limits from an `eb_hooks.py` and also warns on a statement such + /// as `msg+ "..."` that drops its sum. `--node` prints the hook + /// parallelism beside `limit_by_memory` for a core count and a memory, + /// and with an `mib_per_job` the memory cap + /// `min(current, memory // mib_per_job)`. + /// The command does not fail on a warning. Pass `--easystack`, + /// `--node`, `--hooks`, or a mix. #[command( - after_long_help = "Examples:\n eb-stack stack oom --easystack eessi-2026.06-eb-5.4.0-2026.1.yml\n eb-stack stack oom --arch x86_64/amd/zen4 --easystack eessi-2026.06-eb-5.4.0-2026.1.yml --easyconfigs easybuild-easyconfigs/easybuild/easyconfigs --installed installed-zen4.txt\n eb-stack stack oom --node 16:32768:x86_64/amd/zen4 --node 48:30016:aarch64/a64fx" + after_long_help = "Examples:\n eb-stack stack oom --easystack eessi-2026.06-eb-5.4.0-2026.1.yml\n eb-stack stack oom --arch x86_64/amd/zen4 --easystack eessi-2026.06-eb-5.4.0-2026.1.yml --easyconfigs easybuild-easyconfigs/easybuild/easyconfigs --installed installed-zen4.txt\n eb-stack stack oom --node 16:32768:x86_64/amd/zen4 --node 48:30016:aarch64/a64fx\n eb-stack stack oom --hooks software-layer-scripts/eb_hooks.py --node 16:32768:x86_64/amd/zen4" )] Oom { /// CPU target, `x86_64/amd/zen4`, or a prefix such as `aarch64`. @@ -1554,7 +1558,7 @@ enum StackCommand { #[arg(long = "arch")] arch: Vec, /// Easystack whose entries are the plan. Repeatable. Required - /// unless `--node` is passed. + /// unless `--node` or `--hooks` is passed. #[arg(long)] easystack: Vec, /// Robot tree the dependency closure and `maxparallel` are read @@ -1573,14 +1577,38 @@ enum StackCommand { #[arg(long)] resources: Option, /// Node size `CORES:MEMORY_MIB` or `CORES:MEMORY_MIB:TARGET`. - /// Repeatable. Memory is MiB. Prints the hook parallelism and + /// Repeatable. Memory is MiB. Prints the hook parallelism, its + /// memory cap when the package has an `mib_per_job`, and /// `limit_by_memory` for each size. With no `--easystack`, this /// report is the whole output. #[arg(long)] node: Vec, - /// Replace the bundled per-package `mib_per_job` table. - #[arg(long)] + /// Replace the bundled per-package `mib_per_job` table, + /// `data/job-memory.toml`. For a package with a number, `--node` + /// prints the memory cap `min(current, memory // mib_per_job)` + /// beside the hook's level. + #[arg(long, visible_alias = "mib-per-job")] job_memory: Option, + /// EESSI `eb_hooks.py` to read `PARALLELISM_LIMITS` from, in place + /// of the hook rows of the bot resource table. The nodes and + /// EasyBuild's max-parallel still come from the table. The file is + /// parsed, not run. An op other than `divide_by_factor` or + /// `set_maximum` is refused with its line. A statement that adds to + /// a name and drops the sum, such as `msg+ "..."`, is a warning. + #[arg(long)] + hooks: Option, + /// EasyBuild's `--parallel` for the job, as `EASYBUILD_PARALLEL` + /// would set it. EasyBuild then uses it in place of the cores, + /// capped at max-parallel. The hook still applies its op to the + /// node's cores and keeps the result only when it is lower. The bot + /// sets none. + #[arg(long)] + parallel: Option, + /// EasyBuild's `--max-parallel` for the job, in place of + /// `max_parallel` in the bot resource table, EasyBuild's default. + /// The hook does not read it. + #[arg(long)] + max_parallel: Option, }, /// Say what an EESSI bot job's FAILURE was: a real failure, ERROR-only /// noise, an empty tarball, or infra. @@ -3265,6 +3293,9 @@ fn run_stack(command: StackCommand) -> Result<()> { resources, node, job_memory, + hooks, + parallel, + max_parallel, } => { let report = eb_stack::oom_profile::run_oom_check(&eb_stack::oom_profile::OomCheckRequest { @@ -3276,6 +3307,9 @@ fn run_stack(command: StackCommand) -> Result<()> { resources, nodes: node, job_memory, + hooks, + parallel, + max_parallel, })?; print!("{}", report.stdout); Ok(()) diff --git a/src/mcp.rs b/src/mcp.rs index 32e9bf24..8a7397d6 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -540,7 +540,7 @@ fn tool_catalog() -> Vec { ), tool_with_optional( "eb_stack_oom", - "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory. Pass easystack or node. A warning is not an error.", + "Warn about plan entries likely to run out of memory in an EESSI bot job, as `stack oom` does. Each build's peak comes from the build memory table at the parallel level the job gives it, after the EESSI PARALLELISM_LIMITS and the easyconfig's maxparallel. An estimate that reaches the job's memory is a warning with remedies as text. A divide_by_factor that is not an integer is also a warning: Python's // returns a float (16 // 2.5 is 6.0). hooks is an eb_hooks.py whose PARALLELISM_LIMITS replace the table's hook rows; a statement such as msg+ \"...\" that drops its sum is returned in hook_warnings. node is CORES:MEMORY_MIB or CORES:MEMORY_MIB:TARGET and prints the hook parallelism beside limit_by_memory; for a package with an mib_per_job in job_memory (the CLI's --job-memory or --mib-per-job) it also prints the memory cap min(current, memory // mib_per_job), where 0 means not one job fits. parallel and max_parallel are EasyBuild's --parallel and --max-parallel for the job; the hook applies its op to the node's cores, not to either, and keeps the result only when it is lower. Pass easystack, node, or hooks. A warning is not an error.", &[], &[ ("easystack", "array"), @@ -551,6 +551,9 @@ fn tool_catalog() -> Vec { ("resources", "string"), ("node", "array"), ("job_memory", "string"), + ("hooks", "string"), + ("parallel", "integer"), + ("max_parallel", "integer"), ], ), tool_with_optional( @@ -3416,6 +3419,9 @@ fn stack_oom_tool(arguments: &Value) -> Result { resources: optional_path(arguments, "resources"), nodes: string_array(arguments, "node")?, job_memory: optional_path(arguments, "job_memory"), + hooks: optional_path(arguments, "hooks"), + parallel: optional_u32(arguments, "parallel")?, + max_parallel: optional_u32(arguments, "max_parallel")?, }) .map_err(workflow_error)?; Ok(json!({ @@ -3440,6 +3446,7 @@ fn stack_oom_tool(arguments: &Value) -> Result { "nodes": report.partitions, "unknown_arch": report.unknown_arches, "factor_warnings": report.factor_warnings, + "hook_warnings": report.hook_warnings, })) } @@ -3936,6 +3943,12 @@ fn optional_u64(arguments: &Value, name: &str) -> Result, String> { } } +fn optional_u32(arguments: &Value, name: &str) -> Result, String> { + optional_u64(arguments, name)? + .map(|value| u32::try_from(value).map_err(|_| format!("argument {name} is too large"))) + .transpose() +} + fn u64_array(arguments: &Value, name: &str) -> Result, String> { match arguments.get(name) { None => Ok(Vec::new()), diff --git a/src/oom_profile.rs b/src/oom_profile.rs index 299d68b7..b0c311f8 100644 --- a/src/oom_profile.rs +++ b/src/oom_profile.rs @@ -13,9 +13,11 @@ //! The hook ops are `divide_by_factor` and `set_maximum` from //! `eb_hooks.py`. A divide factor that is not an integer makes Python's //! `//` return a float (`16 // 2.5` is `6.0`); the command warns and uses -//! the numeric floor for the estimate. `data/job-memory.toml` holds -//! `mib_per_job` for `limit_by_memory`. A row with no number is -//! unmeasured. `--node` prints the hook parallelism beside that rule. +//! the numeric floor for the estimate. `--hooks` reads the rows from an +//! `eb_hooks.py` instead of the table, through [`crate::eb_hooks`]. +//! `data/job-memory.toml` holds `mib_per_job` for `limit_by_memory`. A row +//! with no number is unmeasured. `--node` prints the hook parallelism +//! beside that rule. use crate::arch_support::{plan_requirers, split_easyconfig_filename}; use crate::build_order::{closure_against_installed, reverse_uses, Choice, ModuleKey}; @@ -364,14 +366,14 @@ impl std::fmt::Debug for DivideFactor { } impl DivideFactor { - fn from_integer(value: u32) -> Self { + pub(crate) fn from_integer(value: u32) -> Self { Self { bits: f64::from(value).to_bits(), integer_literal: true, } } - fn from_float(value: f64) -> Self { + pub(crate) fn from_float(value: f64) -> Self { Self { bits: value.to_bits(), integer_literal: false, @@ -464,6 +466,22 @@ pub fn limit_by_memory(parallel: u32, mib_per_job: u32, memory_mib: u64) -> u32 parallel.min(by_memory).max(1) } +/// The core cap with a memory ceiling on top: +/// `min(core_cap, memory_mib // mib_per_job)`. +/// +/// `core_cap` is the level the hook leaves. Unlike [`limit_by_memory`] +/// the result is not raised to 1: 0 says one job needs more memory than +/// the node has. Flooring the MiB first does not change the quotient, +/// so this is also `memory_bytes // (mib_per_job * 2**20)`. A zero +/// `mib_per_job` is not a budget; the table parser refuses it, and this +/// returns `core_cap`. +pub fn memory_cap(core_cap: u32, mib_per_job: u32, memory_mib: u64) -> u32 { + let Some(jobs) = memory_mib.checked_div(u64::from(mib_per_job)) else { + return core_cap; + }; + core_cap.min(u32::try_from(jobs).unwrap_or(u32::MAX)) +} + impl LimitOp { fn apply(self, cores: u32) -> u32 { match self { @@ -519,6 +537,9 @@ pub struct HookLimit { pub target: String, /// The limit. pub op: LimitOp, + /// Line of the `(op, argument)` tuple in the `eb_hooks.py` the row was + /// read from. `None` for a row of the bot resource table. + pub line: Option, } /// The bot job table and the EESSI hook limits. @@ -526,6 +547,9 @@ pub struct HookLimit { pub struct BotResources { /// EasyBuild's default `max-parallel`. pub max_parallel: u32, + /// EasyBuild's `--parallel` for the job. `None`, which is what the bot + /// passes, lets EasyBuild count the cores. + pub parallel: Option, /// Node types, in file order. pub partitions: Vec, /// Hook limits, in file order. @@ -603,7 +627,13 @@ pub struct Parallel { pub level: u32, /// CPUs the job holds. pub cores: u32, - /// The hook limit that applied. + /// EasyBuild's `--parallel`, when the job sets one. + pub requested: Option, + /// EasyBuild's `max-parallel`, when it lowered `--parallel` or the + /// cores. + pub max_parallel: Option, + /// The hook limit that matched. It applies only when its result is + /// below the level EasyBuild chose. pub hook: Option, /// `maxparallel` in the easyconfig. pub recipe: Option, @@ -688,10 +718,12 @@ impl BotResources { software: limit.software, target: limit.target, op, + line: None, }); } Ok(Self { max_parallel: raw.easybuild.max_parallel, + parallel: None, partitions, limits, hooks_source: raw.hooks.source, @@ -731,7 +763,8 @@ impl BotResources { /// /// `eb_hooks.py` computes `max(1, parallel // factor)`. A non-integer /// factor makes `//` return a float: `16 // 2.5` is `6.0`. A whole - /// number written as a float does the same: `16 // 2.0` is `8.0`. + /// number written as a float does the same: `16 // 2.0` is `8.0`. A + /// row read from an `eb_hooks.py` names its file and line. pub fn factor_warnings(&self) -> Vec { let mut lines = Vec::new(); for limit in &self.limits { @@ -748,8 +781,12 @@ impl BotResources { } else { format!("{shown}, which is not an integer") }; + let at = limit + .line + .map(|line| format!("{}:{line}: ", self.hooks_source)) + .unwrap_or_default(); lines.push(format!( - "warning: PARALLELISM_LIMITS divide_by_factor for {who} on {} uses {kind}; eb_hooks.py computes max(1, parallel // factor), and {}. Use an integer factor.", + "warning: {at}PARALLELISM_LIMITS divide_by_factor for {who} on {} uses {kind}; eb_hooks.py computes max(1, parallel // factor), and {}. Use an integer factor.", limit.target, float_example(factor) )); @@ -757,9 +794,27 @@ impl BotResources { lines } + /// The level EasyBuild's `set_parallel` gives a build before the hook. + /// + /// `--parallel` replaces the core count, so it may be above the cores. + /// `max-parallel` caps either. The second value is that cap when it + /// lowered the level. + fn easybuild_level(&self, cores: u32) -> (u32, Option) { + let asked = self.parallel.unwrap_or(cores); + if self.max_parallel < asked { + (self.max_parallel, Some(self.max_parallel)) + } else { + (asked, None) + } + } + /// The parallel level `name` builds with on `partition`. + /// + /// `post_ready_hook` applies its op to the cores it detects, not to + /// `--parallel` or `max-parallel`, and keeps the result only when it + /// is below the level EasyBuild chose. pub fn parallel(&self, name: &str, partition: &Partition, recipe: Option) -> Parallel { - let mut level = partition.cores.min(self.max_parallel); + let (mut level, max_parallel) = self.easybuild_level(partition.cores); if let Some(recipe) = recipe { level = level.min(recipe.max(1)); } @@ -770,6 +825,8 @@ impl BotResources { Parallel { level, cores: partition.cores, + requested: self.parallel, + max_parallel, hook, recipe, } @@ -929,28 +986,53 @@ fn estimate_text(entry: &MemoryEntry, parallel: u32, peak: f64) -> String { } fn parallel_reason(name: &str, parallel: &Parallel) -> String { - let mut parts = vec![format!("{} cores", parallel.cores)]; + let cores = parallel.cores; + let mut parts = vec![match parallel.requested { + Some(requested) => format!("--parallel {requested} on {cores} cores"), + None => format!("{cores} cores"), + }]; + if let Some(max_parallel) = parallel.max_parallel { + parts.push(format!("max-parallel {max_parallel}")); + } if let Some(recipe) = parallel.recipe { parts.push(format!("maxparallel = {recipe} in the easyconfig")); } - match parallel.hook { - Some(LimitOp::Divide(factor)) => { + let Some(op) = parallel.hook else { + parts.push(format!("no eb_hooks.py limit for {name}")); + return parts.join(", "); + }; + let mut hook = match op { + LimitOp::Divide(factor) => { let shown = format_factor(factor); - let divided = divide_by_factor(parallel.cores, factor); - if divided.float_result { - parts.push(format!( - "eb_hooks.py divides {name} by {shown} ({} // {shown} is {:.1}, a float)", - parallel.cores, divided.level as f64 - )); + let divided = divide_by_factor(cores, factor); + let mut text = if divided.float_result { + format!( + "eb_hooks.py divides {name} by {shown} ({cores} // {shown} is {:.1}, a float)", + divided.level as f64 + ) } else { - parts.push(format!("eb_hooks.py divides {name} by {shown}")); + format!("eb_hooks.py divides {name} by {shown}") + }; + if let Some(requested) = parallel.requested { + text.push_str(&format!( + " on the {cores} cores, not on --parallel {requested}" + )); } + text } - Some(LimitOp::Maximum(value)) => { - parts.push(format!("eb_hooks.py caps {name} at {value}")); + LimitOp::Maximum(value) => format!("eb_hooks.py caps {name} at {value}"), + }; + let hooked = op.apply(cores); + if hooked > parallel.level { + if op != LimitOp::Maximum(hooked) { + hook.push_str(&format!(", which gives {hooked}")); } - None => parts.push(format!("no eb_hooks.py limit for {name}")), + hook.push_str(&format!( + ", not below {}, so it does not apply", + parallel.level + )); } + parts.push(hook); parts.join(", ") } @@ -1308,9 +1390,25 @@ fn node_partition(node: &NodeSize) -> Partition { } } +/// EasyBuild's level before the hook on `cores`, and the setting that +/// decided it when that is not the cores. +fn easybuild_bound(resources: &BotResources, cores: u32) -> (u32, Option) { + let (level, capped) = resources.easybuild_level(cores); + let bound = match (capped, resources.parallel) { + (Some(max_parallel), _) => Some(format!("max-parallel {max_parallel}")), + (None, Some(requested)) => Some(format!("--parallel {requested}")), + (None, None) => None, + }; + (level, bound) +} + fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32, String) { let partition = node_partition(node); let got = resources.parallel(name, &partition, None); + let (_, bound) = easybuild_bound(resources, node.cores); + let then = bound + .clone() + .unwrap_or_else(|| format!("{} cores", node.cores)); let key = resources .matching_limit(name, &partition.target) .map(|limit| limit.target.as_str()); @@ -1326,14 +1424,11 @@ fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32 if divided.level != got.level { if divided.float_result { text.push_str(&format!( - " = {:.1}, a float, then max-parallel {}", - divided.level as f64, resources.max_parallel + " = {:.1}, a float, then {then}", + divided.level as f64 )); } else { - text.push_str(&format!( - " = {}, then max-parallel {}", - divided.level, resources.max_parallel - )); + text.push_str(&format!(" = {}, then {then}", divided.level)); } } else if divided.float_result { text.push_str(&format!(" = {:.1}, a float", divided.level as f64)); @@ -1344,34 +1439,42 @@ fn current_clause(resources: &BotResources, name: &str, node: &NodeSize) -> (u32 let raw = set_maximum(node.cores, value); let mut text = format!("set_maximum{on}, min({}, {value})", node.cores); if raw != got.level { - text.push_str(&format!( - " = {raw}, then max-parallel {}", - resources.max_parallel - )); + text.push_str(&format!(" = {raw}, then {then}")); } text } - None if node.cores > resources.max_parallel => { - format!("no hook limit, max-parallel {}", resources.max_parallel) - } - None => "no hook limit".to_string(), + None => match bound { + Some(bound) => format!("no hook limit, {bound}"), + None => "no hook limit".to_string(), + }, }; (got.level, text) } -fn memory_clause(mib_per_job: Option, node: &NodeSize, max_parallel: u32) -> String { +fn memory_cap_clause(core_cap: u32, mib_per_job: u32, node: &NodeSize) -> String { + let cap = memory_cap(core_cap, mib_per_job, node.memory_mib); + let formula = format!("min({core_cap}, {} // {mib_per_job})", node.memory_mib); + if cap == 0 { + format!("memory cap 0 ({formula}), not one {mib_per_job} MiB job fits") + } else { + format!("memory cap {cap} ({formula})") + } +} + +fn memory_clause(mib_per_job: Option, node: &NodeSize, resources: &BotResources) -> String { let Some(mib_per_job) = mib_per_job else { return "memory unmeasured".to_string(); }; let raw = limit_by_memory(node.cores, mib_per_job, node.memory_mib); - let curr = node.cores.min(max_parallel).max(1); - let effective = if raw < curr { raw } else { curr }; + let (curr, bound) = easybuild_bound(resources, node.cores); + let effective = raw.min(curr); let formula = format!( "max(1, min({}, {} // {mib_per_job}))", node.cores, node.memory_mib ); if effective != raw { - format!("memory {effective} ({formula} = {raw}, then max-parallel {max_parallel})") + let bound = bound.unwrap_or_else(|| format!("{} cores", node.cores)); + format!("memory {effective} ({formula} = {raw}, then {bound})") } else { format!("memory {effective} ({formula})") } @@ -1422,9 +1525,10 @@ fn job_memory_footer(jobs: &JobMemoryTable) -> String { /// /// Each node is a core count, a memory in MiB, and an optional CPU /// target. The hook column is what `post_ready_hook` would leave, including -/// EasyBuild's `max-parallel`. The memory column is +/// EasyBuild's `--parallel` and `max-parallel`. The memory column is /// `limit_by_memory(cores, mib_per_job)` when the table has a number, and -/// `unmeasured` when it does not. No number is invented. +/// `unmeasured` when it does not. With a number, the hook column is +/// followed by its [`memory_cap`]. No number is invented. pub fn render_parallelism( resources: &BotResources, jobs: &JobMemoryTable, @@ -1433,23 +1537,30 @@ pub fn render_parallelism( let mut out = String::from("parallelism under the current eb_hooks.py limit and under limit_by_memory\n"); let names = report_names(resources, jobs); + let requested = resources + .parallel + .map(|parallel| format!(", --parallel {parallel}")) + .unwrap_or_default(); for node in nodes { match &node.target { Some(target) => out.push_str(&format!( - "node {} cores, {} MiB, {target}\n", + "node {} cores, {} MiB, {target}{requested}\n", node.cores, node.memory_mib )), None => out.push_str(&format!( - "node {} cores, {} MiB\n", + "node {} cores, {} MiB{requested}\n", node.cores, node.memory_mib )), } for name in &names { let (level, clause) = current_clause(resources, name, node); let mib = jobs.get(name).and_then(|package| package.mib_per_job); + let cap = mib + .map(|mib| format!("; {}", memory_cap_clause(level, mib, node))) + .unwrap_or_default(); out.push_str(&format!( - " {name}: current {level} ({clause}); {}\n", - memory_clause(mib, node, resources.max_parallel) + " {name}: current {level} ({clause}){cap}; {}\n", + memory_clause(mib, node, resources) )); } let sentinel = "\u{0}"; @@ -1489,6 +1600,13 @@ pub struct OomCheckRequest { pub nodes: Vec, /// Override `data/job-memory.toml`. pub job_memory: Option, + /// An EESSI `eb_hooks.py` whose `PARALLELISM_LIMITS` replace the hook + /// rows of the bot resource table. + pub hooks: Option, + /// EasyBuild's `--parallel` for the job. `None` counts the cores. + pub parallel: Option, + /// EasyBuild's `--max-parallel` for the job, in place of the table's. + pub max_parallel: Option, } /// What `stack oom` prints. @@ -1507,32 +1625,65 @@ pub struct OomCheckReport { pub unknown_arches: Vec, /// Divide factors that are not integers. Also included in `stdout`. pub factor_warnings: Vec, + /// `--hooks` statements that add to a name and drop the sum, such as + /// `msg+ "..."`. Also included in `stdout`. + pub hook_warnings: Vec, } /// Estimate each plan build's peak on each bot node and warn where it /// reaches the job's memory. pub fn run_oom_check(request: &OomCheckRequest) -> Result { - if request.easystacks.is_empty() && request.nodes.is_empty() { + if request.easystacks.is_empty() && request.nodes.is_empty() && request.hooks.is_none() { return Err(usage( - "pass at least one --easystack, or --node to compare parallelism", + "pass at least one --easystack, --node to compare parallelism, or --hooks to check an eb_hooks.py", )); } let mut nodes = Vec::new(); for text in &request.nodes { nodes.push(parse_node_size(text).map_err(usage)?); } - let resources = match &request.resources { + for (flag, value) in [ + ("--parallel", request.parallel), + ("--max-parallel", request.max_parallel), + ] { + if value == Some(0) { + return Err(usage(format!("{flag} must be a positive integer"))); + } + } + let mut resources = match &request.resources { Some(path) => BotResources::load(path)?, None => BotResources::bundled()?, }; + resources.parallel = request.parallel; + if let Some(max_parallel) = request.max_parallel { + resources.max_parallel = max_parallel; + } let jobs = match &request.job_memory { Some(path) => JobMemoryTable::load(path)?, None => JobMemoryTable::bundled()?, }; + let mut stdout = String::new(); + let mut hook_warnings = Vec::new(); + if let Some(path) = &request.hooks { + let file = path.display().to_string(); + let text = std::fs::read_to_string(path).with_context(|| format!("--hooks {file}"))?; + let hooks = crate::eb_hooks::parse_hooks(&text, &file)?; + stdout.push_str(&format!( + "hooks: {} {} rows from {file}\n", + hooks.limits.len(), + crate::eb_hooks::LIMITS_NAME + )); + hook_warnings = hooks + .dropped_sums + .iter() + .map(|sum| sum.message(&file)) + .collect(); + resources.limits = hooks.limits; + resources.hooks_source = file; + } let factor_warnings = resources.factor_warnings(); if request.easystacks.is_empty() { - let mut stdout = String::new(); - for line in &factor_warnings { + for line in factor_warnings.iter().chain(&hook_warnings) { stdout.push_str(line); stdout.push('\n'); } @@ -1547,6 +1698,7 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { partitions: 0, unknown_arches: Vec::new(), factor_warnings, + hook_warnings, }); } let profile = match &request.profile { @@ -1623,13 +1775,12 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { } } - let mut stdout = String::new(); for arch in &unknown_arches { stdout.push_str(&format!( "{arch} has no bot node in the bot resource table, so it is not checked\n" )); } - for line in &factor_warnings { + for line in factor_warnings.iter().chain(&hook_warnings) { stdout.push_str(line); stdout.push('\n'); } @@ -1658,6 +1809,7 @@ pub fn run_oom_check(request: &OomCheckRequest) -> Result { partitions: partitions.len(), unknown_arches, factor_warnings, + hook_warnings, }) } @@ -1856,6 +2008,70 @@ mod tests { assert_eq!(resources.parallel("libxc", zen4, Some(4)).level, 4); } + #[test] + fn the_hook_applies_to_the_cores_not_to_easybuild_parallel() { + let mut resources = BotResources::bundled().unwrap(); + let zen4 = zen4(&resources).clone(); + let a64fx = resources + .partitions + .iter() + .find(|p| p.target == "aarch64/a64fx") + .unwrap() + .clone(); + + resources.parallel = Some(12); + let libxc = resources.parallel("libxc", &zen4, None); + // 16 // 2, not 12 // 2. + assert_eq!(libxc.level, 8); + assert_eq!( + parallel_reason("libxc", &libxc), + "--parallel 12 on 16 cores, eb_hooks.py divides libxc by 2 on the 16 cores, not on --parallel 12" + ); + assert_eq!(resources.parallel("zlib", &zen4, None).level, 12); + + resources.parallel = Some(4); + let libxc = resources.parallel("libxc", &zen4, None); + assert_eq!(libxc.level, 4); + assert_eq!( + parallel_reason("libxc", &libxc), + "--parallel 4 on 16 cores, eb_hooks.py divides libxc by 2 on the 16 cores, not on --parallel 4, which gives 8, not below 4, so it does not apply" + ); + let rocblas = resources.parallel("rocBLAS", &zen4, None); + assert_eq!( + parallel_reason("rocBLAS", &rocblas), + "--parallel 4 on 16 cores, eb_hooks.py caps rocBLAS at 8, not below 4, so it does not apply" + ); + + // EasyBuild takes --parallel as given, above the cores, up to + // max-parallel. The hook still starts from the cores. + resources.parallel = Some(32); + resources.max_parallel = 64; + assert_eq!(resources.parallel("zlib", &zen4, None).level, 32); + assert_eq!(resources.parallel("libxc", &zen4, None).level, 8); + resources.max_parallel = 16; + let zlib = resources.parallel("zlib", &zen4, None); + assert_eq!(zlib.level, 16); + assert_eq!( + parallel_reason("zlib", &zlib), + "--parallel 32 on 16 cores, max-parallel 16, no eb_hooks.py limit for zlib" + ); + + resources.parallel = None; + let llvm = resources.parallel("LLVM", &a64fx, None); + assert_eq!(llvm.level, 16); + assert_eq!( + parallel_reason("LLVM", &llvm), + "48 cores, max-parallel 16, eb_hooks.py divides LLVM by 2, which gives 24, not below 16, so it does not apply" + ); + resources.max_parallel = 48; + let llvm = resources.parallel("LLVM", &a64fx, None); + assert_eq!(llvm.level, 24); + assert_eq!( + parallel_reason("LLVM", &llvm), + "48 cores, eb_hooks.py divides LLVM by 2" + ); + } + #[test] fn version_toolchain_and_target_families_have_boundaries() { assert!(in_family("7.0.0", "7.0", &['.', '-'])); @@ -1912,7 +2128,7 @@ source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6420263 .iter() .find(|e| e.name == "libxc" && !e.toolchains.is_empty()) .unwrap(); - assert_eq!(fitting_parallel(entry, 8, 32.0), Some(7)); + assert_eq!(fitting_parallel(entry, 8, 32.0), Some(4)); let hip = profile .entries() .iter() @@ -1952,6 +2168,30 @@ source = "https://github.com/EESSI/software-layer/pull/1772#issuecomment-6420263 assert_eq!(limit_by_memory(48, 100, 30_016), 48); } + #[test] + fn the_memory_cap_is_not_raised_to_one_and_matches_the_byte_quotient() { + assert_eq!(memory_cap(8, 8192, 32_768), 4); + assert_eq!(memory_cap(8, 8192, 131_072), 8); + assert_eq!(memory_cap(8, 8192, 4096), 0); + assert_eq!(memory_cap(6, 8192, 30_016), 3); + assert_eq!(memory_cap(8, 0, 32_768), 8); + const BYTES_PER_MIB: u64 = 1 << 20; + let gib = 1024 * BYTES_PER_MIB; + for bytes in [ + 0, + BYTES_PER_MIB - 1, + 32 * gib, + 32 * gib + 12_345, + 30_016 * BYTES_PER_MIB + BYTES_PER_MIB - 1, + ] { + for mib_per_job in [1, 5, 6554, 8192] { + let by_bytes = bytes / (u64::from(mib_per_job) * BYTES_PER_MIB); + let by_mib = memory_cap(u32::MAX, mib_per_job, bytes / BYTES_PER_MIB); + assert_eq!(u64::from(by_mib), by_bytes, "{bytes} // {mib_per_job} MiB"); + } + } + } + #[test] fn a_non_integer_divide_warns_and_uses_the_numeric_floor() { let text = r#" @@ -2104,7 +2344,7 @@ note = "fixture for the report arithmetic, not a measurement of libxc" let one = render_parallelism(&resources, &fixture, &nodes[..1]); assert!( one.contains( - " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory 4 (max(1, min(16, 32768 // 8192)))\n" + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192)))\n" ), "{one}" ); diff --git a/src/site_commands.rs b/src/site_commands.rs index 8e5a5605..26a47d3c 100644 --- a/src/site_commands.rs +++ b/src/site_commands.rs @@ -1716,6 +1716,9 @@ pub fn run_stack_batch(args: &StackBatchRequest) -> Result { resources: args.oom_resources.clone(), nodes: Vec::new(), job_memory: None, + hooks: None, + parallel: None, + max_parallel: None, })?; let oom = oom_report.warnings; let factor_warnings = oom_report.factor_warnings; diff --git a/tests/mcp_v1.rs b/tests/mcp_v1.rs index bdbac047..4bd8ff21 100644 --- a/tests/mcp_v1.rs +++ b/tests/mcp_v1.rs @@ -3792,3 +3792,115 @@ fn every_cli_subcommand_has_an_mcp_tool() { assert!(listed, "CLI subcommand {rendered} has no MCP tool"); } } + +#[test] +fn mcp_stack_oom_reads_an_eb_hooks_py() { + let hooks = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/oom_profile_fixtures/hooks/divide-2.5.py") + .display() + .to_string(); + let response = call(1, "eb_stack_oom", json!({"hooks": hooks})); + let body = &response["result"]["structuredContent"]; + let warnings = body["factor_warnings"].as_array().expect("factor_warnings"); + assert_eq!(warnings.len(), 1, "{body}"); + let warning = warnings[0].as_str().expect("warning text"); + assert!( + warning.starts_with(&format!( + "warning: {hooks}:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5" + )), + "{warning}" + ); + let text = body["text"].as_str().expect("text"); + assert!( + text.starts_with(&format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n")), + "{text}" + ); + assert_eq!(body["hook_warnings"], json!([]), "{body}"); +} + +#[test] +fn mcp_stack_oom_returns_the_dropped_msg_sum() { + let hooks = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/oom_profile_fixtures/hooks/msg-plus.py") + .display() + .to_string(); + let response = call(1, "eb_stack_oom", json!({"hooks": hooks})); + let body = &response["result"]["structuredContent"]; + let warning = format!( + "warning: {hooks}:30: msg+ \"to avoid out-of-memory failures during building/testing\" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends" + ); + assert_eq!(body["hook_warnings"], json!([warning]), "{body}"); + assert_eq!(body["factor_warnings"], json!([]), "{body}"); + let text = body["text"].as_str().expect("text"); + assert!(text.ends_with(&format!("{warning}\n")), "{text}"); +} + +#[test] +fn mcp_stack_oom_prints_the_memory_cap_beside_the_core_cap() { + let job_memory = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/oom_profile_fixtures/job-memory-fixture.toml") + .display() + .to_string(); + let response = call( + 1, + "eb_stack_oom", + json!({"node": ["16:4096:x86_64/amd/zen4"], "job_memory": job_memory}), + ); + let text = response["result"]["structuredContent"]["text"] + .as_str() + .expect("text"); + assert!( + text.contains(" libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192)))\n"), + "{text}" + ); +} + +#[test] +fn mcp_stack_oom_takes_easybuild_parallel_and_max_parallel() { + let response = call( + 1, + "eb_stack_oom", + json!({"node": ["16:32768:x86_64/amd/zen4"], "parallel": 4}), + ); + assert_eq!(response["result"]["isError"], false, "{response}"); + let text = response["result"]["structuredContent"]["text"] + .as_str() + .expect("text"); + assert!( + text.contains("node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 4\n"), + "{text}" + ); + assert!( + text.contains(" libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured\n"), + "{text}" + ); + + let response = call( + 2, + "eb_stack_oom", + json!({"node": ["48:30016:aarch64/a64fx"], "max_parallel": 48}), + ); + let text = response["result"]["structuredContent"]["text"] + .as_str() + .expect("text"); + assert!( + text.contains( + " LLVM: current 24 (divide_by_factor on *, max(1, 48 // 2)); memory unmeasured\n" + ), + "{text}" + ); + + let response = call( + 3, + "eb_stack_oom", + json!({"node": ["16:32768"], "parallel": 0}), + ); + assert_eq!(response["result"]["isError"], true, "{response}"); + let message = response["result"]["content"][0]["text"] + .as_str() + .unwrap_or_default(); + assert!( + message.contains("--parallel must be a positive integer"), + "{message}" + ); +} diff --git a/tests/oom_profile.rs b/tests/oom_profile.rs index d754fef3..475f85f9 100644 --- a/tests/oom_profile.rs +++ b/tests/oom_profile.rs @@ -1,7 +1,8 @@ //! Out-of-memory warnings for a plan, from EESSI software-layer bot jobs. //! //! Each easystack is the lines a real pull request added. The libxc build -//! in pull request 1772 was killed twice on zen4. Highway in 1692 ran out on +//! in pull request 1772 was killed on zen4 twice at 8 and once at 5. +//! Highway in 1692 ran out on //! aarch64/generic. hipBLASLt for rocBLAS in 1549 needed about 90 GiB. The //! PETSc test step in 1608 ran out on zen5 before the hooks halved it. @@ -35,21 +36,24 @@ fn libxc_from_pr_1772_warns_on_zen4_only() { let easystack = arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"); let (code, stdout, stderr) = run(&["stack", "oom", "--easystack", &easystack]); assert_eq!(code, 0, "{stdout}\n{stderr}"); - let first = "libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 4.0 GiB = 32.0 GiB, the job has 32.0 GiB (software-layer #1772)"; + let first = "libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337)"; assert!(stdout.contains(first), "{stdout}"); assert!( stdout.contains(" parallel 8: 16 cores, eb_hooks.py divides libxc by 2\n"), "{stdout}" ); assert!( - stdout.contains(" remedy: maxparallel = 7 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 7 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 4.0 GiB per process is a lower bound, so 7 may still be too many\n"), + stdout.contains(" remedy: maxparallel = 4 in libxc-7.0.0-GCC-15.2.0.eb, or a libxc limit of 4 for x86_64/amd/zen4 in PARALLELISM_LIMITS in software-layer-scripts eb_hooks.py; 6.4 GiB per process is a lower bound, so 4 may still be too many\n"), "{stdout}" ); assert!( stdout.contains(" remedy: build x86_64/amd/zen4 on a node with more memory: eessi-bot-surf amd-zen4 (16 cores, 180.0 GiB, accel nvidia/cc90)\n"), "{stdout}" ); - assert!(stdout.contains("divider of 2.5 or 3"), "{stdout}"); + assert!( + stdout.contains(" remedy: a libxc divider of 3 left 5 cores and ran out too; a reduction of 8, as asked in the review of software-layer-scripts #337 (issuecomment-6599147327), is 2 cores on the AWS build cluster\n"), + "{stdout}" + ); // aarch64/generic built the same easyconfig at 8 cores in that pull request. assert!(!stdout.contains("on aarch64/generic"), "{stdout}"); assert_eq!( @@ -213,6 +217,7 @@ fn a_target_without_a_bot_node_is_reported_not_checked() { fn an_easystack_is_required_and_a_directory_is_refused() { let (code, _, stderr) = run(&["stack", "oom"]); assert_eq!(code, 2, "{stderr}"); + assert!(stderr.contains("--hooks"), "{stderr}"); let (code, _, stderr) = run(&["stack", "oom", "--easystack", &arg("easystacks")]); assert_eq!(code, 2, "{stderr}"); assert!(stderr.contains("is a directory"), "{stderr}"); @@ -237,13 +242,174 @@ fn a_non_integer_divide_warns_and_the_estimate_uses_six() { ), "{stdout}" ); - assert!(!stdout.contains("may run out of memory"), "{stdout}"); assert!( - stdout.ends_with("oom=0 fits=1 builds=5 nodes=1\n"), + stdout.contains( + "libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB" + ), + "{stdout}" + ); + assert!( + stdout.contains( + " parallel 6: 16 cores, eb_hooks.py divides libxc by 2.5 (16 // 2.5 is 6.0, a float)\n" + ), + "{stdout}" + ); + assert!( + stdout.ends_with("oom=1 fits=0 builds=5 nodes=1\n"), + "{stdout}" + ); +} + +#[test] +fn an_eb_hooks_py_with_a_float_divider_warns_at_its_line() { + let hooks = arg("hooks/divide-2.5.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stdout}\n{stderr}"); + assert!( + stdout.starts_with(&format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + warning: {hooks}:20: PARALLELISM_LIMITS divide_by_factor for libxc on * uses 2.5, which is not an integer; eb_hooks.py computes max(1, parallel // factor), and 16 // 2.5 is 6.0, a float. Use an integer factor.\n\ + libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 6 x 6.4 GiB = 38.4 GiB, the job has 32.0 GiB" + )), + "{stdout}" + ); + assert_eq!(stdout.matches("warning:").count(), 1, "{stdout}"); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); +} + +#[test] +fn an_eb_hooks_py_with_integer_dividers_does_not_warn() { + let hooks = arg("hooks/integers.py"); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", &hooks]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n") + ); + + // The fixture's libxc and A64FX rows are the bundled table's, so the + // parallel levels match. + let nodes = [ + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]; + let (_, bundled, _) = run(&[&["stack", "oom"][..], &nodes].concat()); + let (code, read, stderr) = run(&[&["stack", "oom", "--hooks", &hooks][..], &nodes].concat()); + assert_eq!(code, 0, "{stderr}"); + for line in [ + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n", + " (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured\n", + ] { + assert!(bundled.contains(line), "{bundled}"); + assert!(read.contains(line), "{read}"); + } + assert!(!read.contains("warning:"), "{read}"); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains(" parallel 8: 16 cores, eb_hooks.py divides libxc by 2\n"), + "{stdout}" + ); + assert!(!stdout.contains("warning:"), "{stdout}"); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); +} + +#[test] +fn an_eb_hooks_py_row_the_model_cannot_follow_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let hooks = dir.path().join("eb_hooks.py"); + std::fs::write( + &hooks, + "PARALLELISM_LIMITS = {\n 'libxc': {'*': (limit_by_memory, 6400)},\n}\n", + ) + .unwrap(); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", hooks.to_str().unwrap()]); + assert_eq!(code, 1, "{stdout}\n{stderr}"); + assert!( + stderr.contains(&format!( + "{}:2: PARALLELISM_LIMITS row for libxc on * uses limit_by_memory; the ops are divide_by_factor and set_maximum", + hooks.display() + )), + "{stderr}" + ); + assert!(stdout.is_empty(), "{stdout}"); +} + +#[test] +fn a_dropped_msg_sum_in_eb_hooks_py_warns_at_its_line() { + let hooks = arg("hooks/msg-plus.py"); + let warning = format!( + "warning: {hooks}:30: msg+ \"to avoid out-of-memory failures during building/testing\" in post_ready_hook adds to msg and drops the result, so msg does not change; msg += appends\n" + ); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", &hooks]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n{warning}") + ); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.starts_with(&format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n{warning}libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4" + )), + "{stdout}" + ); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), "{stdout}" ); } +#[test] +fn an_eb_hooks_py_that_appends_with_plus_equals_does_not_warn() { + let hooks = arg("hooks/msg-plus-equals.py"); + let (code, stdout, stderr) = run(&["stack", "oom", "--hooks", &hooks]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!("hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n") + ); +} + #[test] fn node_sizes_compare_the_hook_with_the_memory_rule() { let (code, stdout, stderr) = run(&[ @@ -300,7 +466,7 @@ fn node_sizes_compare_the_hook_with_the_memory_rule() { assert_eq!(code, 0, "{stderr}"); assert!( stdout.contains( - " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory 4 (max(1, min(16, 32768 // 8192)))\n" + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192)))\n" ), "{stdout}" ); @@ -311,6 +477,334 @@ fn node_sizes_compare_the_hook_with_the_memory_rule() { ); } +#[test] +fn the_memory_cap_is_the_core_cap_under_the_memory_ceiling() { + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--mib-per-job", + &arg("job-memory-fixture.toml"), + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "16:131072:x86_64/amd/zen4", + "--node", + "16:4096:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + let libxc: Vec<&str> = stdout + .lines() + .filter(|line| line.starts_with(" libxc: ")) + .collect(); + assert_eq!( + libxc, + [ + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 4 (min(8, 32768 // 8192)); memory 4 (max(1, min(16, 32768 // 8192)))", + // The hook already holds libxc below what 128 GiB allows. + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 8 (min(8, 131072 // 8192)); memory 16 (max(1, min(16, 131072 // 8192)))", + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory cap 0 (min(8, 4096 // 8192)), not one 8192 MiB job fits; memory 1 (max(1, min(16, 4096 // 8192)))", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory cap 3 (min(6, 30016 // 8192)); memory 3 (max(1, min(48, 30016 // 8192)))", + ] + ); + // A name the table does not measure gets no cap. + assert!( + stdout.contains( + " LLVM: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n" + ), + "{stdout}" + ); + assert_eq!(stdout.matches("memory cap").count(), 4, "{stdout}"); + + let (_, by_long_name, _) = run(&[ + "stack", + "oom", + "--job-memory", + &arg("job-memory-fixture.toml"), + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "16:131072:x86_64/amd/zen4", + "--node", + "16:4096:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(by_long_name, stdout); +} + +#[test] +fn the_hook_divides_the_node_cores_not_easybuild_parallel() { + let hooks = arg("hooks/integers.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--hooks", + &hooks, + "--parallel", + "12", + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + // 16 // 2 is 8; 12 // 2 would have been 6. + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + parallelism under the current eb_hooks.py limit and under limit_by_memory\n\ + node 16 cores, 32768 MiB, x86_64/amd/zen4, --parallel 12\n \ + libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n\ + node 48 cores, 30016 MiB, aarch64/a64fx, --parallel 12\n \ + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n \ + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured\n\ + job memory: libxc 7.0 unmeasured, no mib_per_job\n" + ) + ); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--hooks", + &hooks, + "--parallel", + "4", + "--node", + "16:32768:x86_64/amd/zen4", + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains( + " libxc: current 4 (divide_by_factor on *, max(1, 16 // 2) = 8, then --parallel 4); memory unmeasured\n" + ), + "{stdout}" + ); + + // EasyBuild uses --parallel as given, above the cores; a build with no + // hook row runs at 32 on 16 cores once max-parallel allows it. + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--parallel", + "32", + "--max-parallel", + "64", + "--node", + "16:32768:x86_64/amd/zen4", + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains(" Qt5: current 32 (no hook limit, --parallel 32); memory unmeasured\n"), + "{stdout}" + ); + assert!( + stdout.contains( + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured\n" + ), + "{stdout}" + ); + + let (_, default, _) = run(&["stack", "oom", "--node", "48:30016:aarch64/a64fx"]); + assert!( + default.contains(" LLVM: current 16 (divide_by_factor on *, max(1, 48 // 2) = 24, then max-parallel 16); memory unmeasured\n"), + "{default}" + ); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--max-parallel", + "48", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains( + " LLVM: current 24 (divide_by_factor on *, max(1, 48 // 2)); memory unmeasured\n" + ), + "{stdout}" + ); +} + +#[test] +fn easybuild_parallel_12_still_leaves_libxc_at_8_on_zen4() { + let easystack = arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--parallel", + "12", + "--easystack", + &easystack, + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.contains(" parallel 8: --parallel 12 on 16 cores, eb_hooks.py divides libxc by 2 on the 16 cores, not on --parallel 12\n"), + "{stdout}" + ); + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--parallel", + "4", + "--easystack", + &easystack, + ]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!(stdout, "oom=0 fits=2 builds=5 nodes=2\n"); +} + +#[test] +fn a_zero_easybuild_parallel_level_is_rejected() { + for flag in ["--parallel", "--max-parallel"] { + let (code, stdout, stderr) = run(&["stack", "oom", flag, "0", "--node", "16:32768"]); + assert_eq!(code, 2, "{stdout}\n{stderr}"); + assert!( + stderr.contains(&format!("{flag} must be a positive integer")), + "{stderr}" + ); + assert!(stdout.is_empty(), "{stdout}"); + } +} + +#[test] +fn a_libxc_cap_of_8_is_the_divide_by_2_on_16_cores() { + let nodes = [ + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + "--node", + "64:262144:x86_64/amd/zen4", + "--node", + "12:24576:x86_64/amd/zen4", + ]; + let hooks = arg("hooks/set-maximum-8.py"); + let (code, stdout, stderr) = run(&[&["stack", "oom", "--hooks", &hooks][..], &nodes].concat()); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + parallelism under the current eb_hooks.py limit and under limit_by_memory\n\ + node 16 cores, 32768 MiB, x86_64/amd/zen4\n \ + libxc: current 8 (set_maximum on *, min(16, 8)); memory unmeasured\n\ + node 48 cores, 30016 MiB, aarch64/a64fx\n \ + libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n \ + (other builds): current 12 (divide_by_factor on aarch64/a64fx, max(1, 48 // 4)); memory unmeasured\n\ + node 64 cores, 262144 MiB, x86_64/amd/zen4\n \ + libxc: current 8 (set_maximum on *, min(64, 8)); memory unmeasured\n\ + node 12 cores, 24576 MiB, x86_64/amd/zen4\n \ + libxc: current 8 (set_maximum on *, min(12, 8)); memory unmeasured\n\ + job memory: libxc 7.0 unmeasured, no mib_per_job\n" + ) + ); + + // The same nodes under the divide by 2 that the scripts ship. + let (code, divide, stderr) = run(&[ + &["stack", "oom", "--hooks", &arg("hooks/integers.py")][..], + &nodes, + ] + .concat()); + assert_eq!(code, 0, "{stderr}"); + let libxc: Vec<&str> = divide + .lines() + .filter(|line| line.starts_with(" libxc: ")) + .collect(); + assert_eq!( + libxc, + [ + " libxc: current 8 (divide_by_factor on *, max(1, 16 // 2)); memory unmeasured", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured", + " libxc: current 16 (divide_by_factor on *, max(1, 64 // 2) = 32, then max-parallel 16); memory unmeasured", + " libxc: current 6 (divide_by_factor on *, max(1, 12 // 2)); memory unmeasured", + ] + ); +} + +#[test] +fn libxc_from_pr_1772_still_warns_on_zen4_under_a_cap_of_8() { + let hooks = arg("hooks/set-maximum-8.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert!( + stdout.starts_with(&format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + libxc-7.0.0-GCC-15.2.0.eb may run out of memory on x86_64/amd/zen4 (eessi-bot-mc-aws amd-zen4): 8 x 6.4 GiB = 51.2 GiB, the job has 32.0 GiB (software-layer #1772, software-layer-scripts #337)\n \ + parallel 8: 16 cores, eb_hooks.py caps libxc at 8\n" + )), + "{stdout}" + ); + assert!(!stdout.contains("warning:"), "{stdout}"); + // The 180 GiB zen4 node fits at 8 under either row. + assert!( + stdout.ends_with("oom=1 fits=1 builds=5 nodes=2\n"), + "{stdout}" + ); +} + +#[test] +fn a_reduction_of_8_leaves_libxc_2_cores() { + let hooks = arg("hooks/divide-8.py"); + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--hooks", + &hooks, + "--node", + "16:32768:x86_64/amd/zen4", + "--node", + "48:30016:aarch64/a64fx", + ]); + assert_eq!(code, 0, "{stderr}"); + for line in [ + " libxc: current 2 (divide_by_factor on *, max(1, 16 // 8)); memory unmeasured\n", + " libxc: current 6 (set_maximum on aarch64/a64fx, min(48, 6)); memory unmeasured\n", + ] { + assert!(stdout.contains(line), "{stdout}"); + } + + let (code, stdout, stderr) = run(&[ + "stack", + "oom", + "--arch", + "x86_64/amd/zen4", + "--hooks", + &hooks, + "--easystack", + &arg("easystacks/pr1772-eessi-2026.06-eb-5.4.0-2026.1.yml"), + ]); + assert_eq!(code, 0, "{stderr}"); + assert_eq!( + stdout, + format!( + "hooks: 3 PARALLELISM_LIMITS rows from {hooks}\n\ + oom=0 fits=2 builds=5 nodes=2\n" + ) + ); +} + #[test] fn a_node_size_without_cores_or_memory_is_rejected() { let (code, _, stderr) = run(&["stack", "oom", "--node", "16"]); @@ -443,6 +937,6 @@ fn the_how_to_shows_the_real_output() { "{command}" ); } - assert_eq!(oom_blocks, 7); + assert_eq!(oom_blocks, 16); assert_eq!(rss_blocks, 1); } diff --git a/tests/oom_profile_fixtures/hooks/divide-2.5.py b/tests/oom_profile_fixtures/hooks/divide-2.5.py new file mode 100644 index 00000000..56ca96b3 --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/divide-2.5.py @@ -0,0 +1,23 @@ +# integers.py with the libxc divider the maintainers suggested in +# EESSI/software-layer#1772 (issuecomment-6420263033): 2.5 instead of 2. +# 16 // 2.5 is 6.0 in Python, a float. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2.5), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} diff --git a/tests/oom_profile_fixtures/hooks/divide-8.py b/tests/oom_profile_fixtures/hooks/divide-8.py new file mode 100644 index 00000000..64741285 --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/divide-8.py @@ -0,0 +1,24 @@ +# integers.py with libxc's '*' row a reduction of 8, (divide_by_factor, 8), +# as asked in the review of EESSI/software-layer-scripts#337 +# (issuecomment-6599147327): 2 cores on a 16-core build node. The A64FX +# row stays at 6. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 8), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} diff --git a/tests/oom_profile_fixtures/hooks/integers.py b/tests/oom_profile_fixtures/hooks/integers.py new file mode 100644 index 00000000..1706c63a --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/integers.py @@ -0,0 +1,22 @@ +# PARALLELISM_LIMITS in the layout of software-layer-scripts eb_hooks.py, +# cut down to the A64FX row and libxc. Every divide_by_factor is an integer. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} diff --git a/tests/oom_profile_fixtures/hooks/msg-plus-equals.py b/tests/oom_profile_fixtures/hooks/msg-plus-equals.py new file mode 100644 index 00000000..6da7e6d6 --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/msg-plus-equals.py @@ -0,0 +1,28 @@ +# msg-plus.py with the log message appended by msg += "...". +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} + + +def post_ready_hook(self, *args, **kwargs): + new_parallel = 8 + msg = "limiting parallelism to %s for %s " + msg += "to avoid out-of-memory failures during building/testing" + print(msg % (new_parallel, self.name)) diff --git a/tests/oom_profile_fixtures/hooks/msg-plus.py b/tests/oom_profile_fixtures/hooks/msg-plus.py new file mode 100644 index 00000000..afe7292e --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/msg-plus.py @@ -0,0 +1,31 @@ +# integers.py with a post_ready_hook that ends its log message the way +# software-layer-scripts eb_hooks.py does since 8132905f: msg+ "..." on a +# line of its own. That builds the longer string and drops it, so the +# second half of the message never prints. msg += "..." appends. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (divide_by_factor, 2), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +} + + +def post_ready_hook(self, *args, **kwargs): + new_parallel = 8 + msg = "limiting parallelism to %s for %s " + msg+ "to avoid out-of-memory failures during building/testing" + print(msg % (new_parallel, self.name)) diff --git a/tests/oom_profile_fixtures/hooks/set-maximum-8.py b/tests/oom_profile_fixtures/hooks/set-maximum-8.py new file mode 100644 index 00000000..72a7d08d --- /dev/null +++ b/tests/oom_profile_fixtures/hooks/set-maximum-8.py @@ -0,0 +1,23 @@ +# integers.py with libxc's '*' row a cap of 8, (set_maximum, 8), in place +# of the divide by 2: the form proposed in the review of +# EESSI/software-layer-scripts#337. The A64FX row stays at 6. +# stack oom --hooks parses this file and never runs it. + +CPU_TARGET_A64FX = 'aarch64/a64fx' + + +def divide_by_factor(parallel, factor): + return max(1, parallel // factor) + + +def set_maximum(parallel, max_value): + return min(parallel, max_value) + + +PARALLELISM_LIMITS = { + CPU_TARGET_A64FX: (divide_by_factor, 4), + 'libxc': { + '*': (set_maximum, 8), + CPU_TARGET_A64FX: (set_maximum, 6), + }, +}