diff --git a/.github/scripts/run_parallel_benchmarks.sh b/.github/scripts/run_parallel_benchmarks.sh index 2f262692d..c2d7d3c2a 100755 --- a/.github/scripts/run_parallel_benchmarks.sh +++ b/.github/scripts/run_parallel_benchmarks.sh @@ -64,22 +64,43 @@ if [ "$device" = "gpu" ] && [ "$cluster" = "phoenix" ]; then master_exit=0 echo "Single-node bench-pair job completed successfully." else - # --- Other clusters / Phoenix CPU: two concurrent jobs, monitored serially --- + # --- Other clusters / Phoenix CPU: concurrent jobs, monitored serially --- # The bench script must come from the PR tree (master may not have it). PR_BENCH_SCRIPT="$(cd "${SCRIPT_DIR}/../workflows/common" && pwd)/bench.sh" - # Phase 1: Submit both SLURM jobs (no monitoring yet) - echo "Submitting PR benchmark..." - (cd pr && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster") - pr_job_id=$(cat "pr/${log_slug}.slurm_job_id") - echo "PR job submitted: $pr_job_id" + # Frontier's "normal" QOS caps a job at 2 h, and one bench job there spends ~30 min + # building and ~17 min per case, so the 7-case list needs ~2.5 h and is killed on + # case 6 every time. Split each tree's run across concurrent shards (the same i/N + # scheme the case-optimization lanes use), each of which rebuilds and runs its + # share, then merge the shards' YAMLs so bench_diff still sees one file per tree. + # An empty shard means one unsharded job, which is what every other cluster gets. + case "$cluster" in + frontier|frontier_amd) bench_shards="1/2 2/2" ;; + *) bench_shards="" ;; + esac + + # Per-tree list of the slugs whose .out/.slurm_job_id/.yaml this run produced, so + # the monitor and merge steps below need no knowledge of how many shards there are. + tree_slugs() { # arg: ; prints the job slug submit-slurm-job.sh will use + local shard="$1" + if [ -n "$shard" ]; then + echo "${log_slug}-$(echo "$shard" | sed 's|/|-of-|')" + else + echo "$log_slug" + fi + } - echo "Submitting master benchmark..." - (cd master && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster") - master_job_id=$(cat "master/${log_slug}.slurm_job_id") - echo "Master job submitted: $master_job_id" + # Phase 1: Submit every SLURM job (no monitoring yet) + for dir in pr master; do + for shard in ${bench_shards:-""}; do + slug=$(tree_slugs "$shard") + echo "Submitting ${dir} benchmark${shard:+ (shard $shard)}..." + (cd "$dir" && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster" "$shard") + echo "${dir} job submitted: $(cat "${dir}/${slug}.slurm_job_id")" + done + done - echo "Both SLURM jobs submitted — running concurrently on compute nodes." + echo "All SLURM jobs submitted — running concurrently on compute nodes." echo "Monitoring sequentially to conserve login node memory." # Phase 2: Monitor sequentially (one at a time on login node) @@ -90,11 +111,12 @@ else # resubmitted job no longer overlaps its counterpart, slightly reducing # same-load fairness -- still preferable to failing the run on an infra preempt. : "${MAX_PREEMPT_RESUBMITS:=10}" - monitor_bench_with_resubmit() { # arg: (pr|master); sets BENCH_MON_RC - local dir="$1" - local out="${dir}/${log_slug}.out" - local jobid attempt=0 rc - jobid=$(cat "${dir}/${log_slug}.slurm_job_id") + monitor_bench_with_resubmit() { # args: (pr|master) ; sets BENCH_MON_RC + local dir="$1" shard="$2" + local slug out jobid attempt=0 rc + slug=$(tree_slugs "$shard") + out="${dir}/${slug}.out" + jobid=$(cat "${dir}/${slug}.slurm_job_id") while :; do rc=0 bash "${SCRIPT_DIR}/run_monitored_slurm_job.sh" "$jobid" "$out" || rc=$? @@ -103,26 +125,49 @@ else return fi if [ "$attempt" -ge "$MAX_PREEMPT_RESUBMITS" ]; then - echo "::error::${dir} benchmark preempted ${MAX_PREEMPT_RESUBMITS}x without completing; giving up." + echo "::error::${dir} benchmark${shard:+ shard $shard} preempted ${MAX_PREEMPT_RESUBMITS}x without completing; giving up." BENCH_MON_RC=1 return fi attempt=$((attempt + 1)) echo "::warning::${dir} benchmark job $jobid was preempted; resubmitting (attempt ${attempt}/${MAX_PREEMPT_RESUBMITS})." rm -f "$out" - ( cd "$dir" && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster" ) - jobid=$(cat "${dir}/${log_slug}.slurm_job_id") + ( cd "$dir" && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster" "$shard" ) + jobid=$(cat "${dir}/${slug}.slurm_job_id") echo "${dir} benchmark resubmitted as job $jobid" done } - echo "" - echo "=== Monitoring PR job $pr_job_id ===" - monitor_bench_with_resubmit pr - pr_exit=$BENCH_MON_RC + # Monitor every shard of a tree; the tree's exit is the first non-zero one. + monitor_tree() { # arg: ; sets BENCH_TREE_RC + local dir="$1" shard + BENCH_TREE_RC=0 + for shard in ${bench_shards:-""}; do + echo "" + echo "=== Monitoring ${dir} job $(cat "${dir}/$(tree_slugs "$shard").slurm_job_id")${shard:+ (shard $shard)} ===" + monitor_bench_with_resubmit "$dir" "$shard" + if [ "$BENCH_MON_RC" -ne 0 ] && [ "$BENCH_TREE_RC" -eq 0 ]; then + BENCH_TREE_RC="$BENCH_MON_RC" + fi + done + } + + # Fold a tree's shard YAMLs into the one file bench_diff reads. Unsharded runs already + # wrote that file. Runs from the PR tree: master may predate bench_merge. + merge_tree_shards() { # arg: + local dir="$1" shard inputs="" + [ -z "$bench_shards" ] && return 0 + for shard in $bench_shards; do + inputs="$inputs ../${dir}/$(tree_slugs "$shard").yaml" + done + (cd pr && ./mfc.sh bench_merge -o "../${dir}/${job_slug}.yaml" $inputs) + } + + monitor_tree pr + pr_exit=$BENCH_TREE_RC if [ "$pr_exit" -ne 0 ]; then echo "PR job exited with code: $pr_exit" - tail -n 50 "pr/${log_slug}.out" 2>/dev/null || echo " Could not read PR log" + for shard in ${bench_shards:-""}; do tail -n 50 "pr/$(tree_slugs "$shard").out" 2>/dev/null || echo " Could not read PR log"; done # The PR benchmark run genuinely failed (cases crashed/hung/SIGTERM'd, not a # monitor false-positive -- run_monitored_slurm_job.sh re-checks sacct). Fail # the job instead of falling through to the YAML-exists check, which would let @@ -133,16 +178,19 @@ else echo "PR job completed successfully" fi - echo "" - echo "=== Monitoring master job $master_job_id ===" - monitor_bench_with_resubmit master - master_exit=$BENCH_MON_RC + monitor_tree master + master_exit=$BENCH_TREE_RC if [ "$master_exit" -ne 0 ]; then echo "Master job exited with code: $master_exit" - tail -n 50 "master/${log_slug}.out" 2>/dev/null || echo " Could not read master log" + for shard in ${bench_shards:-""}; do tail -n 50 "master/$(tree_slugs "$shard").out" 2>/dev/null || echo " Could not read master log"; done else echo "Master job completed successfully" fi + + # Only after both trees are known good: a merge over a missing shard file would fail + # here and mask the real cause reported above. + if [ "$pr_exit" -eq 0 ]; then merge_tree_shards pr; fi + if [ "$master_exit" -eq 0 ]; then merge_tree_shards master; fi fi # --- Phase 3: Verify outputs --- diff --git a/.github/workflows/common/bench.sh b/.github/workflows/common/bench.sh index 5d108a2a6..0abaf0161 100644 --- a/.github/workflows/common/bench.sh +++ b/.github/workflows/common/bench.sh @@ -54,10 +54,30 @@ else fi # --- Run benchmark --- +# $job_shard (i/N, from submit-slurm-job.sh) restricts this job to its share of the +# case list; the submitter merges the shards' YAMLs afterwards. Unset runs everything. +shard_opts="" +if [ -n "${job_shard:-}" ]; then + shard_opts="--shard $job_shard" +fi + +# DIAGNOSTIC A/B, Frontier only: since 2026-09-16 every bench case there takes ~3 min +# of solver time and then sits ~14 min in process exit before the next case starts +# (bench.py now prints the split). Darshan, which Frontier LD_PRELOADs into every MPI +# job and which flushes its log in MPI_Finalize, asserted in exactly that phase the +# same day (#1866). Shard 1 runs with Darshan disabled and shard 2 with it on, in the +# same allocation, so one job answers whether the stall is Darshan's. +if [ "$job_cluster" = "frontier" ] || [ "$job_cluster" = "frontier_amd" ]; then + case "${job_shard:-}" in + 1/*) export DARSHAN_DISABLE=1; echo "A/B: DARSHAN_DISABLE=1 for shard $job_shard" ;; + *) unset DARSHAN_DISABLE; echo "A/B: Darshan enabled for shard ${job_shard:-}" ;; + esac +fi + if [ "$job_device" = "gpu" ]; then - ./mfc.sh bench --mem 4 -o "$job_slug.yaml" -- -c $bench_cluster $device_opts -n $n_ranks + ./mfc.sh bench --mem 4 -o "$job_slug.yaml" $shard_opts -- -c $bench_cluster $device_opts -n $n_ranks else - ./mfc.sh bench --mem 1 -o "$job_slug.yaml" -- -c $bench_cluster $device_opts -n $n_ranks + ./mfc.sh bench --mem 1 -o "$job_slug.yaml" $shard_opts -- -c $bench_cluster $device_opts -n $n_ranks fi # --- Phoenix cleanup (trap EXIT handles rm -rf "$currentdir") --- diff --git a/toolchain/main.py b/toolchain/main.py index a2a0783fc..0a9659d1c 100644 --- a/toolchain/main.py +++ b/toolchain/main.py @@ -154,6 +154,10 @@ def __run(): from mfc import bench bench.diff() + elif cmd == "bench_merge": + from mfc import bench + + bench.merge() elif cmd == "count": from mfc import count diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py index 85a182742..703e5246d 100644 --- a/toolchain/mfc/bench.py +++ b/toolchain/mfc/bench.py @@ -43,6 +43,40 @@ def bench_failure_report(log_filepath: str) -> str: return summary or log_tail(log_filepath) +def _report_case_wall(slug: str, log_filepath: str, t_launch: float, t_returned: float) -> None: + """Print how a case's wall time splits between the run itself and its exit. + + mfc.sh run prints its own End-time; the difference between that and the moment + the child actually returned to this process is time spent after the solver was + done -- in MPI_Finalize, profiler log flushes (Darshan on Frontier), or srun step + teardown. That interval shows up nowhere else. + """ + import datetime + import re + + wall = t_returned - t_launch + ended = None + try: + with open(log_filepath, encoding="utf-8", errors="replace") as f: + for line in f: + m = re.search(r"End-time:\s+(\d\d):(\d\d):(\d\d)", line) + if m: + ended = tuple(int(x) for x in m.groups()) + except OSError: + pass + + if ended is None: + cons.print(f"> Wall: [bold]{wall:.0f}s[/bold] launch-to-return (no End-time found in log)") + return + + ret = datetime.datetime.fromtimestamp(t_returned) + end = ret.replace(hour=ended[0], minute=ended[1], second=ended[2], microsecond=0) + if end > ret: # End-time was before midnight, return after it + end -= datetime.timedelta(days=1) + after_exit = (ret - end).total_seconds() + cons.print(f"> Wall: [bold]{wall:.0f}s[/bold] launch-to-return, of which [bold]{after_exit:.0f}s[/bold] after the run printed its End-time (process exit / teardown)") + + def bench(targets=None): if targets is None: targets = ARG("targets") @@ -61,6 +95,21 @@ def bench(targets=None): CASES = [BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH)] + # Same round-robin as caseopt_case_in_shard in .github/scripts: shard i of N owns cases + # i, i+N, i+2N, ... (1-based), so every case lands in exactly one shard and the shards + # stay balanced as the list grows. Concurrent shards merge back into one results file + # for bench_diff; see run_parallel_benchmarks.sh. + shard = ARG("shard") + if shard is not None: + try: + shard_idx, shard_count = (int(x) for x in shard.split("/")) + except ValueError as exc: + raise MFCException(f"--shard must be i/N, got '{shard}'") from exc + if not 1 <= shard_idx <= shard_count: + raise MFCException(f"--shard must satisfy 1 <= i <= N, got '{shard}'") + CASES = [case for i, case in enumerate(CASES) if i % shard_count == shard_idx - 1] + cons.print(f"Shard {shard}: {len(CASES)} case(s): {', '.join(case.slug for case in CASES)}") + for case in CASES: case.args = case.args + ARG("--") case.path = os.path.abspath(case.path) @@ -91,6 +140,7 @@ def bench(targets=None): try: for attempt in range(1, max_attempts + 1): try: + t_launch = time.time() with open(log_filepath, "w") as log_file: result = system( ["./mfc.sh", "run", case.path] + ["--targets"] + [t.name for t in targets] + ["--output-summary", summary_filepath] + case.args + ["--", "--gbpp", str(ARG("mem"))], @@ -101,6 +151,13 @@ def bench(targets=None): # was previously reported as a bare address. env=fault_diagnostic_env(dict(os.environ)), ) + t_returned = time.time() + + # Where a case's wall time goes. On Frontier every case took ~3 min of solver + # time yet ~17 min of wall time, and nothing in any log covered the gap; the + # run's own End-time against when this call returns is what tells a slow solver + # from a process that has finished and is stuck exiting (MPI/profiler teardown). + _report_case_wall(case.slug, log_filepath, t_launch, t_returned) # Check return code (handle CompletedProcess or int defensively) rc = result.returncode if hasattr(result, "returncode") else result @@ -232,6 +289,50 @@ def _write_step_summary(lhs_path: str, rhs_path: str, rows: typing.List[typing.T cons.print(f"[bold yellow]Warning[/bold yellow]: could not write the benchmark step summary: {exc}") +def merge(): + """Fold the result files of concurrent bench shards back into the single file bench_diff reads. + + Each shard records its own invocation (argv differs by --shard and -o), and bench_diff compares + the two sides' metadata for equality before it compares timings, so the merged metadata keeps + the lock and takes the invocation of the first shard with the --shard/-o words removed. Anything + else that differs between shards -- which should be nothing, they run the same build -- is an + error, not something to paper over. + """ + inputs = ARG("inputs") + if not inputs: + raise MFCException("bench_merge needs at least one shard result file.") + + def _strip_shard_words(argv): + out, skip = [], False + for word in argv: + if skip: + skip = False + continue + if word in ("--shard", "-o", "--output"): + skip = True + continue + if word.startswith("--shard=") or word.startswith("--output="): + continue + out.append(word) + return out + + merged = None + for path in inputs: + shard = file_load_yaml(path) + meta = {"invocation": _strip_shard_words(shard["metadata"]["invocation"]), "lock": shard["metadata"]["lock"]} + if merged is None: + merged = {"metadata": meta, "cases": {}} + elif merged["metadata"] != meta: + raise MFCException(f"Shard {path} was not run the same way as {inputs[0]}: {meta} vs {merged['metadata']}.") + dup = set(merged["cases"]) & set(shard["cases"]) + if dup: + raise MFCException(f"Shard {path} repeats case(s) already merged: {', '.join(sorted(dup))}.") + merged["cases"].update(shard["cases"]) + + file_dump_yaml(ARG("output"), merged) + cons.print(f"Merged {len(inputs)} shard file(s), {len(merged['cases'])} case(s), into [magenta]{os.path.relpath(ARG('output'))}[/magenta].") + + def diff(): lhs, rhs = file_load_yaml(ARG("lhs")), file_load_yaml(ARG("rhs")) lhs_path = os.path.relpath(ARG("lhs")) diff --git a/toolchain/mfc/cli/commands.py b/toolchain/mfc/cli/commands.py index e177ec81f..453143661 100644 --- a/toolchain/mfc/cli/commands.py +++ b/toolchain/mfc/cli/commands.py @@ -844,13 +844,21 @@ default=1, metavar="MEM", ), + Argument( + name="shard", + help="Run only shard i of N of the benchmark list, as i/N (shard i owns cases i, i+N, ...). Lets clusters whose walltime cannot hold the full list split it across concurrent jobs.", + default=None, + metavar="i/N", + ), ], examples=[ Example("./mfc.sh bench -o results.yaml", "Run benchmarks and save results"), + Example("./mfc.sh bench -o shard1.yaml --shard 1/2", "Run the first half of the benchmark list"), ], key_options=[ ("-o, --output FILE", "Output file for benchmark results (required)"), ("-m, --mem SIZE", "Memory limit for benchmarks"), + ("--shard i/N", "Run only every Nth case starting at i"), ], ) @@ -865,6 +873,24 @@ ], ) +BENCH_MERGE_COMMAND = Command( + name="bench_merge", + help="Merge sharded MFC benchmark results into one file (for CI).", + include_common=["mfc_config", "jobs", "verbose", "debug_log"], + arguments=[ + Argument( + name="output", + short="o", + help="Path to the merged YAML output file.", + required=True, + metavar="OUTPUT", + ), + ], + positionals=[ + Positional(name="inputs", help="Shard result YAML files, as written by 'bench --shard'.", nargs="+"), + ], +) + COUNT_COMMAND = Command( name="count", help="Count LOC in MFC.", @@ -1465,6 +1491,7 @@ PRECHECK_COMMAND, BENCH_COMMAND, BENCH_DIFF_COMMAND, + BENCH_MERGE_COMMAND, COUNT_COMMAND, COUNT_DIFF_COMMAND, FP_STABILITY_COMMAND, diff --git a/toolchain/mfc/cli/docs_gen.py b/toolchain/mfc/cli/docs_gen.py index 531998d69..12242ddd6 100644 --- a/toolchain/mfc/cli/docs_gen.py +++ b/toolchain/mfc/cli/docs_gen.py @@ -243,7 +243,7 @@ def generate_cli_reference(schema: CLISchema) -> str: core_commands = ["build", "run", "test", "clean", "validate"] utility_commands = ["new", "viz", "params", "packer", "completion", "generate"] dev_commands = ["lint", "format", "spelling", "precheck", "count", "count_diff"] - ci_commands = ["bench", "bench_diff"] + ci_commands = ["bench", "bench_diff", "bench_merge"] other_commands = ["load"] # Core workflow commands first (no header, directly under Commands)