Shard the Frontier benchmark across two SLURM jobs so it fits the 2 h QOS cap

.github/scripts/run_parallel_benchmarks.sh

@@ -64,22 +64,43 @@ if [ "$device" = "gpu" ] && [ "$cluster" = "phoenix" ]; then
     master_exit=0
     echo "Single-node bench-pair job completed successfully."
 else
-    # --- Other clusters / Phoenix CPU: two concurrent jobs, monitored serially ---
+    # --- Other clusters / Phoenix CPU: concurrent jobs, monitored serially ---
     # The bench script must come from the PR tree (master may not have it).
     PR_BENCH_SCRIPT="$(cd "${SCRIPT_DIR}/../workflows/common" && pwd)/bench.sh"
 
-    # Phase 1: Submit both SLURM jobs (no monitoring yet)
-    echo "Submitting PR benchmark..."
-    (cd pr && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster")
-    pr_job_id=$(cat "pr/${log_slug}.slurm_job_id")
-    echo "PR job submitted: $pr_job_id"
+    # Frontier's "normal" QOS caps a job at 2 h, and one bench job there spends ~30 min
+    # building and ~17 min per case, so the 7-case list needs ~2.5 h and is killed on
+    # case 6 every time. Split each tree's run across concurrent shards (the same i/N
+    # scheme the case-optimization lanes use), each of which rebuilds and runs its
+    # share, then merge the shards' YAMLs so bench_diff still sees one file per tree.
+    # An empty shard means one unsharded job, which is what every other cluster gets.
+    case "$cluster" in
+        frontier|frontier_amd) bench_shards="1/2 2/2" ;;
+        *)                     bench_shards="" ;;
+    esac
+
+    # Per-tree list of the slugs whose .out/.slurm_job_id/.yaml this run produced, so
+    # the monitor and merge steps below need no knowledge of how many shards there are.
+    tree_slugs() {  # arg: <shard>; prints the job slug submit-slurm-job.sh will use
+        local shard="$1"
+        if [ -n "$shard" ]; then
+            echo "${log_slug}-$(echo "$shard" | sed 's|/|-of-|')"
+        else
+            echo "$log_slug"
+        fi
+    }
 
-    echo "Submitting master benchmark..."
-    (cd master && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster")
-    master_job_id=$(cat "master/${log_slug}.slurm_job_id")
-    echo "Master job submitted: $master_job_id"
+    # Phase 1: Submit every SLURM job (no monitoring yet)
+    for dir in pr master; do
+        for shard in ${bench_shards:-""}; do
+            slug=$(tree_slugs "$shard")
+            echo "Submitting ${dir} benchmark${shard:+ (shard $shard)}..."
+            (cd "$dir" && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster" "$shard")
+            echo "${dir} job submitted: $(cat "${dir}/${slug}.slurm_job_id")"
+        done
+    done
 
-    echo "Both SLURM jobs submitted — running concurrently on compute nodes."
+    echo "All SLURM jobs submitted — running concurrently on compute nodes."
     echo "Monitoring sequentially to conserve login node memory."
 
     # Phase 2: Monitor sequentially (one at a time on login node)
@@ -90,11 +111,12 @@ else
     # resubmitted job no longer overlaps its counterpart, slightly reducing
     # same-load fairness -- still preferable to failing the run on an infra preempt.
     : "${MAX_PREEMPT_RESUBMITS:=10}"
-    monitor_bench_with_resubmit() {  # arg: <dir> (pr|master); sets BENCH_MON_RC
-        local dir="$1"
-        local out="${dir}/${log_slug}.out"
-        local jobid attempt=0 rc
-        jobid=$(cat "${dir}/${log_slug}.slurm_job_id")
+    monitor_bench_with_resubmit() {  # args: <dir> (pr|master) <shard>; sets BENCH_MON_RC
+        local dir="$1" shard="$2"
+        local slug out jobid attempt=0 rc
+        slug=$(tree_slugs "$shard")
+        out="${dir}/${slug}.out"
+        jobid=$(cat "${dir}/${slug}.slurm_job_id")
         while :; do
             rc=0
             bash "${SCRIPT_DIR}/run_monitored_slurm_job.sh" "$jobid" "$out" || rc=$?
@@ -103,26 +125,49 @@ else
                 return
             fi
             if [ "$attempt" -ge "$MAX_PREEMPT_RESUBMITS" ]; then
-                echo "::error::${dir} benchmark preempted ${MAX_PREEMPT_RESUBMITS}x without completing; giving up."
+                echo "::error::${dir} benchmark${shard:+ shard $shard} preempted ${MAX_PREEMPT_RESUBMITS}x without completing; giving up."
                 BENCH_MON_RC=1
                 return
             fi
             attempt=$((attempt + 1))
             echo "::warning::${dir} benchmark job $jobid was preempted; resubmitting (attempt ${attempt}/${MAX_PREEMPT_RESUBMITS})."
             rm -f "$out"
-            ( cd "$dir" && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster" )
-            jobid=$(cat "${dir}/${log_slug}.slurm_job_id")
+            ( cd "$dir" && SUBMIT_ONLY=1 bash "${SCRIPT_DIR}/submit-slurm-job.sh" "$PR_BENCH_SCRIPT" "$device" "$interface" "$cluster" "$shard" )
+            jobid=$(cat "${dir}/${slug}.slurm_job_id")
             echo "${dir} benchmark resubmitted as job $jobid"
         done
     }
 
-    echo ""
-    echo "=== Monitoring PR job $pr_job_id ==="
-    monitor_bench_with_resubmit pr
-    pr_exit=$BENCH_MON_RC
+    # Monitor every shard of a tree; the tree's exit is the first non-zero one.
+    monitor_tree() {  # arg: <dir>; sets BENCH_TREE_RC
+        local dir="$1" shard
+        BENCH_TREE_RC=0
+        for shard in ${bench_shards:-""}; do
+            echo ""
+            echo "=== Monitoring ${dir} job $(cat "${dir}/$(tree_slugs "$shard").slurm_job_id")${shard:+ (shard $shard)} ==="
+            monitor_bench_with_resubmit "$dir" "$shard"
+            if [ "$BENCH_MON_RC" -ne 0 ] && [ "$BENCH_TREE_RC" -eq 0 ]; then
+                BENCH_TREE_RC="$BENCH_MON_RC"
+            fi
+        done
+    }
+
+    # Fold a tree's shard YAMLs into the one file bench_diff reads. Unsharded runs already
+    # wrote that file. Runs from the PR tree: master may predate bench_merge.
+    merge_tree_shards() {  # arg: <dir>
+        local dir="$1" shard inputs=""
+        [ -z "$bench_shards" ] && return 0
+        for shard in $bench_shards; do
+            inputs="$inputs ../${dir}/$(tree_slugs "$shard").yaml"
+        done
+        (cd pr && ./mfc.sh bench_merge -o "../${dir}/${job_slug}.yaml" $inputs)
+    }
+
+    monitor_tree pr
+    pr_exit=$BENCH_TREE_RC
     if [ "$pr_exit" -ne 0 ]; then
         echo "PR job exited with code: $pr_exit"
-        tail -n 50 "pr/${log_slug}.out" 2>/dev/null || echo "  Could not read PR log"
+        for shard in ${bench_shards:-""}; do tail -n 50 "pr/$(tree_slugs "$shard").out" 2>/dev/null || echo "  Could not read PR log"; done
         # The PR benchmark run genuinely failed (cases crashed/hung/SIGTERM'd, not a
         # monitor false-positive -- run_monitored_slurm_job.sh re-checks sacct). Fail
         # the job instead of falling through to the YAML-exists check, which would let
@@ -133,16 +178,19 @@ else
         echo "PR job completed successfully"
     fi
 
-    echo ""
-    echo "=== Monitoring master job $master_job_id ==="
-    monitor_bench_with_resubmit master
-    master_exit=$BENCH_MON_RC
+    monitor_tree master
+    master_exit=$BENCH_TREE_RC
     if [ "$master_exit" -ne 0 ]; then
         echo "Master job exited with code: $master_exit"
-        tail -n 50 "master/${log_slug}.out" 2>/dev/null || echo "  Could not read master log"
+        for shard in ${bench_shards:-""}; do tail -n 50 "master/$(tree_slugs "$shard").out" 2>/dev/null || echo "  Could not read master log"; done
     else
         echo "Master job completed successfully"
     fi
+
+    # Only after both trees are known good: a merge over a missing shard file would fail
+    # here and mask the real cause reported above.
+    if [ "$pr_exit" -eq 0 ]; then merge_tree_shards pr; fi
+    if [ "$master_exit" -eq 0 ]; then merge_tree_shards master; fi
 fi
 
 # --- Phase 3: Verify outputs ---

.github/workflows/common/bench.sh

@@ -54,10 +54,30 @@ else
 fi
 
 # --- Run benchmark ---
+# $job_shard (i/N, from submit-slurm-job.sh) restricts this job to its share of the
+# case list; the submitter merges the shards' YAMLs afterwards. Unset runs everything.
+shard_opts=""
+if [ -n "${job_shard:-}" ]; then
+    shard_opts="--shard $job_shard"
+fi
+
+# DIAGNOSTIC A/B, Frontier only: since 2026-09-16 every bench case there takes ~3 min
+# of solver time and then sits ~14 min in process exit before the next case starts
+# (bench.py now prints the split). Darshan, which Frontier LD_PRELOADs into every MPI
+# job and which flushes its log in MPI_Finalize, asserted in exactly that phase the
+# same day (#1866). Shard 1 runs with Darshan disabled and shard 2 with it on, in the
+# same allocation, so one job answers whether the stall is Darshan's.
+if [ "$job_cluster" = "frontier" ] || [ "$job_cluster" = "frontier_amd" ]; then
+    case "${job_shard:-}" in
+        1/*) export DARSHAN_DISABLE=1; echo "A/B: DARSHAN_DISABLE=1 for shard $job_shard" ;;
+        *)   unset DARSHAN_DISABLE;    echo "A/B: Darshan enabled for shard ${job_shard:-<unsharded>}" ;;
+    esac
+fi
+
 if [ "$job_device" = "gpu" ]; then
-    ./mfc.sh bench --mem 4 -o "$job_slug.yaml" -- -c $bench_cluster $device_opts -n $n_ranks
+    ./mfc.sh bench --mem 4 -o "$job_slug.yaml" $shard_opts -- -c $bench_cluster $device_opts -n $n_ranks
 else
-    ./mfc.sh bench --mem 1 -o "$job_slug.yaml" -- -c $bench_cluster $device_opts -n $n_ranks
+    ./mfc.sh bench --mem 1 -o "$job_slug.yaml" $shard_opts -- -c $bench_cluster $device_opts -n $n_ranks
 fi
 
 # --- Phoenix cleanup (trap EXIT handles rm -rf "$currentdir") ---

toolchain/main.py

@@ -154,6 +154,10 @@ def __run():
         from mfc import bench
 
         bench.diff()
+    elif cmd == "bench_merge":
+        from mfc import bench
+
+        bench.merge()
     elif cmd == "count":
         from mfc import count
 

toolchain/mfc/bench.py

@@ -43,6 +43,40 @@ def bench_failure_report(log_filepath: str) -> str:
     return summary or log_tail(log_filepath)
 
 
+def _report_case_wall(slug: str, log_filepath: str, t_launch: float, t_returned: float) -> None:
+    """Print how a case's wall time splits between the run itself and its exit.
+
+    mfc.sh run prints its own End-time; the difference between that and the moment
+    the child actually returned to this process is time spent after the solver was
+    done -- in MPI_Finalize, profiler log flushes (Darshan on Frontier), or srun step
+    teardown. That interval shows up nowhere else.
+    """
+    import datetime
+    import re
+
+    wall = t_returned - t_launch
+    ended = None
+    try:
+        with open(log_filepath, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                m = re.search(r"End-time:\s+(\d\d):(\d\d):(\d\d)", line)
+                if m:
+                    ended = tuple(int(x) for x in m.groups())
+    except OSError:
+        pass
+
+    if ended is None:
+        cons.print(f"> Wall:    [bold]{wall:.0f}s[/bold] launch-to-return (no End-time found in log)")
+        return
+
+    ret = datetime.datetime.fromtimestamp(t_returned)
+    end = ret.replace(hour=ended[0], minute=ended[1], second=ended[2], microsecond=0)
+    if end > ret:  # End-time was before midnight, return after it
+        end -= datetime.timedelta(days=1)
+    after_exit = (ret - end).total_seconds()
+    cons.print(f"> Wall:    [bold]{wall:.0f}s[/bold] launch-to-return, of which [bold]{after_exit:.0f}s[/bold] after the run printed its End-time (process exit / teardown)")
+
+
 def bench(targets=None):
     if targets is None:
         targets = ARG("targets")
@@ -61,6 +95,21 @@ def bench(targets=None):
 
         CASES = [BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH)]
 
+        # Same round-robin as caseopt_case_in_shard in .github/scripts: shard i of N owns cases
+        # i, i+N, i+2N, ... (1-based), so every case lands in exactly one shard and the shards
+        # stay balanced as the list grows. Concurrent shards merge back into one results file
+        # for bench_diff; see run_parallel_benchmarks.sh.
+        shard = ARG("shard")
+        if shard is not None:
+            try:
+                shard_idx, shard_count = (int(x) for x in shard.split("/"))
+            except ValueError as exc:
+                raise MFCException(f"--shard must be i/N, got '{shard}'") from exc
+            if not 1 <= shard_idx <= shard_count:
+                raise MFCException(f"--shard must satisfy 1 <= i <= N, got '{shard}'")
+            CASES = [case for i, case in enumerate(CASES) if i % shard_count == shard_idx - 1]
+            cons.print(f"Shard {shard}: {len(CASES)} case(s): {', '.join(case.slug for case in CASES)}")
+
         for case in CASES:
             case.args = case.args + ARG("--")
             case.path = os.path.abspath(case.path)
@@ -91,6 +140,7 @@ def bench(targets=None):
             try:
                 for attempt in range(1, max_attempts + 1):
                     try:
+                        t_launch = time.time()
                         with open(log_filepath, "w") as log_file:
                             result = system(
                                 ["./mfc.sh", "run", case.path] + ["--targets"] + [t.name for t in targets] + ["--output-summary", summary_filepath] + case.args + ["--", "--gbpp", str(ARG("mem"))],
@@ -101,6 +151,13 @@ def bench(targets=None):
                                 # was previously reported as a bare address.
                                 env=fault_diagnostic_env(dict(os.environ)),
                             )
+                        t_returned = time.time()
+
+                        # Where a case's wall time goes. On Frontier every case took ~3 min of solver
+                        # time yet ~17 min of wall time, and nothing in any log covered the gap; the
+                        # run's own End-time against when this call returns is what tells a slow solver
+                        # from a process that has finished and is stuck exiting (MPI/profiler teardown).
+                        _report_case_wall(case.slug, log_filepath, t_launch, t_returned)
 
                         # Check return code (handle CompletedProcess or int defensively)
                         rc = result.returncode if hasattr(result, "returncode") else result
@@ -232,6 +289,50 @@ def _write_step_summary(lhs_path: str, rhs_path: str, rows: typing.List[typing.T
         cons.print(f"[bold yellow]Warning[/bold yellow]: could not write the benchmark step summary: {exc}")
 
 
+def merge():
+    """Fold the result files of concurrent bench shards back into the single file bench_diff reads.
+
+    Each shard records its own invocation (argv differs by --shard and -o), and bench_diff compares
+    the two sides' metadata for equality before it compares timings, so the merged metadata keeps
+    the lock and takes the invocation of the first shard with the --shard/-o words removed. Anything
+    else that differs between shards -- which should be nothing, they run the same build -- is an
+    error, not something to paper over.
+    """
+    inputs = ARG("inputs")
+    if not inputs:
+        raise MFCException("bench_merge needs at least one shard result file.")
+
+    def _strip_shard_words(argv):
+        out, skip = [], False
+        for word in argv:
+            if skip:
+                skip = False
+                continue
+            if word in ("--shard", "-o", "--output"):
+                skip = True
+                continue
+            if word.startswith("--shard=") or word.startswith("--output="):
+                continue
+            out.append(word)
+        return out
+
+    merged = None
+    for path in inputs:
+        shard = file_load_yaml(path)
+        meta = {"invocation": _strip_shard_words(shard["metadata"]["invocation"]), "lock": shard["metadata"]["lock"]}
+        if merged is None:
+            merged = {"metadata": meta, "cases": {}}
+        elif merged["metadata"] != meta:
+            raise MFCException(f"Shard {path} was not run the same way as {inputs[0]}: {meta} vs {merged['metadata']}.")
+        dup = set(merged["cases"]) & set(shard["cases"])
+        if dup:
+            raise MFCException(f"Shard {path} repeats case(s) already merged: {', '.join(sorted(dup))}.")
+        merged["cases"].update(shard["cases"])
+
+    file_dump_yaml(ARG("output"), merged)
+    cons.print(f"Merged {len(inputs)} shard file(s), {len(merged['cases'])} case(s), into [magenta]{os.path.relpath(ARG('output'))}[/magenta].")
+
+
 def diff():
     lhs, rhs = file_load_yaml(ARG("lhs")), file_load_yaml(ARG("rhs"))
     lhs_path = os.path.relpath(ARG("lhs"))

toolchain/mfc/cli/commands.py

@@ -844,13 +844,21 @@
             default=1,
             metavar="MEM",
         ),
+        Argument(
+            name="shard",
+            help="Run only shard i of N of the benchmark list, as i/N (shard i owns cases i, i+N, ...). Lets clusters whose walltime cannot hold the full list split it across concurrent jobs.",
+            default=None,
+            metavar="i/N",
+        ),
     ],
     examples=[
         Example("./mfc.sh bench -o results.yaml", "Run benchmarks and save results"),
+        Example("./mfc.sh bench -o shard1.yaml --shard 1/2", "Run the first half of the benchmark list"),
     ],
     key_options=[
         ("-o, --output FILE", "Output file for benchmark results (required)"),
         ("-m, --mem SIZE", "Memory limit for benchmarks"),
+        ("--shard i/N", "Run only every Nth case starting at i"),
     ],
 )
 
@@ -865,6 +873,24 @@
     ],
 )
 
+BENCH_MERGE_COMMAND = Command(
+    name="bench_merge",
+    help="Merge sharded MFC benchmark results into one file (for CI).",
+    include_common=["mfc_config", "jobs", "verbose", "debug_log"],
+    arguments=[
+        Argument(
+            name="output",
+            short="o",
+            help="Path to the merged YAML output file.",
+            required=True,
+            metavar="OUTPUT",
+        ),
+    ],
+    positionals=[
+        Positional(name="inputs", help="Shard result YAML files, as written by 'bench --shard'.", nargs="+"),
+    ],
+)
+
 COUNT_COMMAND = Command(
     name="count",
     help="Count LOC in MFC.",
@@ -1465,6 +1491,7 @@
         PRECHECK_COMMAND,
         BENCH_COMMAND,
         BENCH_DIFF_COMMAND,
+        BENCH_MERGE_COMMAND,
         COUNT_COMMAND,
         COUNT_DIFF_COMMAND,
         FP_STABILITY_COMMAND,

toolchain/mfc/cli/docs_gen.py

@@ -243,7 +243,7 @@ def generate_cli_reference(schema: CLISchema) -> str:
     core_commands = ["build", "run", "test", "clean", "validate"]
     utility_commands = ["new", "viz", "params", "packer", "completion", "generate"]
     dev_commands = ["lint", "format", "spelling", "precheck", "count", "count_diff"]
-    ci_commands = ["bench", "bench_diff"]
+    ci_commands = ["bench", "bench_diff", "bench_merge"]
     other_commands = ["load"]
 
     # Core workflow commands first (no header, directly under Commands)