From 907742307a076ee5966ce5fdfe13eabc25134939 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Sat, 21 Nov 2026 12:40:55 -0300 Subject: [PATCH 1/5] ci(perf-gate): cumulative budget against the last release tag The gate compared each PR with its merge base only, so regressions under the 3% threshold could add up across a release cycle. The same job now also measures the last v* tag with the same driver and harness and fails when any gated benchmark grows more than 3% over it. The table shows release, base and head with both deltas. The release binary is cached per release commit, driver and toolchain, warmed on pushes to main. Callgrind runs are pinned to one CPU, and threads_request_seq takes the median of nine run pairs: under load the host scheduler moved it by up to 3.7%, as much as the threshold. --- .github/workflows/perf-gate.yml | 39 +++++- CHANGELOG.md | 9 ++ benchmarks/README.md | 4 +- benchmarks/hotpath.py | 209 +++++++++++++++++++++++++++----- docs/TESTING.md | 43 +++++-- 5 files changed, 259 insertions(+), 45 deletions(-) diff --git a/.github/workflows/perf-gate.yml b/.github/workflows/perf-gate.yml index 17c948339..18f7e7ff1 100644 --- a/.github/workflows/perf-gate.yml +++ b/.github/workflows/perf-gate.yml @@ -1,6 +1,7 @@ name: Perf gate -# Hot-path instruction counts, PR against its merge base +# Hot-path instruction counts, PR against its merge base and against the last +# release tag, a cumulative budget per release cycle # (benchmarks/hotpath.py gate; docs/TESTING.md, "Performance gate"). # The workflow always runs so the check can be required; the gate itself is # skipped (and passes) unless runtime source, dependencies, toolchain or driver changed. @@ -9,6 +10,10 @@ on: pull_request: branches: ["main"] merge_group: + # On main only the release binary is built (when missing) and cached, so PRs, + # which can only restore caches saved on main, reuse it. + push: + branches: ["main"] concurrency: group: perf-gate-${{ github.event.pull_request.number || github.ref }} @@ -19,11 +24,13 @@ env: CARGO_INCREMENTAL: "0" # Maximum growth of instructions per operation, in percent. HOTPATH_THRESHOLD: "3" + # Maximum growth over the last release (git describe --match 'v*'), in percent. + HOTPATH_RELEASE_THRESHOLD: "3" jobs: perf-gate: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 50 steps: - uses: actions/checkout@v4 with: @@ -34,6 +41,14 @@ jobs: PR_BASE: ${{ github.event.pull_request.base.sha }} MQ_BASE: ${{ github.event.merge_group.base_sha }} run: | + release=$(git describe --tags --abbrev=0 --match 'v*' HEAD) + echo "release=$release" >> "$GITHUB_OUTPUT" + echo "release_sha=$(git rev-parse "$release^{commit}")" >> "$GITHUB_OUTPUT" + if [ "${{ github.event_name }}" = push ]; then + echo "run=false" >> "$GITHUB_OUTPUT" + echo "warm=true" >> "$GITHUB_OUTPUT" + exit 0 + fi base=$(git merge-base HEAD "${PR_BASE:-$MQ_BASE}") echo "base=$base" >> "$GITHUB_OUTPUT" changed=$(git diff --name-only "$base" HEAD) @@ -44,13 +59,29 @@ jobs: echo "Perf gate skipped: no runtime source, dependency, toolchain or driver change." >> "$GITHUB_STEP_SUMMARY" fi - uses: dtolnay/rust-toolchain@1.93.0 - if: steps.scope.outputs.run == 'true' + if: steps.scope.outputs.run == 'true' || steps.scope.outputs.warm == 'true' + # The release binary: same driver, same toolchain, same release commit. + - name: Release binary cache + if: steps.scope.outputs.run == 'true' || steps.scope.outputs.warm == 'true' + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/hotpath-release + key: hotpath-release-${{ steps.scope.outputs.release_sha }}-${{ hashFiles('benchmarks/hotpath/driver.rs', 'rust-toolchain.toml') }} + - name: Build the release binary + if: steps.scope.outputs.warm == 'true' + run: > + python3 benchmarks/hotpath.py build-release + --release ${{ steps.scope.outputs.release }} + --release-cache ${{ runner.temp }}/hotpath-release + --work ${{ runner.temp }}/hotpath - name: Install valgrind if: steps.scope.outputs.run == 'true' run: sudo apt-get update -q && sudo apt-get install -y -q valgrind - - name: Instruction counts, head vs merge base + - name: Instruction counts, head vs merge base and last release if: steps.scope.outputs.run == 'true' run: > python3 benchmarks/hotpath.py gate --base ${{ steps.scope.outputs.base }} --head HEAD + --release ${{ steps.scope.outputs.release }} + --release-cache ${{ runner.temp }}/hotpath-release --work ${{ runner.temp }}/hotpath diff --git a/CHANGELOG.md b/CHANGELOG.md index c8eb616f7..a4d54d283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,15 @@ non-blocking disposal and checks the queue drains and every worker retires after its keep-alive (`disposal.keep_alive` failpoint with `failpoints`). The soak harness judges the three gauges. +- `perf-gate` also holds every PR to a cumulative budget: the head may grow + at most 3% instructions per operation over the last release tag + (`git describe --match 'v*'`), not only over its merge base, so small + per-PR regressions cannot add up across a release cycle. The job summary + table shows release, base and head with both changes. The release binary + is built once per release and cached (`hotpath.py build-release` on pushes + to main). Callgrind runs are pinned to one CPU and + `threads_request_seq` takes the median of nine run pairs: under load the + host scheduler had moved it by up to 3.7%, as much as the threshold. - Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup` (a 20 ms startup deadline raced the fast starts on a loaded host, which also made cargo-mutants report surviving mutants as caught): the replacement's diff --git a/benchmarks/README.md b/benchmarks/README.md index daafe2144..5e01e5bce 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -56,7 +56,9 @@ python3 benchmarks/hotpath.py abba --cores 12-15 --reps 30 # wall clock, before ``` `gate` is the `perf-gate` CI check: it fails when instructions per operation -grow more than 3% over the merge base. `abba` alternates base and head on the +grow more than 3% over the merge base (callgrind pinned to one CPU), or more than 3% over the last release +tag (`--release`; the cumulative budget per release cycle, so small per-PR +regressions cannot add up). `abba` alternates base and head on the same cores and reports the median and spread. See [performance gate](../docs/TESTING.md#performance-gate). diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index c7bfc0be7..b9e4dfb71 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -2,6 +2,8 @@ """Hot-path regression checks: a revision against its base. hotpath.py gate [--base REV] [--head REV] [--threshold PCT] [--features F] + [--release REV] [--release-cache DIR] + hotpath.py build-release --release REV --release-cache DIR [--features F] hotpath.py abba [--base REV] [--head REV] [--reps N] [--cores LIST] [--features F] Both build `benchmarks/hotpath/driver.rs` (the head's copy, in both trees, so @@ -10,14 +12,25 @@ `gate` counts instructions with callgrind, which is deterministic enough for noisy CI runners. Each benchmark runs at two sizes; instructions per operation is the difference divided by the extra operations, so process start, runtime -and actor setup cancel. It fails when any benchmark grows by more than +and actor setup cancel; the median over `--reps` run pairs (`--contended-reps` for the +two-thread benchmark). Every run is pinned to one CPU, so the host scheduler +cannot change how the threads interleave (see `instructions`). It fails when any benchmark grows by more than `--threshold` percent (default 3). It is the CI check. +With `--release` (CI passes the last `v*` tag), `gate` also measures that +release and fails when any benchmark grows by more than `--release-threshold` +percent (default: `--threshold`) over it, so regressions that each pass the +per-PR check cannot add up across a release cycle. `--release-cache DIR` keeps +the release binary there, keyed by release commit, driver and toolchain, so +it is built once per release rather than per PR; `build-release` only fills +that cache (CI runs it on every push to main, where PRs can restore it from). + `abba` measures wall-clock throughput, alternating base and head on the same cores (ABBA order), and reports the median and spread. Run it on a quiet machine before a release; it is never run in CI. """ import argparse +import hashlib import os import re import shutil @@ -51,6 +64,7 @@ def default_base(head): def build(work, label, rev, features=None): """Check out `rev` in a worktree, add the driver, build it; return the binary.""" tree = work / label + work.mkdir(parents=True, exist_ok=True) sh(["git", "worktree", "add", "--detach", "--force", str(tree), rev], cwd=ROOT, capture_output=True) shutil.copy(DRIVER, tree / "concurrency" / "examples" / "hotpath.rs") @@ -64,6 +78,33 @@ def build(work, label, rev, features=None): return binary +def release_key(rev, features=None): + """Cache key of a release binary: its commit, the driver and the compiler.""" + driver = hashlib.sha256(DRIVER.read_bytes()).hexdigest()[:16] + rustc = sh(["rustc", "-V"], cwd=ROOT, capture_output=True).stdout + toolchain = hashlib.sha256(rustc.encode()).hexdigest()[:16] + commit = git("rev-parse", f"{rev}^{{commit}}") # a tag's commit, not the tag object + extra = "-" + re.sub(r"[^A-Za-z0-9]+", "_", features) if features else "" + return f"hotpath-{commit[:16]}-{driver}-{toolchain}{extra}" + + +def build_release(work, rev, cache, features=None): + """The release binary, from `cache` when an identical build is there.""" + if not cache: + return build(work, "release", rev, features) + cache = Path(cache).resolve() + cached = cache / release_key(rev, features) + if cached.exists(): + print(f"release binary from cache: {cached}", file=sys.stderr) + return cached + binary = build(work, "release", rev, features) + cache.mkdir(parents=True, exist_ok=True) + for stale in cache.glob("hotpath-*"): + stale.unlink() + shutil.copy(binary, cached) + return cached + + def benchmarks(binary, only, gate=False): """The driver's benchmarks; the gate takes only the deterministic ones.""" names = sh([str(binary), "list-gate" if gate else "list"], capture_output=True).stdout.split() @@ -73,22 +114,52 @@ def benchmarks(binary, only, gate=False): IR = re.compile(r"Collected\s*:\s*(\d+)") +def one_cpu(): + """Pin the measured process to one CPU (see `instructions`).""" + if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(0, {max(os.sched_getaffinity(0))}) + + def instructions(binary, name, ops): + # Valgrind runs one guest thread at a time, but a thread blocked in a + # syscall (futex wait, park) releases that lock, so on several CPUs the + # host scheduler decides how the requester and the actor interleave and + # how often they spin, park and wake: up to 3.7% instructions per + # operation under load on `threads_request_seq`. On one CPU the + # interleaving barely depends on the host (0.9%); the code paths measured + # are the same. out = subprocess.run( ["valgrind", "--tool=callgrind", "--fair-sched=yes", "--callgrind-out-file=/dev/null", str(binary), name, str(ops)], - text=True, capture_output=True, timeout=180) + text=True, capture_output=True, timeout=180, preexec_fn=one_cpu) if out.returncode != 0: sys.exit(f"{binary.name} {name} {ops} failed:\n{out.stderr[-3000:]}") return int(IR.search(out.stderr).group(1)) -def per_op(binary, name, small, large, reps): - samples = [ - (instructions(binary, name, large) - instructions(binary, name, small)) / (large - small) - for _ in range(reps) - ] - return statistics.median(samples), samples +def contended(name): + """Benchmarks whose count varies with thread interleaving (not `async_ct_*`).""" + return not name.startswith("async_ct_") + + +def per_op(binary, name, small, large, reps, csv=None, label=""): + """Instructions per operation: the median over run pairs of + (instructions at `large` - at `small`) / extra operations, and the pairs. + + The two-thread benchmark gets more pairs (`--contended-reps`). The minimum + is not a stable estimate here: the noise is in both sizes, so the + difference of the minima moved 3.3% across ten loaded runs, against 1.4% + for the median. + """ + runs = {small: [], large: []} + for rep in range(reps): + for size in (small, large): + ir = instructions(binary, name, size) + runs[size].append(ir) + if csv: + csv.write(f"{label},{name},{size},{rep},{ir}\n") + pairs = [(lg - sm) / (large - small) for sm, lg in zip(runs[small], runs[large])] + return statistics.median(pairs), pairs def report(lines): @@ -100,36 +171,75 @@ def report(lines): f.write(text) -def gate(args, base_bin, head_bin, base, head): +def gate(args, base_bin, head_bin, base, head, release_bin=None, release=None): names = benchmarks(head_bin, args.only, gate=True) - rows, failed = [], [] + rows, failed, failed_release = [], [], [] + csv = open(args.csv, "w") if args.csv else None + if csv: + csv.write("side,benchmark,ops,rep,instructions\n") for name in names: - b, bs = per_op(base_bin, name, args.small, args.large, args.reps) - h, hs = per_op(head_bin, name, args.small, args.large, args.reps) + reps = args.contended_reps if contended(name) else args.reps + b, bs = per_op(base_bin, name, args.small, args.large, reps, csv, "base") + h, hs = per_op(head_bin, name, args.small, args.large, reps, csv, "head") delta = 100 * (h / b - 1) bad = delta > args.threshold if bad: failed.append(name) spread = max(max(bs) - min(bs), max(hs) - min(hs)) - rows.append(f"| `{name}` | {b:,.0f} | {h:,.0f} | {delta:+.1f}% | ±{spread:,.0f} | " - f"{'FAIL' if bad else 'ok'} |") - print(f"{name}: base {b:.0f} head {h:.0f} ({delta:+.1f}%)", file=sys.stderr) - verdict = (f"**FAIL**: {', '.join(failed)} grew more than {args.threshold}%" - if failed else f"**PASS**: no benchmark grew more than {args.threshold}%") + if release_bin is None: + rows.append(f"| `{name}` | {b:,.0f} | {h:,.0f} | {delta:+.1f}% | {spread:,.0f} | " + f"{'FAIL' if bad else 'ok'} |") + print(f"{name}: base {b:.0f} head {h:.0f} ({delta:+.1f}%)", file=sys.stderr) + continue + if release_bin == base_bin: + r, rs = b, bs + else: + r, rs = per_op(release_bin, name, args.small, args.large, reps, csv, "release") + rdelta = 100 * (h / r - 1) + rbad = rdelta > args.release_threshold + if rbad: + failed_release.append(name) + spread = max(spread, max(rs) - min(rs)) + result = ("FAIL" if bad else "ok") + " / " + ("FAIL" if rbad else "ok") + rows.append(f"| `{name}` | {r:,.0f} | {b:,.0f} | {h:,.0f} | {delta:+.1f}% | " + f"{rdelta:+.1f}% | {spread:,.0f} | {result} |") + print(f"{name}: release {r:.0f} base {b:.0f} head {h:.0f} " + f"({delta:+.1f}% vs base, {rdelta:+.1f}% vs release)", file=sys.stderr) + verdict = [f"**FAIL**: {', '.join(failed)} grew more than {args.threshold}% over the base" + if failed else f"**PASS**: no benchmark grew more than {args.threshold}% over the base"] + if release_bin is None: + header = ["| benchmark | base Ir/op | head Ir/op | change | raw spread | result |", + "|---|---:|---:|---:|---:|---|"] + revisions = f"Base `{base[:10]}`, head `{head[:10]}`." + thresholds = f"threshold +{args.threshold}%" + else: + header = ["| benchmark | release Ir/op | base Ir/op | head Ir/op | Δ vs base | " + "Δ vs release | raw spread | result (base / release) |", + "|---|---:|---:|---:|---:|---:|---:|---|"] + revisions = (f"Release `{args.release}` (`{release[:10]}`), base `{base[:10]}`, " + f"head `{head[:10]}`.") + thresholds = (f"thresholds +{args.threshold}% over the base, " + f"+{args.release_threshold}% over the release") + verdict.append( + f"**FAIL**: {', '.join(failed_release)} grew more than {args.release_threshold}% " + f"since `{args.release}` (cumulative budget; see docs/TESTING.md, " + "\"Performance gate\")" if failed_release else + f"**PASS**: no benchmark grew more than {args.release_threshold}% " + f"since `{args.release}`") report([ "## Hot-path instruction counts", "", - f"Base `{base[:10]}`, head `{head[:10]}`. Instructions per operation " - f"(callgrind, sizes {args.small} and {args.large}, median of {args.reps}); " - f"threshold +{args.threshold}%.", + f"{revisions} Instructions per operation " + f"(callgrind, sizes {args.small} and {args.large}, median of {args.reps} run " + f"pairs, {args.contended_reps} for the two-thread `threads_*`, one CPU); " + f"{thresholds}.", "", - "| benchmark | base Ir/op | head Ir/op | change | max spread | result |", - "|---|---:|---:|---:|---:|---|", + *header, *rows, "", - verdict, + *(f"{v} " for v in verdict), ]) - return 1 if failed else 0 + return 1 if failed or failed_release else 0 def wallclock(binary, name, ops, cores): @@ -180,7 +290,7 @@ def abba(args, base_bin, head_bin, base, head): def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("mode", choices=["gate", "abba"]) + p.add_argument("mode", choices=["gate", "abba", "build-release"]) p.add_argument("--head", default="HEAD") p.add_argument("--base", help="default: merge base of --head and main") p.add_argument("--only", nargs="*", help="benchmark names (default: all)") @@ -188,28 +298,63 @@ def main(): p.add_argument("--threshold", type=float, default=float(os.environ.get("HOTPATH_THRESHOLD", "3")), help="gate: max growth in percent (default 3, or $HOTPATH_THRESHOLD)") + p.add_argument("--release", help="gate: also hold the head to this release (a tag)") + p.add_argument("--release-threshold", type=float, + default=(float(os.environ["HOTPATH_RELEASE_THRESHOLD"]) + if os.environ.get("HOTPATH_RELEASE_THRESHOLD") else None), + help="gate: max growth over --release in percent " + "(default --threshold, or $HOTPATH_RELEASE_THRESHOLD)") + p.add_argument("--release-cache", help="gate: directory that keeps the release binary") p.add_argument("--small", type=int, default=2000, help="gate: smaller size") p.add_argument("--large", type=int, default=6000, help="gate: larger size") p.add_argument("--reps", type=int, help="gate: runs per size (default 3); abba: repetitions (default 20)") p.add_argument("--ops", type=int, default=500_000, help="abba: operations per run") p.add_argument("--cores", help="abba: taskset core list, e.g. 12-15") - p.add_argument("--csv", help="abba: raw samples") - p.add_argument("--features", help="cargo features for both sides, e.g. metrics " - "(default: the crate's defaults, as CI gates)") + p.add_argument("--contended-reps", type=int, default=9, + help="gate: runs per size for benchmarks with more than one thread " + "(default 9)") + p.add_argument("--csv", help="raw samples") args = p.parse_args() args.reps = args.reps or (3 if args.mode == "gate" else 20) - + if args.release_threshold is None: + args.release_threshold = args.threshold + if args.release and args.mode == "abba": + p.error("--release is a gate option") + if args.mode == "build-release" and not (args.release and args.release_cache): + p.error("build-release needs --release and --release-cache") + + if args.mode == "build-release": + work = Path(args.work or tempfile.mkdtemp(prefix="hotpath-")).resolve() + try: + print(build_release(work, args.release, args.release_cache, args.features)) + finally: + subprocess.run(["git", "worktree", "remove", "--force", str(work / "release")], + cwd=ROOT, capture_output=True) + if not args.work: + shutil.rmtree(work, ignore_errors=True) + return head = git("rev-parse", args.head) base = git("rev-parse", args.base) if args.base else default_base(head) + release = git("rev-parse", f"{args.release}^{{commit}}") if args.release else None work = Path(args.work or tempfile.mkdtemp(prefix="hotpath-")).resolve() work.mkdir(parents=True, exist_ok=True) try: base_bin = build(work, "base", base, args.features) head_bin = build(work, "head", head, args.features) - run = gate if args.mode == "gate" else abba - sys.exit(run(args, base_bin, head_bin, base, head)) + if args.mode == "abba": + sys.exit(abba(args, base_bin, head_bin, base, head)) + release_bin = None + if release: + try: + release_bin = (base_bin if release == base + else build_release(work, release, args.release_cache, args.features)) + except subprocess.CalledProcessError: + sys.exit(f"the driver does not build against {args.release}: " + "benchmarks/hotpath/driver.rs must only use APIs that the last " + "release also has") + sys.exit(gate(args, base_bin, head_bin, base, head, release_bin, release)) finally: - for label in ("base", "head"): + for label in ("base", "head", "release"): subprocess.run(["git", "worktree", "remove", "--force", str(work / label)], cwd=ROOT, capture_output=True) if not args.work: diff --git a/docs/TESTING.md b/docs/TESTING.md index 6848b1728..79d649e97 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2101,18 +2101,44 @@ across releases (`#[actor]` handlers, `request()`, `send()`). **CI: instruction counts** (`perf-gate` workflow, every PR and the merge queue). Wall-clock numbers on shared runners are too noisy, so the gate counts instructions with callgrind. Each benchmark runs at two sizes (2,000 and 6,000 -operations, three runs each); instructions per operation is the difference -divided by the extra operations, so process start, runtime and actor setup -cancel. Only deterministic benchmarks are gated: the async ones on a Tokio -current-thread runtime (`async_ct_*`: burst, sequential and 16-task -`request()`, `send` from one and two producer tasks; identical counts run to -run) and threads-mode sequential `request()` (a lockstep pair of threads, -within 0.5%). On a multi-threaded runtime, or with free-running OS-thread +operations); instructions per operation is the difference divided by the +extra operations, so process start, runtime and actor setup cancel, and the +gate takes the median over run pairs. Only deterministic benchmarks are +gated: the async ones on a Tokio current-thread runtime (`async_ct_*`: burst, +sequential and 16-task `request()`, `send` from one and two producer tasks; +identical counts run to run, three pairs) and threads-mode sequential +`request()` (a lockstep pair of threads, nine pairs). Valgrind runs one guest +thread at a time, but a thread blocked in a syscall lets the other run, so +with several CPUs the host scheduler decides how the requester and the actor +interleave, and how often they spin, park and wake. Under load that moved +`threads_request_seq` by up to 3.7% between runs, as much as the threshold. +Every callgrind run is therefore pinned to one CPU, which leaves the code +paths unchanged and cuts that spread to 0.9%. A minimum over runs (on the +theory that contention only adds instructions) was tried and rejected: the +noise is in both sizes, so the difference of the minima moved 3.3% across +ten loaded runs. On a multi-threaded runtime, or with free-running OS-thread producers (threads-mode burst and `send`), how often actors park and wake changes from run to run (up to ±20% instructions), so those run only in the wall-clock check. The gate fails when any benchmark's instructions per operation grow more than 3% over the merge base (`--threshold`, or `HOTPATH_THRESHOLD` in the -workflow). The job summary has the per-benchmark table. The workflow always +workflow). + +A per-PR budget alone lets regressions add up: ten PRs at +2.5% each all pass. +So the same job also measures the last release tag (`git describe --tags +--abbrev=0 --match 'v*'`) with the same driver and harness, and fails when +any benchmark grows more than 3% over that release (`--release`, +`--release-threshold`, or `HOTPATH_RELEASE_THRESHOLD`): the cumulative budget +for a release cycle. A PR that is under 3% on its own can still fail because +of earlier PRs; then the fix is to win back instructions on that path, not to +raise the threshold. A deliberate cost (a new guarantee on the request path) +is argued in the PR and the release budget is reset only by tagging the next +release. The release binary is built once per release, driver and toolchain: +pushes to main build it into the Actions cache (`hotpath.py build-release`), +and PRs restore it. The release is built with the default features, which +every release has; features removed since (such as the `*-experiments` ones +in 0.9) are not used by the driver. The job summary has the per-benchmark +table: release, base and head instructions per operation, the change against +the base and against the release. The workflow always reports, so it can be a required check, but it only builds and measures when `concurrency/src`, `rt/src`, `macros/src`, the driver, a Cargo manifest or lockfile, or the pinned Rust toolchain changed. Release version bumps run the @@ -2122,6 +2148,7 @@ gate too; they cannot inherit a passing source check without measurement. python3 benchmarks/hotpath.py gate # HEAD vs merge base with main python3 benchmarks/hotpath.py gate --base origin/main --head origin/my-branch --threshold 3 python3 benchmarks/hotpath.py gate --features metrics # both sides with the metrics feature +python3 benchmarks/hotpath.py gate --release v0.8.0 # also the cumulative check, as in CI ``` CI gates the default build. `--features` builds both sides with extra From b759ff9e4310003c485675f55bf3700cf0e226e4 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Sat, 21 Nov 2026 12:41:55 -0300 Subject: [PATCH 2/5] ci(perf-gate): keep --features after the rebase --- benchmarks/hotpath.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index b9e4dfb71..f9c30374f 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -314,6 +314,8 @@ def main(): help="gate: runs per size for benchmarks with more than one thread " "(default 9)") p.add_argument("--csv", help="raw samples") + p.add_argument("--features", help="cargo features for every side, e.g. metrics " + "(default: the crate's defaults, as CI gates)") args = p.parse_args() args.reps = args.reps or (3 if args.mode == "gate" else 20) if args.release_threshold is None: @@ -347,7 +349,8 @@ def main(): if release: try: release_bin = (base_bin if release == base - else build_release(work, release, args.release_cache, args.features)) + else build_release(work, release, args.release_cache, + args.features)) except subprocess.CalledProcessError: sys.exit(f"the driver does not build against {args.release}: " "benchmarks/hotpath/driver.rs must only use APIs that the last " From 39f2c17282bac5849eab355ad5d7b0cff01b79a2 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Sat, 21 Nov 2026 12:57:32 -0300 Subject: [PATCH 3/5] docs(perf-gate): noise numbers from ten pinned gates under load on fede --- CHANGELOG.md | 2 +- benchmarks/hotpath.py | 11 +++++------ docs/TESTING.md | 17 ++++++++++------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d54d283..4c7895d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,7 @@ is built once per release and cached (`hotpath.py build-release` on pushes to main). Callgrind runs are pinned to one CPU and `threads_request_seq` takes the median of nine run pairs: under load the - host scheduler had moved it by up to 3.7%, as much as the threshold. + host scheduler had swung it by several percent, as much as the threshold. - Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup` (a 20 ms startup deadline raced the fast starts on a loaded host, which also made cargo-mutants report surviving mutants as caught): the replacement's diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index f9c30374f..cddeaea40 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -124,10 +124,10 @@ def instructions(binary, name, ops): # Valgrind runs one guest thread at a time, but a thread blocked in a # syscall (futex wait, park) releases that lock, so on several CPUs the # host scheduler decides how the requester and the actor interleave and - # how often they spin, park and wake: up to 3.7% instructions per - # operation under load on `threads_request_seq`. On one CPU the - # interleaving barely depends on the host (0.9%); the code paths measured - # are the same. + # how often they spin, park and wake: run pairs of `threads_request_seq` + # spread up to 6% under load. On one CPU the interleaving barely depends + # on the host (1.5%, the gate's median of nine within 0.9%); the code + # paths measured are the same. out = subprocess.run( ["valgrind", "--tool=callgrind", "--fair-sched=yes", "--callgrind-out-file=/dev/null", str(binary), name, str(ops)], @@ -148,8 +148,7 @@ def per_op(binary, name, small, large, reps, csv=None, label=""): The two-thread benchmark gets more pairs (`--contended-reps`). The minimum is not a stable estimate here: the noise is in both sizes, so the - difference of the minima moved 3.3% across ten loaded runs, against 1.4% - for the median. + difference of the minima moved 3.3% across ten loaded runs. """ runs = {small: [], large: []} for rep in range(reps): diff --git a/docs/TESTING.md b/docs/TESTING.md index 79d649e97..f414cde3b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2110,13 +2110,16 @@ identical counts run to run, three pairs) and threads-mode sequential `request()` (a lockstep pair of threads, nine pairs). Valgrind runs one guest thread at a time, but a thread blocked in a syscall lets the other run, so with several CPUs the host scheduler decides how the requester and the actor -interleave, and how often they spin, park and wake. Under load that moved -`threads_request_seq` by up to 3.7% between runs, as much as the threshold. -Every callgrind run is therefore pinned to one CPU, which leaves the code -paths unchanged and cuts that spread to 0.9%. A minimum over runs (on the -theory that contention only adds instructions) was tried and rejected: the -noise is in both sizes, so the difference of the minima moved 3.3% across -ten loaded runs. On a multi-threaded runtime, or with free-running OS-thread +interleave, and how often they spin, park and wake. On fede under load +(load average 40-80 on 30 CPUs) single run pairs of `threads_request_seq` +spread up to 6%, and ten back-to-back gates moved its change against the base +by up to 4.6 points (one false +3.5%). Every callgrind run is therefore pinned +to one CPU, which leaves the code paths unchanged: in ten more gates under +the same kind of load, run pairs spread 1.5%, each side's median of nine +stayed within 0.9%, and the change against the base stayed within 0.6 +points. A minimum over runs (on the theory that contention only adds +instructions) was tried and rejected: the noise is in both sizes, so the +difference of the minima moved 3.3%. On a multi-threaded runtime, or with free-running OS-thread producers (threads-mode burst and `send`), how often actors park and wake changes from run to run (up to ±20% instructions), so those run only in the wall-clock check. The gate fails when any benchmark's instructions per operation grow From 63d57f10cc107be6da3995e8bee712815bdcdec1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Sat, 21 Nov 2026 13:22:57 -0300 Subject: [PATCH 4/5] ci(perf-gate): no CPU pinning; median of nine pairs for threads_request_seq Pinning helped on loaded fede but on the CI runner it turned a repeatable count (run pairs within 0.05%) into a 10% spread and a false failure. Unpinned, the median of nine pairs kept the change against an identical base within -0.4% to +1.1% across ten gates under heavy load. --- CHANGELOG.md | 6 +++--- benchmarks/README.md | 2 +- benchmarks/hotpath.py | 35 ++++++++++++++++------------------- docs/TESTING.md | 24 +++++++++++++----------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c7895d13..345e27ab1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,9 +52,9 @@ per-PR regressions cannot add up across a release cycle. The job summary table shows release, base and head with both changes. The release binary is built once per release and cached (`hotpath.py build-release` on pushes - to main). Callgrind runs are pinned to one CPU and - `threads_request_seq` takes the median of nine run pairs: under load the - host scheduler had swung it by several percent, as much as the threshold. + to main). `threads_request_seq`, whose thread + interleaving varies on a loaded host, takes the median of nine run pairs + instead of three. - Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup` (a 20 ms startup deadline raced the fast starts on a loaded host, which also made cargo-mutants report surviving mutants as caught): the replacement's diff --git a/benchmarks/README.md b/benchmarks/README.md index 5e01e5bce..23f723e0c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -56,7 +56,7 @@ python3 benchmarks/hotpath.py abba --cores 12-15 --reps 30 # wall clock, before ``` `gate` is the `perf-gate` CI check: it fails when instructions per operation -grow more than 3% over the merge base (callgrind pinned to one CPU), or more than 3% over the last release +grow more than 3% over the merge base, or more than 3% over the last release tag (`--release`; the cumulative budget per release cycle, so small per-PR regressions cannot add up). `abba` alternates base and head on the same cores and reports the median and spread. See diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index cddeaea40..2cb8c2061 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -13,8 +13,8 @@ noisy CI runners. Each benchmark runs at two sizes; instructions per operation is the difference divided by the extra operations, so process start, runtime and actor setup cancel; the median over `--reps` run pairs (`--contended-reps` for the -two-thread benchmark). Every run is pinned to one CPU, so the host scheduler -cannot change how the threads interleave (see `instructions`). It fails when any benchmark grows by more than +two-thread benchmark, whose thread interleaving varies under load; see +`per_op`). It fails when any benchmark grows by more than `--threshold` percent (default 3). It is the CI check. With `--release` (CI passes the last `v*` tag), `gate` also measures that @@ -114,24 +114,19 @@ def benchmarks(binary, only, gate=False): IR = re.compile(r"Collected\s*:\s*(\d+)") -def one_cpu(): - """Pin the measured process to one CPU (see `instructions`).""" - if hasattr(os, "sched_setaffinity"): - os.sched_setaffinity(0, {max(os.sched_getaffinity(0))}) - - def instructions(binary, name, ops): # Valgrind runs one guest thread at a time, but a thread blocked in a - # syscall (futex wait, park) releases that lock, so on several CPUs the - # host scheduler decides how the requester and the actor interleave and - # how often they spin, park and wake: run pairs of `threads_request_seq` - # spread up to 6% under load. On one CPU the interleaving barely depends - # on the host (1.5%, the gate's median of nine within 0.9%); the code - # paths measured are the same. + # syscall (futex wait, park) releases that lock, so the host scheduler + # decides how the requester and the actor of `threads_request_seq` + # interleave and how often they spin, park and wake. On an idle machine + # a woken thread finds a free CPU at once and the count repeats; under + # load it does not (see `per_op`). Do not pin the run to one CPU: then + # every hand-off depends on wakeup preemption, and on the CI runner run + # pairs spread 10% (against 0.05% unpinned). out = subprocess.run( ["valgrind", "--tool=callgrind", "--fair-sched=yes", "--callgrind-out-file=/dev/null", str(binary), name, str(ops)], - text=True, capture_output=True, timeout=180, preexec_fn=one_cpu) + text=True, capture_output=True, timeout=180) if out.returncode != 0: sys.exit(f"{binary.name} {name} {ops} failed:\n{out.stderr[-3000:]}") return int(IR.search(out.stderr).group(1)) @@ -146,9 +141,11 @@ def per_op(binary, name, small, large, reps, csv=None, label=""): """Instructions per operation: the median over run pairs of (instructions at `large` - at `small`) / extra operations, and the pairs. - The two-thread benchmark gets more pairs (`--contended-reps`). The minimum - is not a stable estimate here: the noise is in both sizes, so the - difference of the minima moved 3.3% across ten loaded runs. + The two-thread benchmark gets more pairs (`--contended-reps`): across ten + gates on a heavily loaded host its change against an identical base + stayed within -0.4% to +1.1% with nine pairs, against -1.1% to +1.5% + with three. The minimum is not a stable estimate: the noise is in both + sizes, so the difference of the minima moved -1.6% to +3.5%. """ runs = {small: [], large: []} for rep in range(reps): @@ -230,7 +227,7 @@ def gate(args, base_bin, head_bin, base, head, release_bin=None, release=None): "", f"{revisions} Instructions per operation " f"(callgrind, sizes {args.small} and {args.large}, median of {args.reps} run " - f"pairs, {args.contended_reps} for the two-thread `threads_*`, one CPU); " + f"pairs, {args.contended_reps} for the two-thread `threads_*`); " f"{thresholds}.", "", *header, diff --git a/docs/TESTING.md b/docs/TESTING.md index f414cde3b..2ac87e5f9 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2109,17 +2109,19 @@ sequential and 16-task `request()`, `send` from one and two producer tasks; identical counts run to run, three pairs) and threads-mode sequential `request()` (a lockstep pair of threads, nine pairs). Valgrind runs one guest thread at a time, but a thread blocked in a syscall lets the other run, so -with several CPUs the host scheduler decides how the requester and the actor -interleave, and how often they spin, park and wake. On fede under load -(load average 40-80 on 30 CPUs) single run pairs of `threads_request_seq` -spread up to 6%, and ten back-to-back gates moved its change against the base -by up to 4.6 points (one false +3.5%). Every callgrind run is therefore pinned -to one CPU, which leaves the code paths unchanged: in ten more gates under -the same kind of load, run pairs spread 1.5%, each side's median of nine -stayed within 0.9%, and the change against the base stayed within 0.6 -points. A minimum over runs (on the theory that contention only adds -instructions) was tried and rejected: the noise is in both sizes, so the -difference of the minima moved 3.3%. On a multi-threaded runtime, or with free-running OS-thread +the host scheduler decides how the requester and the actor interleave, and +how often they spin, park and wake. On an idle machine, such as the CI +runner, a woken thread finds a free CPU at once and the count repeats (run +pairs within 0.05%). Under load it does not: on fede (load average 40-80 on +30 CPUs) run pairs spread 5-6%. Across ten gates there, the change against +an identical base stayed within -1.1% to +1.5% with a median of three +pairs, and within -0.4% to +1.1% with the median of nine the gate now uses. +Two other fixes were tried and rejected. The minimum over runs, on the +theory that contention only adds instructions, moved -1.6% to +3.5%, one +false failure: the noise is in both sizes, so the difference of the minima +is not a lower bound. Pinning each run to one CPU gave 1.5% spreads on +loaded fede but 10% on the CI runner, where every hand-off then depends on +wakeup preemption. On a multi-threaded runtime, or with free-running OS-thread producers (threads-mode burst and `send`), how often actors park and wake changes from run to run (up to ±20% instructions), so those run only in the wall-clock check. The gate fails when any benchmark's instructions per operation grow From ae7dbb65ecb3b6f162bec8e0bc31d4196ffd51f1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Sat, 21 Nov 2026 13:40:54 -0300 Subject: [PATCH 5/5] docs(perf-gate): noise numbers from twenty loaded gates on fede --- benchmarks/hotpath.py | 5 ++--- docs/TESTING.md | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index 2cb8c2061..6226a69a4 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -141,10 +141,9 @@ def per_op(binary, name, small, large, reps, csv=None, label=""): """Instructions per operation: the median over run pairs of (instructions at `large` - at `small`) / extra operations, and the pairs. - The two-thread benchmark gets more pairs (`--contended-reps`): across ten + The two-thread benchmark gets more pairs (`--contended-reps`): in twenty gates on a heavily loaded host its change against an identical base - stayed within -0.4% to +1.1% with nine pairs, against -1.1% to +1.5% - with three. The minimum is not a stable estimate: the noise is in both + stayed within -0.8% to +1.4% with nine pairs; three reached +3.8%. The minimum is not a stable estimate: the noise is in both sizes, so the difference of the minima moved -1.6% to +3.5%. """ runs = {small: [], large: []} diff --git a/docs/TESTING.md b/docs/TESTING.md index 2ac87e5f9..56964afc0 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2113,9 +2113,11 @@ the host scheduler decides how the requester and the actor interleave, and how often they spin, park and wake. On an idle machine, such as the CI runner, a woken thread finds a free CPU at once and the count repeats (run pairs within 0.05%). Under load it does not: on fede (load average 40-80 on -30 CPUs) run pairs spread 5-6%. Across ten gates there, the change against -an identical base stayed within -1.1% to +1.5% with a median of three -pairs, and within -0.4% to +1.1% with the median of nine the gate now uses. +30 CPUs) run pairs spread 5-6%. In twenty gates there (two sets of ten) +the median of nine pairs the gate now uses kept the change against an +identical base within -0.8% to +1.4% and against v0.8.0 within -0.2% to ++1.6%. On the same samples the old median of three reached +3.8% against +the release, a false failure. Two other fixes were tried and rejected. The minimum over runs, on the theory that contention only adds instructions, moved -1.6% to +3.5%, one false failure: the noise is in both sizes, so the difference of the minima