diff --git a/.github/workflows/perf-gate.yml b/.github/workflows/perf-gate.yml index 17c94833..18f7e7ff 100644 --- a/.github/workflows/perf-gate.yml +++ b/.github/workflows/perf-gate.yml @@ -1,6 +1,7 @@ name: Perf gate -# Hot-path instruction counts, PR against its merge base +# Hot-path instruction counts, PR against its merge base and against the last +# release tag, a cumulative budget per release cycle # (benchmarks/hotpath.py gate; docs/TESTING.md, "Performance gate"). # The workflow always runs so the check can be required; the gate itself is # skipped (and passes) unless runtime source, dependencies, toolchain or driver changed. @@ -9,6 +10,10 @@ on: pull_request: branches: ["main"] merge_group: + # On main only the release binary is built (when missing) and cached, so PRs, + # which can only restore caches saved on main, reuse it. + push: + branches: ["main"] concurrency: group: perf-gate-${{ github.event.pull_request.number || github.ref }} @@ -19,11 +24,13 @@ env: CARGO_INCREMENTAL: "0" # Maximum growth of instructions per operation, in percent. HOTPATH_THRESHOLD: "3" + # Maximum growth over the last release (git describe --match 'v*'), in percent. + HOTPATH_RELEASE_THRESHOLD: "3" jobs: perf-gate: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 50 steps: - uses: actions/checkout@v4 with: @@ -34,6 +41,14 @@ jobs: PR_BASE: ${{ github.event.pull_request.base.sha }} MQ_BASE: ${{ github.event.merge_group.base_sha }} run: | + release=$(git describe --tags --abbrev=0 --match 'v*' HEAD) + echo "release=$release" >> "$GITHUB_OUTPUT" + echo "release_sha=$(git rev-parse "$release^{commit}")" >> "$GITHUB_OUTPUT" + if [ "${{ github.event_name }}" = push ]; then + echo "run=false" >> "$GITHUB_OUTPUT" + echo "warm=true" >> "$GITHUB_OUTPUT" + exit 0 + fi base=$(git merge-base HEAD "${PR_BASE:-$MQ_BASE}") echo "base=$base" >> "$GITHUB_OUTPUT" changed=$(git diff --name-only "$base" HEAD) @@ -44,13 +59,29 @@ jobs: echo "Perf gate skipped: no runtime source, dependency, toolchain or driver change." >> "$GITHUB_STEP_SUMMARY" fi - uses: dtolnay/rust-toolchain@1.93.0 - if: steps.scope.outputs.run == 'true' + if: steps.scope.outputs.run == 'true' || steps.scope.outputs.warm == 'true' + # The release binary: same driver, same toolchain, same release commit. + - name: Release binary cache + if: steps.scope.outputs.run == 'true' || steps.scope.outputs.warm == 'true' + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/hotpath-release + key: hotpath-release-${{ steps.scope.outputs.release_sha }}-${{ hashFiles('benchmarks/hotpath/driver.rs', 'rust-toolchain.toml') }} + - name: Build the release binary + if: steps.scope.outputs.warm == 'true' + run: > + python3 benchmarks/hotpath.py build-release + --release ${{ steps.scope.outputs.release }} + --release-cache ${{ runner.temp }}/hotpath-release + --work ${{ runner.temp }}/hotpath - name: Install valgrind if: steps.scope.outputs.run == 'true' run: sudo apt-get update -q && sudo apt-get install -y -q valgrind - - name: Instruction counts, head vs merge base + - name: Instruction counts, head vs merge base and last release if: steps.scope.outputs.run == 'true' run: > python3 benchmarks/hotpath.py gate --base ${{ steps.scope.outputs.base }} --head HEAD + --release ${{ steps.scope.outputs.release }} + --release-cache ${{ runner.temp }}/hotpath-release --work ${{ runner.temp }}/hotpath diff --git a/CHANGELOG.md b/CHANGELOG.md index c8eb616f..345e27ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,15 @@ non-blocking disposal and checks the queue drains and every worker retires after its keep-alive (`disposal.keep_alive` failpoint with `failpoints`). The soak harness judges the three gauges. +- `perf-gate` also holds every PR to a cumulative budget: the head may grow + at most 3% instructions per operation over the last release tag + (`git describe --match 'v*'`), not only over its merge base, so small + per-PR regressions cannot add up across a release cycle. The job summary + table shows release, base and head with both changes. The release binary + is built once per release and cached (`hotpath.py build-release` on pushes + to main). `threads_request_seq`, whose thread + interleaving varies on a loaded host, takes the median of nine run pairs + instead of three. - Fix the flaky `supervision::transient_startup_timeout_retries_after_actual_cleanup` (a 20 ms startup deadline raced the fast starts on a loaded host, which also made cargo-mutants report surviving mutants as caught): the replacement's diff --git a/benchmarks/README.md b/benchmarks/README.md index daafe214..23f723e0 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -56,7 +56,9 @@ python3 benchmarks/hotpath.py abba --cores 12-15 --reps 30 # wall clock, before ``` `gate` is the `perf-gate` CI check: it fails when instructions per operation -grow more than 3% over the merge base. `abba` alternates base and head on the +grow more than 3% over the merge base, or more than 3% over the last release +tag (`--release`; the cumulative budget per release cycle, so small per-PR +regressions cannot add up). `abba` alternates base and head on the same cores and reports the median and spread. See [performance gate](../docs/TESTING.md#performance-gate). diff --git a/benchmarks/hotpath.py b/benchmarks/hotpath.py index c7bfc0be..6226a69a 100755 --- a/benchmarks/hotpath.py +++ b/benchmarks/hotpath.py @@ -2,6 +2,8 @@ """Hot-path regression checks: a revision against its base. hotpath.py gate [--base REV] [--head REV] [--threshold PCT] [--features F] + [--release REV] [--release-cache DIR] + hotpath.py build-release --release REV --release-cache DIR [--features F] hotpath.py abba [--base REV] [--head REV] [--reps N] [--cores LIST] [--features F] Both build `benchmarks/hotpath/driver.rs` (the head's copy, in both trees, so @@ -10,14 +12,25 @@ `gate` counts instructions with callgrind, which is deterministic enough for noisy CI runners. Each benchmark runs at two sizes; instructions per operation is the difference divided by the extra operations, so process start, runtime -and actor setup cancel. It fails when any benchmark grows by more than +and actor setup cancel; the median over `--reps` run pairs (`--contended-reps` for the +two-thread benchmark, whose thread interleaving varies under load; see +`per_op`). It fails when any benchmark grows by more than `--threshold` percent (default 3). It is the CI check. +With `--release` (CI passes the last `v*` tag), `gate` also measures that +release and fails when any benchmark grows by more than `--release-threshold` +percent (default: `--threshold`) over it, so regressions that each pass the +per-PR check cannot add up across a release cycle. `--release-cache DIR` keeps +the release binary there, keyed by release commit, driver and toolchain, so +it is built once per release rather than per PR; `build-release` only fills +that cache (CI runs it on every push to main, where PRs can restore it from). + `abba` measures wall-clock throughput, alternating base and head on the same cores (ABBA order), and reports the median and spread. Run it on a quiet machine before a release; it is never run in CI. """ import argparse +import hashlib import os import re import shutil @@ -51,6 +64,7 @@ def default_base(head): def build(work, label, rev, features=None): """Check out `rev` in a worktree, add the driver, build it; return the binary.""" tree = work / label + work.mkdir(parents=True, exist_ok=True) sh(["git", "worktree", "add", "--detach", "--force", str(tree), rev], cwd=ROOT, capture_output=True) shutil.copy(DRIVER, tree / "concurrency" / "examples" / "hotpath.rs") @@ -64,6 +78,33 @@ def build(work, label, rev, features=None): return binary +def release_key(rev, features=None): + """Cache key of a release binary: its commit, the driver and the compiler.""" + driver = hashlib.sha256(DRIVER.read_bytes()).hexdigest()[:16] + rustc = sh(["rustc", "-V"], cwd=ROOT, capture_output=True).stdout + toolchain = hashlib.sha256(rustc.encode()).hexdigest()[:16] + commit = git("rev-parse", f"{rev}^{{commit}}") # a tag's commit, not the tag object + extra = "-" + re.sub(r"[^A-Za-z0-9]+", "_", features) if features else "" + return f"hotpath-{commit[:16]}-{driver}-{toolchain}{extra}" + + +def build_release(work, rev, cache, features=None): + """The release binary, from `cache` when an identical build is there.""" + if not cache: + return build(work, "release", rev, features) + cache = Path(cache).resolve() + cached = cache / release_key(rev, features) + if cached.exists(): + print(f"release binary from cache: {cached}", file=sys.stderr) + return cached + binary = build(work, "release", rev, features) + cache.mkdir(parents=True, exist_ok=True) + for stale in cache.glob("hotpath-*"): + stale.unlink() + shutil.copy(binary, cached) + return cached + + def benchmarks(binary, only, gate=False): """The driver's benchmarks; the gate takes only the deterministic ones.""" names = sh([str(binary), "list-gate" if gate else "list"], capture_output=True).stdout.split() @@ -74,6 +115,14 @@ def benchmarks(binary, only, gate=False): def instructions(binary, name, ops): + # Valgrind runs one guest thread at a time, but a thread blocked in a + # syscall (futex wait, park) releases that lock, so the host scheduler + # decides how the requester and the actor of `threads_request_seq` + # interleave and how often they spin, park and wake. On an idle machine + # a woken thread finds a free CPU at once and the count repeats; under + # load it does not (see `per_op`). Do not pin the run to one CPU: then + # every hand-off depends on wakeup preemption, and on the CI runner run + # pairs spread 10% (against 0.05% unpinned). out = subprocess.run( ["valgrind", "--tool=callgrind", "--fair-sched=yes", "--callgrind-out-file=/dev/null", str(binary), name, str(ops)], @@ -83,12 +132,29 @@ def instructions(binary, name, ops): return int(IR.search(out.stderr).group(1)) -def per_op(binary, name, small, large, reps): - samples = [ - (instructions(binary, name, large) - instructions(binary, name, small)) / (large - small) - for _ in range(reps) - ] - return statistics.median(samples), samples +def contended(name): + """Benchmarks whose count varies with thread interleaving (not `async_ct_*`).""" + return not name.startswith("async_ct_") + + +def per_op(binary, name, small, large, reps, csv=None, label=""): + """Instructions per operation: the median over run pairs of + (instructions at `large` - at `small`) / extra operations, and the pairs. + + The two-thread benchmark gets more pairs (`--contended-reps`): in twenty + gates on a heavily loaded host its change against an identical base + stayed within -0.8% to +1.4% with nine pairs; three reached +3.8%. The minimum is not a stable estimate: the noise is in both + sizes, so the difference of the minima moved -1.6% to +3.5%. + """ + runs = {small: [], large: []} + for rep in range(reps): + for size in (small, large): + ir = instructions(binary, name, size) + runs[size].append(ir) + if csv: + csv.write(f"{label},{name},{size},{rep},{ir}\n") + pairs = [(lg - sm) / (large - small) for sm, lg in zip(runs[small], runs[large])] + return statistics.median(pairs), pairs def report(lines): @@ -100,36 +166,75 @@ def report(lines): f.write(text) -def gate(args, base_bin, head_bin, base, head): +def gate(args, base_bin, head_bin, base, head, release_bin=None, release=None): names = benchmarks(head_bin, args.only, gate=True) - rows, failed = [], [] + rows, failed, failed_release = [], [], [] + csv = open(args.csv, "w") if args.csv else None + if csv: + csv.write("side,benchmark,ops,rep,instructions\n") for name in names: - b, bs = per_op(base_bin, name, args.small, args.large, args.reps) - h, hs = per_op(head_bin, name, args.small, args.large, args.reps) + reps = args.contended_reps if contended(name) else args.reps + b, bs = per_op(base_bin, name, args.small, args.large, reps, csv, "base") + h, hs = per_op(head_bin, name, args.small, args.large, reps, csv, "head") delta = 100 * (h / b - 1) bad = delta > args.threshold if bad: failed.append(name) spread = max(max(bs) - min(bs), max(hs) - min(hs)) - rows.append(f"| `{name}` | {b:,.0f} | {h:,.0f} | {delta:+.1f}% | ±{spread:,.0f} | " - f"{'FAIL' if bad else 'ok'} |") - print(f"{name}: base {b:.0f} head {h:.0f} ({delta:+.1f}%)", file=sys.stderr) - verdict = (f"**FAIL**: {', '.join(failed)} grew more than {args.threshold}%" - if failed else f"**PASS**: no benchmark grew more than {args.threshold}%") + if release_bin is None: + rows.append(f"| `{name}` | {b:,.0f} | {h:,.0f} | {delta:+.1f}% | {spread:,.0f} | " + f"{'FAIL' if bad else 'ok'} |") + print(f"{name}: base {b:.0f} head {h:.0f} ({delta:+.1f}%)", file=sys.stderr) + continue + if release_bin == base_bin: + r, rs = b, bs + else: + r, rs = per_op(release_bin, name, args.small, args.large, reps, csv, "release") + rdelta = 100 * (h / r - 1) + rbad = rdelta > args.release_threshold + if rbad: + failed_release.append(name) + spread = max(spread, max(rs) - min(rs)) + result = ("FAIL" if bad else "ok") + " / " + ("FAIL" if rbad else "ok") + rows.append(f"| `{name}` | {r:,.0f} | {b:,.0f} | {h:,.0f} | {delta:+.1f}% | " + f"{rdelta:+.1f}% | {spread:,.0f} | {result} |") + print(f"{name}: release {r:.0f} base {b:.0f} head {h:.0f} " + f"({delta:+.1f}% vs base, {rdelta:+.1f}% vs release)", file=sys.stderr) + verdict = [f"**FAIL**: {', '.join(failed)} grew more than {args.threshold}% over the base" + if failed else f"**PASS**: no benchmark grew more than {args.threshold}% over the base"] + if release_bin is None: + header = ["| benchmark | base Ir/op | head Ir/op | change | raw spread | result |", + "|---|---:|---:|---:|---:|---|"] + revisions = f"Base `{base[:10]}`, head `{head[:10]}`." + thresholds = f"threshold +{args.threshold}%" + else: + header = ["| benchmark | release Ir/op | base Ir/op | head Ir/op | Δ vs base | " + "Δ vs release | raw spread | result (base / release) |", + "|---|---:|---:|---:|---:|---:|---:|---|"] + revisions = (f"Release `{args.release}` (`{release[:10]}`), base `{base[:10]}`, " + f"head `{head[:10]}`.") + thresholds = (f"thresholds +{args.threshold}% over the base, " + f"+{args.release_threshold}% over the release") + verdict.append( + f"**FAIL**: {', '.join(failed_release)} grew more than {args.release_threshold}% " + f"since `{args.release}` (cumulative budget; see docs/TESTING.md, " + "\"Performance gate\")" if failed_release else + f"**PASS**: no benchmark grew more than {args.release_threshold}% " + f"since `{args.release}`") report([ "## Hot-path instruction counts", "", - f"Base `{base[:10]}`, head `{head[:10]}`. Instructions per operation " - f"(callgrind, sizes {args.small} and {args.large}, median of {args.reps}); " - f"threshold +{args.threshold}%.", + f"{revisions} Instructions per operation " + f"(callgrind, sizes {args.small} and {args.large}, median of {args.reps} run " + f"pairs, {args.contended_reps} for the two-thread `threads_*`); " + f"{thresholds}.", "", - "| benchmark | base Ir/op | head Ir/op | change | max spread | result |", - "|---|---:|---:|---:|---:|---|", + *header, *rows, "", - verdict, + *(f"{v} " for v in verdict), ]) - return 1 if failed else 0 + return 1 if failed or failed_release else 0 def wallclock(binary, name, ops, cores): @@ -180,7 +285,7 @@ def abba(args, base_bin, head_bin, base, head): def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("mode", choices=["gate", "abba"]) + p.add_argument("mode", choices=["gate", "abba", "build-release"]) p.add_argument("--head", default="HEAD") p.add_argument("--base", help="default: merge base of --head and main") p.add_argument("--only", nargs="*", help="benchmark names (default: all)") @@ -188,28 +293,66 @@ def main(): p.add_argument("--threshold", type=float, default=float(os.environ.get("HOTPATH_THRESHOLD", "3")), help="gate: max growth in percent (default 3, or $HOTPATH_THRESHOLD)") + p.add_argument("--release", help="gate: also hold the head to this release (a tag)") + p.add_argument("--release-threshold", type=float, + default=(float(os.environ["HOTPATH_RELEASE_THRESHOLD"]) + if os.environ.get("HOTPATH_RELEASE_THRESHOLD") else None), + help="gate: max growth over --release in percent " + "(default --threshold, or $HOTPATH_RELEASE_THRESHOLD)") + p.add_argument("--release-cache", help="gate: directory that keeps the release binary") p.add_argument("--small", type=int, default=2000, help="gate: smaller size") p.add_argument("--large", type=int, default=6000, help="gate: larger size") p.add_argument("--reps", type=int, help="gate: runs per size (default 3); abba: repetitions (default 20)") p.add_argument("--ops", type=int, default=500_000, help="abba: operations per run") p.add_argument("--cores", help="abba: taskset core list, e.g. 12-15") - p.add_argument("--csv", help="abba: raw samples") - p.add_argument("--features", help="cargo features for both sides, e.g. metrics " + p.add_argument("--contended-reps", type=int, default=9, + help="gate: runs per size for benchmarks with more than one thread " + "(default 9)") + p.add_argument("--csv", help="raw samples") + p.add_argument("--features", help="cargo features for every side, e.g. metrics " "(default: the crate's defaults, as CI gates)") args = p.parse_args() args.reps = args.reps or (3 if args.mode == "gate" else 20) - + if args.release_threshold is None: + args.release_threshold = args.threshold + if args.release and args.mode == "abba": + p.error("--release is a gate option") + if args.mode == "build-release" and not (args.release and args.release_cache): + p.error("build-release needs --release and --release-cache") + + if args.mode == "build-release": + work = Path(args.work or tempfile.mkdtemp(prefix="hotpath-")).resolve() + try: + print(build_release(work, args.release, args.release_cache, args.features)) + finally: + subprocess.run(["git", "worktree", "remove", "--force", str(work / "release")], + cwd=ROOT, capture_output=True) + if not args.work: + shutil.rmtree(work, ignore_errors=True) + return head = git("rev-parse", args.head) base = git("rev-parse", args.base) if args.base else default_base(head) + release = git("rev-parse", f"{args.release}^{{commit}}") if args.release else None work = Path(args.work or tempfile.mkdtemp(prefix="hotpath-")).resolve() work.mkdir(parents=True, exist_ok=True) try: base_bin = build(work, "base", base, args.features) head_bin = build(work, "head", head, args.features) - run = gate if args.mode == "gate" else abba - sys.exit(run(args, base_bin, head_bin, base, head)) + if args.mode == "abba": + sys.exit(abba(args, base_bin, head_bin, base, head)) + release_bin = None + if release: + try: + release_bin = (base_bin if release == base + else build_release(work, release, args.release_cache, + args.features)) + except subprocess.CalledProcessError: + sys.exit(f"the driver does not build against {args.release}: " + "benchmarks/hotpath/driver.rs must only use APIs that the last " + "release also has") + sys.exit(gate(args, base_bin, head_bin, base, head, release_bin, release)) finally: - for label in ("base", "head"): + for label in ("base", "head", "release"): subprocess.run(["git", "worktree", "remove", "--force", str(work / label)], cwd=ROOT, capture_output=True) if not args.work: diff --git a/docs/TESTING.md b/docs/TESTING.md index 6848b172..56964afc 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -2101,18 +2101,51 @@ across releases (`#[actor]` handlers, `request()`, `send()`). **CI: instruction counts** (`perf-gate` workflow, every PR and the merge queue). Wall-clock numbers on shared runners are too noisy, so the gate counts instructions with callgrind. Each benchmark runs at two sizes (2,000 and 6,000 -operations, three runs each); instructions per operation is the difference -divided by the extra operations, so process start, runtime and actor setup -cancel. Only deterministic benchmarks are gated: the async ones on a Tokio -current-thread runtime (`async_ct_*`: burst, sequential and 16-task -`request()`, `send` from one and two producer tasks; identical counts run to -run) and threads-mode sequential `request()` (a lockstep pair of threads, -within 0.5%). On a multi-threaded runtime, or with free-running OS-thread +operations); instructions per operation is the difference divided by the +extra operations, so process start, runtime and actor setup cancel, and the +gate takes the median over run pairs. Only deterministic benchmarks are +gated: the async ones on a Tokio current-thread runtime (`async_ct_*`: burst, +sequential and 16-task `request()`, `send` from one and two producer tasks; +identical counts run to run, three pairs) and threads-mode sequential +`request()` (a lockstep pair of threads, nine pairs). Valgrind runs one guest +thread at a time, but a thread blocked in a syscall lets the other run, so +the host scheduler decides how the requester and the actor interleave, and +how often they spin, park and wake. On an idle machine, such as the CI +runner, a woken thread finds a free CPU at once and the count repeats (run +pairs within 0.05%). Under load it does not: on fede (load average 40-80 on +30 CPUs) run pairs spread 5-6%. In twenty gates there (two sets of ten) +the median of nine pairs the gate now uses kept the change against an +identical base within -0.8% to +1.4% and against v0.8.0 within -0.2% to ++1.6%. On the same samples the old median of three reached +3.8% against +the release, a false failure. +Two other fixes were tried and rejected. The minimum over runs, on the +theory that contention only adds instructions, moved -1.6% to +3.5%, one +false failure: the noise is in both sizes, so the difference of the minima +is not a lower bound. Pinning each run to one CPU gave 1.5% spreads on +loaded fede but 10% on the CI runner, where every hand-off then depends on +wakeup preemption. On a multi-threaded runtime, or with free-running OS-thread producers (threads-mode burst and `send`), how often actors park and wake changes from run to run (up to ±20% instructions), so those run only in the wall-clock check. The gate fails when any benchmark's instructions per operation grow more than 3% over the merge base (`--threshold`, or `HOTPATH_THRESHOLD` in the -workflow). The job summary has the per-benchmark table. The workflow always +workflow). + +A per-PR budget alone lets regressions add up: ten PRs at +2.5% each all pass. +So the same job also measures the last release tag (`git describe --tags +--abbrev=0 --match 'v*'`) with the same driver and harness, and fails when +any benchmark grows more than 3% over that release (`--release`, +`--release-threshold`, or `HOTPATH_RELEASE_THRESHOLD`): the cumulative budget +for a release cycle. A PR that is under 3% on its own can still fail because +of earlier PRs; then the fix is to win back instructions on that path, not to +raise the threshold. A deliberate cost (a new guarantee on the request path) +is argued in the PR and the release budget is reset only by tagging the next +release. The release binary is built once per release, driver and toolchain: +pushes to main build it into the Actions cache (`hotpath.py build-release`), +and PRs restore it. The release is built with the default features, which +every release has; features removed since (such as the `*-experiments` ones +in 0.9) are not used by the driver. The job summary has the per-benchmark +table: release, base and head instructions per operation, the change against +the base and against the release. The workflow always reports, so it can be a required check, but it only builds and measures when `concurrency/src`, `rt/src`, `macros/src`, the driver, a Cargo manifest or lockfile, or the pinned Rust toolchain changed. Release version bumps run the @@ -2122,6 +2155,7 @@ gate too; they cannot inherit a passing source check without measurement. python3 benchmarks/hotpath.py gate # HEAD vs merge base with main python3 benchmarks/hotpath.py gate --base origin/main --head origin/my-branch --threshold 3 python3 benchmarks/hotpath.py gate --features metrics # both sides with the metrics feature +python3 benchmarks/hotpath.py gate --release v0.8.0 # also the cumulative check, as in CI ``` CI gates the default build. `--features` builds both sides with extra