From c39b7c37d538743a8e302a9e8ff3360104046c6e Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Thu, 5 Nov 2026 18:51:12 +1100 Subject: [PATCH 1/5] ci(perf): measure a core's release PR against the head with only that core at its tag A core's release PR was measured against the whole tree at the core's last tag, so SDK changes made since then, already gated and released on the SDK's own release PR, failed the core's release. The base is now the PR's own tree with only that core's package checked out at its tag. The SDK's release PR and feature PRs keep their bases. The build-hash self-test now also builds each kind of base on a synthetic repo. --- .github/scripts/perf-ir-base.sh | 29 ++++-- .github/scripts/perf-ir-hash-selftest.sh | 109 ++++++++++++++++++++++- .github/scripts/perf-ir.sh | 15 +++- .github/workflows/ci.yml | 4 +- .github/workflows/instruction-counts.yml | 6 +- CONTRIBUTING.md | 2 +- 6 files changed, 151 insertions(+), 14 deletions(-) diff --git a/.github/scripts/perf-ir-base.sh b/.github/scripts/perf-ir-base.sh index 31290891..e4befe88 100644 --- a/.github/scripts/perf-ir-base.sh +++ b/.github/scripts/perf-ir-base.sh @@ -1,17 +1,24 @@ #!/usr/bin/env bash # Pick the commit the instruction-count gate measures a pull request against, -# and write it to $GITHUB_OUTPUT as `sha` (what to build) and `label` (how the -# comment names it). Run from a checkout of the PR merge commit with its first -# parent. +# and write it to $GITHUB_OUTPUT as `sha` (what to build), `package` (empty, or +# the one package the base takes from `sha`) and `label` (how the comment names +# it). Run from a checkout of the PR merge commit with its first parent. # # A feature PR is measured against its base, the merge commit's first parent. # A release PR (head branch release-please--branches--main--components--) # is rebased onto main on every push, so against main it would measure only a # version bump. It is measured against that package's last release instead: # tag -v, where is the package's entry in the base's -# .release-please-manifest.json. The comment then shows every workload's drift -# since that release. A tag that cannot be fetched fails the job; it never -# falls back to main, which would read PASS without measuring anything. +# .release-please-manifest.json. A tag that cannot be fetched fails the job; it +# never falls back to main, which would read PASS without measuring anything. +# +# The SDK's release (packages/cachekit) is measured against the whole tagged +# tree, so its comment shows every workload's drift since that release. A +# core's release ships only that core's package, so its base is the PR's own +# tree with only that package taken from the tag (perf-ir.sh trees): every +# workload, the SDK ones that run through the core included, then measures the +# core's drift alone. SDK drift since the core's tag is the SDK's own release's +# to gate, against the SDK's tag. # # CI: HEAD_REF=${{ github.head_ref }} bash .github/scripts/perf-ir-base.sh # Local: HEAD_REF=release-please--branches--main--components--cachekit \ @@ -20,6 +27,7 @@ set -euo pipefail if [[ ${HEAD_REF:-} != release-please--*--components--* ]]; then sha=$(git rev-parse HEAD^1) + package= label=$sha else component=${HEAD_REF##*--components--} @@ -38,10 +46,17 @@ else exit 1 } sha=$(git rev-parse "$tag^{commit}") - label="$tag ($(git rev-parse --short=7 "$sha")), last release" + if [[ $path == packages/cachekit ]]; then + package= + label="$tag ($(git rev-parse --short=7 "$sha")), last release" + else + package=$path + label="this PR with $path at $tag ($(git rev-parse --short=7 "$sha")), last release" + fi fi echo "Base: $label" { echo "sha=$sha" + echo "package=$package" echo "label=$label" } >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/perf-ir-hash-selftest.sh b/.github/scripts/perf-ir-hash-selftest.sh index 99dffe67..60ff8991 100755 --- a/.github/scripts/perf-ir-hash-selftest.sh +++ b/.github/scripts/perf-ir-hash-selftest.sh @@ -4,9 +4,17 @@ # match, so a file the workloads load that the hash misses lets a regression # pass unmeasured. The hash must move when a shipped file of any package or # the lockfile changes, stay put when an unshipped file changes, and not -# depend on where the tree sits. Needs node; no build, no valgrind. +# depend on where the tree sits. +# +# Then the base the gate builds (perf-ir-base.sh, then perf-ir.sh trees), on a +# synthetic repo of that tree. On a core's release PR, SDK drift since the +# core's tag must hash alike on both trees, so it cannot fail the gate, and the +# core's own drift must hash apart, measured against the tagged core. The SDK's +# release PR and a feature PR keep their bases: the SDK's tag, and HEAD^1. +# Needs node and git; no build, no valgrind. set -euo pipefail -src=$(cd "$(dirname "$0")/../../packages/cachekit/bench" && pwd) +scripts=$(cd "$(dirname "$0")" && pwd) +src=$(cd "$scripts/../../packages/cachekit/bench" && pwd) work=$(mktemp -d) trap 'rm -rf "${work:?}"' EXIT @@ -75,4 +83,101 @@ if hash "$work/a" > /dev/null 2>&1; then else echo "ok: a files glob that matches nothing fails the run" fi + +g() { git -C "$1" -c user.name=selftest -c user.email=selftest@example.invalid -c commit.gpgsign=false \ + -c core.hooksPath=/dev/null "${@:2}"; } +# The released repo: both tags on one commit, as if every package had just +# released. +origin=$work/origin +tree "$origin" +cp "$scripts/../../release-please-config.json" "$scripts/../../.release-please-manifest.json" "$origin/" +manifest=$(jq -r '.["packages/cachekit"] as $s | .["packages/cachekit-core-wasm"] as $w | "\($s) \($w)"' \ + "$origin/.release-please-manifest.json") +g "$origin" init -q -b main +g "$origin" add -A +g "$origin" commit -q -m release +g "$origin" tag "cachekit-v${manifest% *}" +g "$origin" tag "cachekit-core-wasm-v${manifest#* }" + +# pick ...: append to each file on main after +# the release, check out the PR's merge commit (its own change touches only an +# unshipped changelog), and pick and build the base trees as CI does. +pick() { + local repo=$work/$1 + git clone -q --no-tags "file://$origin" "$repo" + for f in "${@:3}"; do echo '// drift' >> "$repo/$f"; done + g "$repo" add -A + g "$repo" commit -q -m drift + g "$repo" switch -q -c pr + echo 'changelog' >> "$repo/packages/cachekit-core-wasm/CHANGELOG.md" + g "$repo" add -A + g "$repo" commit -q -m 'the PR' + g "$repo" switch -q --detach main + g "$repo" merge -q --no-ff -m merge pr + mkdir -p "$work/$1-out" + (cd "$repo" && HEAD_REF=$2 GITHUB_OUTPUT=$work/$1-out/base bash "$scripts/perf-ir-base.sh" > /dev/null) + local sha package + sha=$(sed -n 's/^sha=//p' "$work/$1-out/base") + package=$(sed -n 's/^package=//p' "$work/$1-out/base") + (cd "$repo" && BASE_SHA=$sha BASE_PACKAGE=$package OUT=$work/$1-out IR_ROOT=$work/$1-ir \ + bash "$scripts/perf-ir.sh" trees) +} +check() { # check same|differs