ci(perf): measure a core's release PR against the head with only that core at its tag (LAB-12798)

.github/scripts/perf-ir-base.sh

@@ -1,23 +1,35 @@
 #!/usr/bin/env bash
 # Pick the commit the instruction-count gate measures a pull request against,
-# and write it to $GITHUB_OUTPUT as `sha` (what to build) and `label` (how the
-# comment names it). Run from a checkout of the PR merge commit with its first
-# parent.
+# and write it to $GITHUB_OUTPUT as `sha` (what to build), `package` (empty, or
+# the one package the base takes from `sha`) and `label` (how the comment names
+# it). Run from a checkout of the PR merge commit with its first parent.
 #
 # A feature PR is measured against its base, the merge commit's first parent.
 # A release PR (head branch release-please--branches--main--components--<c>)
 # is rebased onto main on every push, so against main it would measure only a
 # version bump. It is measured against that package's last release instead:
 # tag <c>-v<version>, where <version> is the package's entry in the base's
-# .release-please-manifest.json. The comment then shows every workload's drift
-# since that release. A tag that cannot be fetched fails the job; it never
-# falls back to main, which would read PASS without measuring anything.
+# .release-please-manifest.json. A tag that cannot be fetched fails the job; it
+# never falls back to main, which would read PASS without measuring anything.
+#
+# The SDK's release (packages/cachekit) is measured against the whole tagged
+# tree, so its comment shows every workload's drift since that release. A
+# core's release ships only that core's package, so its base is the PR's own
+# tree with only that package taken from the tag (perf-ir.sh trees): every
+# workload, the SDK ones that run through the core included, then measures the
+# core's code since its tag, with the two exceptions perf-ir.sh's header names
+# (dependency drift, and SDK imports of core exports the tag lacks). SDK drift
+# since the core's tag is the SDK's own release's to gate, against the SDK's
+# tag. Anyone can name a branch, so a PR gets that
+# base only when it changes nothing outside the core's package and the
+# manifest; any other PR on such a branch gets the whole tagged tree.
 #
 #   CI:    HEAD_REF=${{ github.head_ref }} bash .github/scripts/perf-ir-base.sh
 #   Local: HEAD_REF=release-please--branches--main--components--cachekit \
 #            GITHUB_OUTPUT=/dev/stdout bash .github/scripts/perf-ir-base.sh
 set -euo pipefail
 
+package=
 if [[ ${HEAD_REF:-} != release-please--*--components--* ]]; then
   sha=$(git rev-parse HEAD^1)
   label=$sha
@@ -39,9 +51,20 @@ else
   }
   sha=$(git rev-parse "$tag^{commit}")
   label="$tag ($(git rev-parse --short=7 "$sha")), last release"
+  if [[ $path != packages/cachekit ]]; then
+    outside=$(git diff --name-only HEAD^1 HEAD -- . ":!$path/" ':!.release-please-manifest.json' | paste -sd, -)
+    if [[ -z $outside ]]; then
+      package=$path
+      label="this PR with $path at $label"
+    else
+      echo "::notice::release PR changes files outside $path ($outside), so it is measured against the whole $tag tree"
+      label="$label, whole tree: the PR changes files outside $path"
+    fi
+  fi
 fi
 echo "Base: $label"
 {
   echo "sha=$sha"
+  echo "package=$package"
   echo "label=$label"
 } >> "$GITHUB_OUTPUT"

.github/scripts/perf-ir-hash-selftest.sh

@@ -1,12 +1,23 @@
 #!/usr/bin/env bash
+# shellcheck disable=SC2329 # the checks run through assert and $DRIFT
 # Self-test for `bench/ir.mjs --build-hash`, on a synthetic three-package tree.
 # The instruction-count gate skips measuring when the base and head hashes
 # match, so a file the workloads load that the hash misses lets a regression
 # pass unmeasured. The hash must move when a shipped file of any package or
 # the lockfile changes, stay put when an unshipped file changes, and not
-# depend on where the tree sits. Needs node; no build, no valgrind.
+# depend on where the tree sits.
+#
+# Then the base the gate builds (perf-ir-base.sh, then perf-ir.sh trees), on a
+# synthetic repo of that tree. On a core's release PR, the base must be the
+# PR's own tree outside the core's package, so SDK drift since the core's tag
+# cannot fail the gate, and the core's own drift must hash apart, measured
+# against the tagged core. A PR on a
+# core's release branch that changes the SDK, the SDK's release PR and a
+# feature PR keep the whole-tree bases: the tag, and HEAD^1.
+# Needs node, git and jq; no build, no valgrind.
 set -euo pipefail
-src=$(cd "$(dirname "$0")/../../packages/cachekit/bench" && pwd)
+scripts=$(cd "$(dirname "$0")" && pwd)
+src=$(cd "$scripts/../../packages/cachekit/bench" && pwd)
 work=$(mktemp -d)
 trap 'rm -rf "${work:?}"' EXIT
 
@@ -75,4 +86,111 @@ if hash "$work/a" > /dev/null 2>&1; then
 else
   echo "ok: a files glob that matches nothing fails the run"
 fi
+
+g() { git -C "$1" -c user.name=selftest -c user.email=selftest@example.invalid -c commit.gpgsign=false \
+  -c core.hooksPath=/dev/null "${@:2}"; }
+# The released repo: core-wasm's release, then a later SDK release that also
+# changed the core, so each tag names a tree of its own.
+origin=$work/origin
+tree "$origin"
+cp "$scripts/../../release-please-config.json" "$scripts/../../.release-please-manifest.json" "$origin/"
+sdk_tag=cachekit-v$(jq -r '.["packages/cachekit"]' "$origin/.release-please-manifest.json")
+wasm_tag=cachekit-core-wasm-v$(jq -r '.["packages/cachekit-core-wasm"]' "$origin/.release-please-manifest.json")
+sdk=packages/cachekit/dist/index.js glue=packages/cachekit-core-wasm/pkg/glue.js
+added=packages/cachekit-core-wasm/pkg/added.js wasm=packages/cachekit-core-wasm
+g "$origin" init -q -b main
+g "$origin" add -A
+g "$origin" commit -q -m 'core-wasm release'
+g "$origin" tag "$wasm_tag"
+echo '// SDK release' | tee -a "$origin/$sdk" >> "$origin/$glue"
+g "$origin" commit -q -am 'SDK release'
+g "$origin" tag "$sdk_tag"
+
+# set_json <file> <jq filter>
+set_json() { jq "$2" "$1" > "$1.new" && mv "$1.new" "$1"; }
+# pick <scenario> <PR head branch> <PR file|-> <file>...: append to each <file>
+# on main after the releases (and run $DRIFT on the repo, if set), then open
+# the PR. A release PR edits what release-please edits: the package's version,
+# its CHANGELOG.md and its manifest entry. The PR also appends to <PR file>.
+# Check out its merge commit, and pick and build the base trees as CI does.
+pick() {
+  local repo=$work/$1 path
+  git clone -q --no-tags "file://$origin" "$repo"
+  for f in "${@:4}"; do echo '// drift' >> "$repo/$f"; done
+  [[ -z ${DRIFT:-} ]] || "$DRIFT" "$repo"
+  g "$repo" add -A
+  g "$repo" commit -q -m drift
+  g "$repo" switch -q -c pr
+  if [[ $2 == *--components--* ]]; then
+    path=$(jq -r --arg c "${2##*--components--}" \
+      '.packages | to_entries[] | select(.value.component == $c) | .key' "$repo/release-please-config.json")
+    set_json "$repo/$path/package.json" '.version = "9.9.9"'
+    set_json "$repo/.release-please-manifest.json" ".[\"$path\"] = \"9.9.9\""
+    echo 'release notes' >> "$repo/$path/CHANGELOG.md"
+  fi
+  [[ $3 == - ]] || echo '// the PR' >> "$repo/$3"
+  g "$repo" add -A
+  g "$repo" commit -q -m 'the PR'
+  g "$repo" switch -q --detach main
+  g "$repo" merge -q --no-ff -m merge pr
+  mkdir -p "$work/$1-out"
+  (cd "$repo" && HEAD_REF=$2 GITHUB_OUTPUT=$work/$1-out/base bash "$scripts/perf-ir-base.sh" > /dev/null)
+  (cd "$repo" && BASE_SHA=$(out "$1" sha) BASE_PACKAGE=$(out "$1" package) OUT=$work/$1-out IR_ROOT=$work/$1-ir \
+    bash "$scripts/perf-ir.sh" trees > "$work/$1-out/trees.log")
+}
+# A core change since its tag that the SDK's bench relies on: a dependency
+# bump and a workload the tagged core may not run.
+core_api() {
+  set_json "$1/$wasm/package.json" '.devDependencies = {"x": "^2.0.0"}'
+  echo "WORKLOADS['added-since-tag'] = WORKLOADS.keygen;" >> "$1/packages/cachekit/bench/ir-workload.mjs"
+}
+out() { sed -n "s/^$2=//p" "$work/$1-out/base"; }
+assert() { # assert <label> <command>...: passes when the command succeeds
+  if "${@:2}"; then echo "ok: $1"; else
+    echo "FAIL: $1"
+    fail=1
+  fi
+}
+head_outside_core() { # the base's files outside the core are the PR merge commit's
+  [[ -n $(out "$1" package) ]] && git -C "$work/$1-ir/base" diff --quiet "$(g "$work/$1" rev-parse HEAD)" -- . ":!$wasm/"
+}
+warned() { grep -q "dependencies changed since its tag" "$work/$1-out/trees.log"; }
+not_warned() { ! warned "$1"; }
+other_hash() { [[ $(hash "$work/$1-ir/base") != "$(hash "$work/$1-ir/head")" ]]; }
+from_tag() { [[ $(cat "$work/$1-ir/base/$3") == "$(g "$origin" show "$2:$3")" ]]; }
+absent() { [[ ! -e $work/$1-ir/base/$2 ]]; }
+# The base's core package.json: the head's dependency fields, the rest the tag's.
+manifest_mix() {
+  local fields='["dependencies", "devDependencies", "optionalDependencies", "peerDependencies"]'
+  local deps="with_entries(select(.key | IN(${fields}[])))" rest="with_entries(select(.key | IN(${fields}[]) | not))"
+  local base=$work/$1-ir/base/$wasm/package.json head=$work/$1-ir/head/$wasm/package.json
+  [[ $(jq -c "$deps" "$base") == "$(jq -c "$deps" "$head")" &&
+    $(jq -c "$rest" "$base") == "$(g "$origin" show "$wasm_tag:$wasm/package.json" | jq -c "$rest")" ]]
+}
+new_on_base() { ! grep -qx "$2" "$work/$1-out/base-workloads.txt" && grep -qx "$2" "$work/$1-out/head-workloads.txt"; }
+whole_tag() { [[ -z $(out "$1" package) && $(out "$1" sha) == "$(g "$origin" rev-parse "$2^{commit}")" ]] && from_tag "$@"; }
+first_parent() { [[ $(out "$1" sha) == "$(g "$work/$1" rev-parse HEAD^1)" && -z $(out "$1" package) ]]; }
+rp=release-please--branches--main--components--
+
+pick core-sdk-drift "${rp}cachekit-core-wasm" - "$sdk"
+assert "core release PR: outside the core's package, the base is the head" head_outside_core core-sdk-drift
+assert 'core release PR: no dependency warning when the dependencies are unchanged' not_warned core-sdk-drift
+
+DRIFT=core_api pick core-drift "${rp}cachekit-core-wasm" - "$sdk" "$glue" "$added"
+assert "core release PR: the core's drift since its tag hashes apart" other_hash core-drift
+assert "core release PR: the base's core file is the core tag's" from_tag core-drift "$wasm_tag" "$glue"
+assert 'core release PR: a core file added since the tag is absent from the base' absent core-drift "$added"
+assert "core release PR: the base's core package.json is the tag's with the head's dependencies" manifest_mix core-drift
+assert 'core release PR: a dependency change since the tag is warned about' warned core-drift
+assert 'core release PR: a workload added since the tag reads new' new_on_base core-drift added-since-tag
+
+pick sdk-change "${rp}cachekit-core-wasm" "$sdk" "$sdk"
+assert "a core release branch's PR that changes the SDK: the base is the whole core tag tree" \
+  whole_tag sdk-change "$wasm_tag" "$sdk"
+
+pick sdk-release "${rp}cachekit" - "$sdk" "$glue"
+assert 'SDK release PR: the base is the whole SDK tag tree' whole_tag sdk-release "$sdk_tag" "$sdk"
+
+pick feature feature/x "$sdk" "$sdk"
+assert 'feature PR: the base is HEAD^1, whole' first_parent feature
 exit "$fail"

.github/scripts/perf-ir.sh

@@ -11,8 +11,21 @@
 # $OUT/same-build when both builds hash alike, and exits with ir.mjs's code.
 # CI restores the Rust cache into the trees between the two.
 #
-#   BASE_SHA=<sha> OUT=<dir> bash .github/scripts/perf-ir.sh trees
-#   OUT=<dir> [IR_ARGS='--shard 1/4'] bash .github/scripts/perf-ir.sh measure
+#   BASE_SHA=<sha> [BASE_PACKAGE=<path>] OUT=<dir> bash .github/scripts/perf-ir.sh trees
+#   OUT=<dir> [BASE_PACKAGE=<path>] [IR_ARGS='--shard 1/4'] bash .github/scripts/perf-ir.sh measure
+#
+# With BASE_PACKAGE (a core's release PR, from perf-ir-base.sh), the base is
+# the head with only that package checked out at BASE_SHA, its files the
+# release lacks removed. Its package.json keeps the head's dependency fields
+# and dependenciesMeta, which the head's pnpm-lock.yaml records and the frozen
+# install checks, so a bump since the tag does not
+# fail the base's frozen install; both trees then build the core with the
+# same dependencies, so that drift goes unmeasured here (the PR that bumped it
+# was measured). The base's workload list still comes from BASE_SHA's bench
+# file, so a workload added since the tag reads `new` where the tagged core
+# cannot run it. If the head's SDK imports a core export by name that the tag
+# lacks, the base's build fails and every workload reads INCOMPLETE: the SDK
+# reads a new core export off the module namespace instead.
 #
 # IR_ARGS goes to both ir.mjs runs after --jobs 4, word-split: --shard, or
 # --only, --repeats and a larger --jobs for a quick local try.
@@ -49,9 +62,23 @@ if [[ ${1:-} == trees ]]; then
     git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA"
   git worktree add --quiet --detach "$root/head" HEAD
   git worktree add --quiet --detach "$root/base" "$BASE_SHA"
-  # Listed before the head's bench/ replaces the base's.
+  # Listed from BASE_SHA itself, before the core swap below and before the
+  # head's bench/ replaces the base's.
   workloads "$root/base" > "$OUT/base-workloads.txt"
   workloads "$root/head" > "$OUT/head-workloads.txt"
+  if [[ -n ${BASE_PACKAGE:-} ]]; then
+    git -C "$root/base" checkout --quiet --detach "$(git rev-parse HEAD)"
+    git -C "$root/base" checkout --quiet --no-overlay "$BASE_SHA" -- "$BASE_PACKAGE"
+    manifest=$BASE_PACKAGE/package.json
+    fields='["dependencies", "devDependencies", "optionalDependencies", "peerDependencies", "dependenciesMeta"]'
+    deps="with_entries(select(.key | IN(${fields}[])))"
+    if [[ $(jq -c "$deps" "$root/base/$manifest") != "$(jq -c "$deps" "$root/head/$manifest")" ]]; then
+      echo "::warning::$manifest's dependencies changed since its tag: the base builds the tagged core with the head's, so this PR does not measure that change"
+      jq --slurpfile head "$root/head/$manifest" "with_entries(select(.key | IN(${fields}[]) | not)) + (\$head[0] | $deps)" \
+        "$root/base/$manifest" > "$OUT/package.json"
+      mv "$OUT/package.json" "$root/base/$manifest"
+    fi
+  fi
   rm -rf "${root:?}/base/packages/cachekit/bench"
   cp -R "$root/head/packages/cachekit/bench" "$root/base/packages/cachekit/bench"
   exit 0
@@ -63,6 +90,10 @@ fi
 read -ra extra <<< "${IR_ARGS:-}"
 new=$(comm -13 <(sort "$OUT/base-workloads.txt") <(sort "$OUT/head-workloads.txt") | paste -sd, -)
 
+# A failed base build leaves no report, and the comment then reads only that
+# the shard left none; say why while the log still shows it.
+[[ -z ${BASE_PACKAGE:-} ]] ||
+  trap 'echo "::error::the base (this PR with $BASE_PACKAGE at its last tag) failed to install or build; one known cause: the SDK imports by name a core export the tag lacks"' EXIT
 for tree in base head; do
   echo "::group::Build the $tree"
   (
@@ -78,6 +109,7 @@ for tree in base head; do
     pnpm --filter @cachekit-io/cachekit-core-wasm build:wasm
   )
   echo "::endgroup::"
+  trap - EXIT
 done
 
 ir() { (cd "$root/$1/packages/cachekit" && node bench/ir.mjs "${@:2}"); }

.github/workflows/ci.yml

@@ -23,8 +23,9 @@ jobs:
         run: bash .github/scripts/verify-tarball-selftest.sh
 
       # The instruction-count gate skips measuring when base and head hash
-      # alike: the hash must move when any file a workload loads changes.
-      - name: Instruction-count build hash self-test
+      # alike: the hash must move when any file a workload loads changes, and
+      # each kind of PR must get its base.
+      - name: Instruction-count build hash and base self-test
         run: bash .github/scripts/perf-ir-hash-selftest.sh
 
       # The release-order gate, run against a stubbed GitHub API: it must

.github/workflows/instruction-counts.yml

@@ -6,8 +6,10 @@ name: Instruction counts
 # Instruction counts move with the machine, Node and valgrind, so nothing is
 # compared against a figure recorded elsewhere. A release-please PR is
 # measured against that package's last release tag instead
-# (.github/scripts/perf-ir-base.sh). The verdict per workload goes to one PR
-# comment, edited in place.
+# (.github/scripts/perf-ir-base.sh); a core's release PR takes only that
+# core's package from the tag, so it measures the core's code, not SDK drift
+# (two exceptions: .github/scripts/perf-ir.sh).
+# The verdict per workload goes to one PR comment, edited in place.
 #
 # The shards split the workloads (balanced on bench/ir-seconds.json), each
 # building both trees, so the check finishes inside 30 minutes. The base job picks the base once, so every
@@ -40,6 +42,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       sha: ${{ steps.base.outputs.sha }}
+      package: ${{ steps.base.outputs.package }}
       label: ${{ steps.base.outputs.label }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -79,6 +82,7 @@ jobs:
       - name: Check out the base and head trees
         env:
           BASE_SHA: ${{ needs.base.outputs.sha }}
+          BASE_PACKAGE: ${{ needs.base.outputs.package }}
         run: bash .github/scripts/perf-ir.sh trees
 
       - name: Setup Rust
@@ -126,6 +130,7 @@ jobs:
       # 0.03%, far under the 0.4% A/A limit.
       - name: Gate the PR against its base
         env:
+          BASE_PACKAGE: ${{ needs.base.outputs.package }}
           IR_ARGS: --shard ${{ matrix.shard }}/${{ env.SHARDS }} --repeats 3
         run: bash .github/scripts/perf-ir.sh measure