diff --git a/.github/scripts/perf-ir-base.sh b/.github/scripts/perf-ir-base.sh index 31290891..010be63d 100644 --- a/.github/scripts/perf-ir-base.sh +++ b/.github/scripts/perf-ir-base.sh @@ -1,23 +1,35 @@ #!/usr/bin/env bash # Pick the commit the instruction-count gate measures a pull request against, -# and write it to $GITHUB_OUTPUT as `sha` (what to build) and `label` (how the -# comment names it). Run from a checkout of the PR merge commit with its first -# parent. +# and write it to $GITHUB_OUTPUT as `sha` (what to build), `package` (empty, or +# the one package the base takes from `sha`) and `label` (how the comment names +# it). Run from a checkout of the PR merge commit with its first parent. # # A feature PR is measured against its base, the merge commit's first parent. # A release PR (head branch release-please--branches--main--components--) # is rebased onto main on every push, so against main it would measure only a # version bump. It is measured against that package's last release instead: # tag -v, where is the package's entry in the base's -# .release-please-manifest.json. The comment then shows every workload's drift -# since that release. A tag that cannot be fetched fails the job; it never -# falls back to main, which would read PASS without measuring anything. +# .release-please-manifest.json. A tag that cannot be fetched fails the job; it +# never falls back to main, which would read PASS without measuring anything. +# +# The SDK's release (packages/cachekit) is measured against the whole tagged +# tree, so its comment shows every workload's drift since that release. A +# core's release ships only that core's package, so its base is the PR's own +# tree with only that package taken from the tag (perf-ir.sh trees): every +# workload, the SDK ones that run through the core included, then measures the +# core's code since its tag, with the two exceptions perf-ir.sh's header names +# (dependency drift, and SDK imports of core exports the tag lacks). SDK drift +# since the core's tag is the SDK's own release's to gate, against the SDK's +# tag. Anyone can name a branch, so a PR gets that +# base only when it changes nothing outside the core's package and the +# manifest; any other PR on such a branch gets the whole tagged tree. # # CI: HEAD_REF=${{ github.head_ref }} bash .github/scripts/perf-ir-base.sh # Local: HEAD_REF=release-please--branches--main--components--cachekit \ # GITHUB_OUTPUT=/dev/stdout bash .github/scripts/perf-ir-base.sh set -euo pipefail +package= if [[ ${HEAD_REF:-} != release-please--*--components--* ]]; then sha=$(git rev-parse HEAD^1) label=$sha @@ -39,9 +51,20 @@ else } sha=$(git rev-parse "$tag^{commit}") label="$tag ($(git rev-parse --short=7 "$sha")), last release" + if [[ $path != packages/cachekit ]]; then + outside=$(git diff --name-only HEAD^1 HEAD -- . ":!$path/" ':!.release-please-manifest.json' | paste -sd, -) + if [[ -z $outside ]]; then + package=$path + label="this PR with $path at $label" + else + echo "::notice::release PR changes files outside $path ($outside), so it is measured against the whole $tag tree" + label="$label, whole tree: the PR changes files outside $path" + fi + fi fi echo "Base: $label" { echo "sha=$sha" + echo "package=$package" echo "label=$label" } >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/perf-ir-hash-selftest.sh b/.github/scripts/perf-ir-hash-selftest.sh index 99dffe67..d515fbe2 100755 --- a/.github/scripts/perf-ir-hash-selftest.sh +++ b/.github/scripts/perf-ir-hash-selftest.sh @@ -1,12 +1,23 @@ #!/usr/bin/env bash +# shellcheck disable=SC2329 # the checks run through assert and $DRIFT # Self-test for `bench/ir.mjs --build-hash`, on a synthetic three-package tree. # The instruction-count gate skips measuring when the base and head hashes # match, so a file the workloads load that the hash misses lets a regression # pass unmeasured. The hash must move when a shipped file of any package or # the lockfile changes, stay put when an unshipped file changes, and not -# depend on where the tree sits. Needs node; no build, no valgrind. +# depend on where the tree sits. +# +# Then the base the gate builds (perf-ir-base.sh, then perf-ir.sh trees), on a +# synthetic repo of that tree. On a core's release PR, the base must be the +# PR's own tree outside the core's package, so SDK drift since the core's tag +# cannot fail the gate, and the core's own drift must hash apart, measured +# against the tagged core. A PR on a +# core's release branch that changes the SDK, the SDK's release PR and a +# feature PR keep the whole-tree bases: the tag, and HEAD^1. +# Needs node, git and jq; no build, no valgrind. set -euo pipefail -src=$(cd "$(dirname "$0")/../../packages/cachekit/bench" && pwd) +scripts=$(cd "$(dirname "$0")" && pwd) +src=$(cd "$scripts/../../packages/cachekit/bench" && pwd) work=$(mktemp -d) trap 'rm -rf "${work:?}"' EXIT @@ -75,4 +86,111 @@ if hash "$work/a" > /dev/null 2>&1; then else echo "ok: a files glob that matches nothing fails the run" fi + +g() { git -C "$1" -c user.name=selftest -c user.email=selftest@example.invalid -c commit.gpgsign=false \ + -c core.hooksPath=/dev/null "${@:2}"; } +# The released repo: core-wasm's release, then a later SDK release that also +# changed the core, so each tag names a tree of its own. +origin=$work/origin +tree "$origin" +cp "$scripts/../../release-please-config.json" "$scripts/../../.release-please-manifest.json" "$origin/" +sdk_tag=cachekit-v$(jq -r '.["packages/cachekit"]' "$origin/.release-please-manifest.json") +wasm_tag=cachekit-core-wasm-v$(jq -r '.["packages/cachekit-core-wasm"]' "$origin/.release-please-manifest.json") +sdk=packages/cachekit/dist/index.js glue=packages/cachekit-core-wasm/pkg/glue.js +added=packages/cachekit-core-wasm/pkg/added.js wasm=packages/cachekit-core-wasm +g "$origin" init -q -b main +g "$origin" add -A +g "$origin" commit -q -m 'core-wasm release' +g "$origin" tag "$wasm_tag" +echo '// SDK release' | tee -a "$origin/$sdk" >> "$origin/$glue" +g "$origin" commit -q -am 'SDK release' +g "$origin" tag "$sdk_tag" + +# set_json +set_json() { jq "$2" "$1" > "$1.new" && mv "$1.new" "$1"; } +# pick ...: append to each +# on main after the releases (and run $DRIFT on the repo, if set), then open +# the PR. A release PR edits what release-please edits: the package's version, +# its CHANGELOG.md and its manifest entry. The PR also appends to . +# Check out its merge commit, and pick and build the base trees as CI does. +pick() { + local repo=$work/$1 path + git clone -q --no-tags "file://$origin" "$repo" + for f in "${@:4}"; do echo '// drift' >> "$repo/$f"; done + [[ -z ${DRIFT:-} ]] || "$DRIFT" "$repo" + g "$repo" add -A + g "$repo" commit -q -m drift + g "$repo" switch -q -c pr + if [[ $2 == *--components--* ]]; then + path=$(jq -r --arg c "${2##*--components--}" \ + '.packages | to_entries[] | select(.value.component == $c) | .key' "$repo/release-please-config.json") + set_json "$repo/$path/package.json" '.version = "9.9.9"' + set_json "$repo/.release-please-manifest.json" ".[\"$path\"] = \"9.9.9\"" + echo 'release notes' >> "$repo/$path/CHANGELOG.md" + fi + [[ $3 == - ]] || echo '// the PR' >> "$repo/$3" + g "$repo" add -A + g "$repo" commit -q -m 'the PR' + g "$repo" switch -q --detach main + g "$repo" merge -q --no-ff -m merge pr + mkdir -p "$work/$1-out" + (cd "$repo" && HEAD_REF=$2 GITHUB_OUTPUT=$work/$1-out/base bash "$scripts/perf-ir-base.sh" > /dev/null) + (cd "$repo" && BASE_SHA=$(out "$1" sha) BASE_PACKAGE=$(out "$1" package) OUT=$work/$1-out IR_ROOT=$work/$1-ir \ + bash "$scripts/perf-ir.sh" trees > "$work/$1-out/trees.log") +} +# A core change since its tag that the SDK's bench relies on: a dependency +# bump and a workload the tagged core may not run. +core_api() { + set_json "$1/$wasm/package.json" '.devDependencies = {"x": "^2.0.0"}' + echo "WORKLOADS['added-since-tag'] = WORKLOADS.keygen;" >> "$1/packages/cachekit/bench/ir-workload.mjs" +} +out() { sed -n "s/^$2=//p" "$work/$1-out/base"; } +assert() { # assert