diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index b8ca1ba9d..c8bee938e 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -63,14 +63,17 @@ CI also lints those scripts and runs their transport tests; lint changed ones ex Rust default gates are `bun run check:local` and `bun run test:local`. Add `--all-features` to equivalent Cargo commands for feature changes; the toolchain Rust tasks already enable all features. Keep default coverage. -Candidate CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per +Candidate and config-compiler CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per test and zero retries, then `cargo test --doc` with the same feature selection. Explicit target selections (`--lib`, `--test `, etc.) skip doctests, matching Cargo; explicit `--doc` uses Cargo alone. Pass Cargo build, feature and target options through this wrapper; use `cargo nextest` directly for nextest-specific options such as `-E`. This prevents another test's spawned child from inheriting a briefly non-CLOEXEC -macOS descriptor and delaying EOF. `bun run test:local` probes `cargo nextest +macOS descriptor and delaying EOF. `bun run test:local` and `bun run test:config-compiler` share the isolating runner. +A name filter matching nothing exits 4 with pinned nextest 0.9.151, whereas +stock Cargo exits 0 with zero tests. The wrapper preserves that nonzero result. +`bun run test:local` probes `cargo nextest --version`, matching the subcommand it executes. A missing or different version prints one fallback line and retains `cargo test` (one libtest thread on macOS), leaving the installed nextest alone. CI and qualification reports/repetitions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 602a4e469..b6dd860d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,9 +254,25 @@ jobs: run: bun -e 'if (process.platform !== process.env.EXPECTED_PLATFORM || process.arch !== process.env.EXPECTED_ARCH) throw new Error("Unexpected compiler release host");' # TypeScript-only pull requests cannot change the compiler crate, which # reads nothing outside packages/config-compiler. + - name: Install pinned process-per-test runner + if: ${{ needs.changes.outputs.compiler_rust == 'true' }} + shell: bash + run: | + bash scripts/install-cargo-nextest.sh "$RUNNER_TEMP/compiler-nextest" + echo "$RUNNER_TEMP/compiler-nextest" >> "$GITHUB_PATH" - name: Check and test the compiler crate if: ${{ needs.changes.outputs.compiler_rust == 'true' }} + env: + HACK_RUST_TEST_RUNNER: nextest + HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/compiler-rust-results run: bun run check:config-compiler && bun run test:config-compiler + - name: Preserve compiler test inventory and results + if: ${{ always() && needs.changes.outputs.compiler_rust == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: compiler-rust-results-${{ matrix.os }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/compiler-rust-results + if-no-files-found: error - run: bun run build:config-compiler # Intentionally permanent: the owner test emits closed stage tokens only, never values. - name: Test private file sources and owner with the matching compiler diff --git a/package.json b/package.json index b1af0b31a..a3db09693 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "build:local": "sh scripts/build-hack-local.sh", "build:config-compiler": "bun scripts/build-config-compiler.ts", "check:config-compiler": "cargo fmt --manifest-path packages/config-compiler/Cargo.toml --check && cargo clippy --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --all-targets --jobs 2 -- -D warnings", - "test:config-compiler": "cargo test --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --jobs 2", + "test:config-compiler": "bash scripts/test-rust.sh --crate=config-compiler", "test:local": "bash scripts/test-runtime-core.sh", "check:native-guest": "sh scripts/check-native-guest.sh", "check:local": "cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check && cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings", diff --git a/packages/config-compiler/README.md b/packages/config-compiler/README.md index 05254f907..a7c16bfd9 100644 --- a/packages/config-compiler/README.md +++ b/packages/config-compiler/README.md @@ -79,9 +79,21 @@ Use Rust 1.97.1 with the locked dependencies. From the repository root: ```sh cargo +1.97.1 fmt --manifest-path packages/config-compiler/Cargo.toml --check cargo +1.97.1 clippy --locked --manifest-path packages/config-compiler/Cargo.toml --all-targets -- -D warnings -cargo +1.97.1 test --locked --manifest-path packages/config-compiler/Cargo.toml +bun run test:config-compiler ``` +The test command shares the [runtime test runner](../runtime-core/README.md) +and uses pinned cargo-nextest 0.9.151: each test gets its own process, zero retries, +then Cargo runs doctests separately. This also isolates the protocol tests that +spawn children with piped stdin from concurrent tests' descriptor inheritance on +macOS. CI requires the pin on every compiler host. Locally, missing or other +nextest versions fall back to Cargo (one libtest thread on macOS), with an optional +pinned install command in the fallback line; the runner never replaces an install. + +An unmatched name filter exits **4** with pinned nextest 0.9.151 ("no tests to run"), +where stock Cargo exits **0** after running zero tests. Check filter spelling; +this wrapper keeps nextest's nonzero result, including in CI. + ## Metadata-only environment planning `compile` and `resolve` return a separate `declared_workloads` map from every diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 677ff280b..2dad33a0d 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -570,6 +570,11 @@ The wrapper accepts Cargo build, feature and target options; nextest-specific options such as `-E` need a direct `cargo nextest` invocation. Documentation-only selection refuses repetition or report settings intended for nextest qualification. +An unmatched name filter exits **4** with pinned nextest 0.9.151 ("no tests to run"), +where stock Cargo exits **0** after running zero tests. Check filter spelling; +this wrapper keeps nextest's nonzero result, including in CI. + + `bun run test:local` checks `cargo nextest --version`, matching Cargo's lookup for the test run. If that resolves the pin, it uses nextest; a missing or different version prints one fallback line and uses stock `cargo test`, leaving the diff --git a/scripts/install-cargo-nextest.sh b/scripts/install-cargo-nextest.sh index e8500018c..b9e0835c9 100755 --- a/scripts/install-cargo-nextest.sh +++ b/scripts/install-cargo-nextest.sh @@ -16,6 +16,10 @@ case "$(uname -s)-$(uname -m)" in target=x86_64-unknown-linux-gnu checksum=1d4ce302b820c41f4b89c5fb9bce45f0f0c94882c1cf08fc360caf08be6e7df4 ;; + Linux-aarch64|Linux-arm64) + target=aarch64-unknown-linux-gnu + checksum=3e35e8f6913af92558ba09d7155f74e35dd1f0024ef3cd891637433ae4eacbdd + ;; Darwin-arm64|Darwin-x86_64) target=universal-apple-darwin checksum=d1d7d025d4bfc9fb154cd76aea3949bd78534829a0589abefa6e5dd4edb135dd diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh index 8d617307a..e9f24d30c 100755 --- a/scripts/test-runtime-core.sh +++ b/scripts/test-runtime-core.sh @@ -1,93 +1,3 @@ #!/usr/bin/env bash set -euo pipefail -root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P) -cd "$root" -version=$(bash scripts/install-cargo-nextest.sh --print-version) -install="cargo install --locked --version $version cargo-nextest" -runner=${HACK_RUST_TEST_RUNNER:-auto} -repetitions=${HACK_RUST_TEST_REPETITIONS:-1} -case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac -case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac -common=(--locked --manifest-path packages/runtime-core/Cargo.toml --target-dir "${HACK_RUST_TEST_TARGET_DIR:-.hack-local/target}") -# Cargo's subcommand lookup can differ from a direct PATH lookup. Probe the -# same command used below, and leave a contributor's other nextest version alone. -actual=$(cargo nextest --version 2>/dev/null) || actual="" -case "$actual" in - "cargo-nextest $version"|"cargo-nextest $version ("*) ;; - *) - if [[ "$runner" == nextest || "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then - echo "Pinned cargo-nextest $version is required. Install: $install" >&2 - exit 2 - fi - echo "cargo nextest $version unavailable; using cargo test." >&2 - # A single libtest thread avoids cross-test spawn/close races in the fallback. - if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi - exec cargo test "${common[@]}" --jobs 2 "$@" - ;; -esac -mode=default -run_doctests=true -doc_only=false -for argument in "$@"; do - case "$argument" in - --) break ;; - --all-features) mode=all-features ;; - --doc) doc_only=true ;; - --lib|--bins|--bin|--bin=*|--tests|--test|--test=*|--examples|--example|--example=*|--benches|--bench|--bench=*|--all-targets) - run_doctests=false ;; - esac -done -# Documentation-only selection has no libtest process for nextest to isolate. -if [[ "$doc_only" == true ]]; then - if [[ "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then - echo 'Documentation-only selection cannot produce nextest qualification evidence.' >&2 - exit 2 - fi - exec cargo test "${common[@]}" --jobs 2 "$@" -fi -if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then - mkdir -p "$HACK_RUST_TEST_REPORT_DIR" - # JUnit contains executed tests; keep discovery separately to count ignored - # fixtures and compare names with the stock libtest inventory. - cargo nextest list "${common[@]}" --build-jobs 2 --config-file "$root/.config/nextest.toml" \ - --message-format json "$@" > "$HACK_RUST_TEST_REPORT_DIR/$mode-inventory.json" - cargo nextest show-config test-groups "${common[@]}" --build-jobs 2 \ - --config-file "$root/.config/nextest.toml" --profile ci --groups host-disk-probes "$@" \ - > "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt" - if [[ "$(uname -s)" == Darwin ]] && ! grep -q 'provider::lifecycle::admission_pool::tests::' "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"; then - echo 'The macOS disk-probe tests are missing their serial nextest group.' >&2 - exit 2 - fi - cargo test "${common[@]}" --jobs 2 "$@" -- --list > "$HACK_RUST_TEST_REPORT_DIR/$mode-libtest.txt" - cargo test "${common[@]}" --jobs 2 "$@" -- --list --ignored > "$HACK_RUST_TEST_REPORT_DIR/$mode-ignored.txt" -fi -for ((iteration=1; iteration<=repetitions; iteration++)); do - echo "Rust $mode suite $iteration/$repetitions: process per test, retries=0" - if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then - # nextest's store is relative to Cargo's workspace (this standalone crate), - # independently of the shared Cargo --target-dir. - source="$root/packages/runtime-core/target/nextest/ci/junit.xml" - # This is nextest's regenerable result, not a user file. A build failure - # must never copy a previous successful iteration as the failed one's proof. - rm -f -- "$source" - fi - status=0 - cargo nextest run "${common[@]}" --build-jobs 2 \ - --config-file "$root/.config/nextest.toml" --profile ci --retries 0 "$@" || status=$? - if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then - if [[ -f "$source" ]]; then - mkdir -p "$HACK_RUST_TEST_REPORT_DIR" - report="$HACK_RUST_TEST_REPORT_DIR/$mode-$iteration.xml" - if [[ -e "$report" ]]; then echo 'Rust report already exists; refusing to replace it.' >&2; exit 2; fi - cp -n "$source" "$report" - elif [[ "$status" == 0 ]]; then - echo 'Nextest succeeded without its required JUnit evidence.' >&2 - exit 2 - fi - fi - # Never reset a failing streak, retry, or continue to another suite after failure. - if [[ "$status" != 0 ]]; then exit "$status"; fi - # Nextest excludes doctests. Cargo excludes them for explicit target selection - # too, and rejects --doc combined with those selectors. - if [[ "$run_doctests" == true ]]; then cargo test --doc "${common[@]}" --jobs 2 "$@"; fi -done +exec bash "$(dirname -- "$0")/test-rust.sh" --crate=runtime-core "$@" diff --git a/scripts/test-rust.sh b/scripts/test-rust.sh new file mode 100644 index 000000000..ddc757056 --- /dev/null +++ b/scripts/test-rust.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P) +cd "$root" +crate=runtime-core +case "${1-}" in + --crate=runtime-core|--crate=config-compiler) crate=${1#--crate=}; shift ;; + --crate=*) echo 'Unsupported Rust test crate.' >&2; exit 2 ;; +esac +case "$crate" in + runtime-core) target=.hack-local/target ;; + config-compiler) target=.hack-local/config-compiler-target ;; +esac +version=$(bash scripts/install-cargo-nextest.sh --print-version) +install="cargo install --locked --version $version cargo-nextest" +runner=${HACK_RUST_TEST_RUNNER:-auto} +repetitions=${HACK_RUST_TEST_REPETITIONS:-1} +case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac +case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac +common=(--locked --manifest-path "packages/$crate/Cargo.toml" --target-dir "${HACK_RUST_TEST_TARGET_DIR:-$target}") +# Cargo's subcommand lookup can differ from a direct PATH lookup. Probe the +# same command used below, and leave a contributor's other nextest version alone. +actual=$(cargo nextest --version 2>/dev/null) || actual="" +case "$actual" in + "cargo-nextest $version"|"cargo-nextest $version ("*) ;; + *) + if [[ "$runner" == nextest || "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + echo "Pinned cargo-nextest $version is required. Install: $install" >&2 + exit 2 + fi + echo "cargo nextest $version unavailable; using cargo test. Optional install: $install" >&2 + # A single libtest thread avoids cross-test spawn/close races in the fallback. + if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi + exec cargo test "${common[@]}" --jobs 2 "$@" + ;; +esac +mode=default +run_doctests=true +doc_only=false +for argument in "$@"; do + case "$argument" in + --) break ;; + --all-features) mode=all-features ;; + --doc) doc_only=true ;; + --lib|--bins|--bin|--bin=*|--tests|--test|--test=*|--examples|--example|--example=*|--benches|--bench|--bench=*|--all-targets) + run_doctests=false ;; + esac +done +# Documentation-only selection has no libtest process for nextest to isolate. +if [[ "$doc_only" == true ]]; then + if [[ "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + echo 'Documentation-only selection cannot produce nextest qualification evidence.' >&2 + exit 2 + fi + exec cargo test "${common[@]}" --jobs 2 "$@" +fi +if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + mkdir -p "$HACK_RUST_TEST_REPORT_DIR" + # JUnit contains executed tests; keep discovery separately to count ignored + # fixtures and compare names with the stock libtest inventory. + cargo nextest list "${common[@]}" --build-jobs 2 --config-file "$root/.config/nextest.toml" \ + --message-format json "$@" > "$HACK_RUST_TEST_REPORT_DIR/$mode-inventory.json" + cargo nextest show-config test-groups "${common[@]}" --build-jobs 2 \ + --config-file "$root/.config/nextest.toml" --profile ci --groups host-disk-probes "$@" \ + > "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt" + if [[ "$crate" == runtime-core && "$(uname -s)" == Darwin ]] && ! grep -q 'provider::lifecycle::admission_pool::tests::' "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"; then + echo 'The macOS disk-probe tests are missing their serial nextest group.' >&2 + exit 2 + fi + cargo test "${common[@]}" --jobs 2 "$@" -- --list > "$HACK_RUST_TEST_REPORT_DIR/$mode-libtest.txt" + cargo test "${common[@]}" --jobs 2 "$@" -- --list --ignored > "$HACK_RUST_TEST_REPORT_DIR/$mode-ignored.txt" +fi +for ((iteration=1; iteration<=repetitions; iteration++)); do + echo "Rust $mode suite $iteration/$repetitions: process per test, retries=0" + if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + # nextest's store is relative to Cargo's workspace (this standalone crate), + # independently of the shared Cargo --target-dir. + source="$root/packages/$crate/target/nextest/ci/junit.xml" + # This is nextest's regenerable result, not a user file. A build failure + # must never copy a previous successful iteration as the failed one's proof. + rm -f -- "$source" + fi + status=0 + cargo nextest run "${common[@]}" --build-jobs 2 \ + --config-file "$root/.config/nextest.toml" --profile ci --retries 0 "$@" || status=$? + if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + if [[ -f "$source" ]]; then + mkdir -p "$HACK_RUST_TEST_REPORT_DIR" + report="$HACK_RUST_TEST_REPORT_DIR/$mode-$iteration.xml" + if [[ -e "$report" ]]; then echo 'Rust report already exists; refusing to replace it.' >&2; exit 2; fi + cp -n "$source" "$report" + elif [[ "$status" == 0 ]]; then + echo 'Nextest succeeded without its required JUnit evidence.' >&2 + exit 2 + fi + fi + # Never reset a failing streak, retry, or continue to another suite after failure. + if [[ "$status" != 0 ]]; then exit "$status"; fi + # Nextest excludes doctests. Cargo excludes them for explicit target selection + # too, and rejects --doc combined with those selectors. + if [[ "$run_doctests" == true ]]; then cargo test --doc "${common[@]}" --jobs 2 "$@"; fi +done diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts index 4fc0dbaec..282180ff2 100644 --- a/tests/runtime-core-test-runner.test.ts +++ b/tests/runtime-core-test-runner.test.ts @@ -19,6 +19,7 @@ afterEach(async () => { }); async function fixture( opts: { + crate?: "runtime-core" | "config-compiler"; nextest?: boolean; platform?: string; fail?: string; @@ -34,7 +35,11 @@ async function fixture( const tools = join(root, "bin"); await mkdir(tools); await mkdir(join(root, "scripts")); - for (const file of ["test-runtime-core.sh", "install-cargo-nextest.sh"]) { + for (const file of [ + "test-runtime-core.sh", + "test-rust.sh", + "install-cargo-nextest.sh", + ]) { await copyFile(resolve("scripts", file), join(root, "scripts", file)); } const log = join(root, "commands"); @@ -80,7 +85,10 @@ async function fixture( FIXTURE_CARGO_VERSION: opts.version ?? version, FIXTURE_DIRECT_LOOKUP: join(root, "direct-lookup"), FIXTURE_REPORT: opts.report - ? join(root, "packages/runtime-core/target/nextest/ci/junit.xml") + ? join( + root, + `packages/${opts.crate ?? "runtime-core"}/target/nextest/ci/junit.xml` + ) : undefined, FIXTURE_FAIL: opts.fail, HACK_RUST_TEST_RUNNER: opts.required ? "nextest" : "auto", @@ -93,7 +101,15 @@ async function fixture( }); const run = async (args: string[] = []) => { const child = Bun.spawn( - ["/bin/bash", join(root, "scripts/test-runtime-core.sh"), ...args], + [ + "/bin/bash", + join( + root, + opts.crate ? "scripts/test-rust.sh" : "scripts/test-runtime-core.sh" + ), + ...(opts.crate ? [`--crate=${opts.crate}`] : []), + ...args, + ], { env, stdin: "ignore", stdout: "pipe", stderr: "pipe", timeout: 5000 } ); const [code, stdout, stderr] = await Promise.all([ @@ -204,7 +220,7 @@ for (const platform of ["Linux", "Darwin"]) { expect(r.commands[0]).toStartWith("test --locked"); expect(r.commands[0]).toContain("client_relay"); expect(r.stderr.trim()).toBe( - `cargo nextest ${r.version} unavailable; using cargo test.` + `cargo nextest ${r.version} unavailable; using cargo test. Optional install: cargo install --locked --version ${r.version} cargo-nextest` ); expect(r.threads).toBe(platform === "Darwin" ? "1" : ""); }); @@ -234,7 +250,7 @@ test("another contributor nextest version falls back without replacing it on eit expect(r.commands[0]).toStartWith("test --locked"); expect(r.commands[0]).toEndWith("--lib --all-features"); expect(r.stderr.trim()).toBe( - `cargo nextest ${r.version} unavailable; using cargo test.` + `cargo nextest ${r.version} unavailable; using cargo test. Optional install: cargo install --locked --version ${r.version} cargo-nextest` ); expect(r.threads).toBe(platform === "Darwin" ? "1" : ""); } @@ -337,3 +353,31 @@ test("the pinned installer refuses corrupt download bytes before extraction or e expect(`${stdout}${stderr}`).toContain("FAILED"); expect(await Bun.file(extracted).exists()).toBe(false); }); + +for (const platform of ["Linux", "Darwin"]) { + test(`compiler uses process isolation and its own report store on ${platform}`, async () => { + const f = await fixture({ + crate: "config-compiler", + platform, + report: true, + required: true, + }); + const r = await f.run(); + expect(r.code, r.stderr).toBe(0); + for (const command of r.commands) { + expect(command).toContain( + "--manifest-path packages/config-compiler/Cargo.toml" + ); + expect(command).not.toContain("packages/runtime-core"); + } + expect(r.commands.filter((c) => c.startsWith("nextest run"))).toHaveLength( + 1 + ); + expect(r.commands.filter((c) => c.startsWith("test --doc"))).toHaveLength( + 1 + ); + expect(await Bun.file(join(f.root, "reports/default-1.xml")).text()).toBe( + "\n" + ); + }); +}