From d7f0c99de0120d2f2a047132008be3f2bb6508f4 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:31:16 +0000 Subject: [PATCH 1/3] fix(ci): isolate Rust tests to prevent macOS descriptor leaks --- .ai/skills/hack-repo-verify/SKILL.md | 10 + .config/nextest.toml | 25 +++ .github/workflows/ci.yml | 30 ++- package.json | 2 +- packages/runtime-core/README.md | 30 +++ scripts/install-cargo-nextest.sh | 37 ++++ scripts/test-runtime-core.sh | 75 ++++++++ tests/runtime-core-test-runner.test.ts | 246 +++++++++++++++++++++++++ 8 files changed, 451 insertions(+), 4 deletions(-) create mode 100644 .config/nextest.toml create mode 100755 scripts/install-cargo-nextest.sh create mode 100755 scripts/test-runtime-core.sh create mode 100644 tests/runtime-core-test-runner.test.ts diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index 331e5aed5..cbc8c1967 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -63,6 +63,16 @@ CI also lints those scripts and runs their transport tests; lint changed ones ex Rust default gates are `bun run check:local` and `bun run test:local`. Add `--all-features` to equivalent Cargo commands for feature changes; the toolchain Rust tasks already enable all features. Keep default coverage. +Candidate CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per +test and zero retries, then `cargo test --doc` with the same feature selection. +This prevents another test's spawned child from inheriting a briefly non-CLOEXEC +macOS descriptor and delaying EOF. `bun run test:local` uses that pin when present; +otherwise it prints `cargo install --locked --version 0.9.151 cargo-nextest` and +retains `cargo test` (one libtest thread on macOS). The nextest test group preserves +the admission-pool tests' serialized host-wide disk probes. A manual CI dispatch +with `rust_repetitions=20` runs 20 consecutive default and 20 all-feature Mac suites, +stopping at the first failure, and preserves per-pass JUnit counts/timings. + `bun run test:local` does not enable `native-stream-relay`, so `stream_relay_contract` reports zero tests. This default pass does not qualify the relay contract. With Zig 0.15.2 available, run the local all-features equivalent: diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000..64056e6ed --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,25 @@ +# Both CI hosts use a separate process for every Rust test. Keep ignored fixtures +# ignored: their parent tests invoke them explicitly with the stock test binary. +[profile.default] +retries = 0 + +[profile.ci] +retries = 0 +fail-fast = true +slow-timeout = { period = "60s", terminate-after = 10 } + +[profile.ci.junit] +path = "junit.xml" + +# The old crate mutex cannot serialize separate processes. Preserve its bound on +# whole-host lsof scans, without serializing unrelated runtime tests. +[test-groups.host-disk-probes] +max-threads = 1 + +[[profile.default.overrides]] +filter = 'test(provider::lifecycle::admission_pool::tests::)' +test-group = 'host-disk-probes' + +[[profile.ci.overrides]] +filter = 'test(provider::lifecycle::admission_pool::tests::)' +test-group = 'host-disk-probes' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef102ebb6..a0daa7c35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,11 @@ on: type: boolean required: true default: true + rust_repetitions: + description: Consecutive macOS Rust qualification passes (stops on first failure) + type: choice + options: ["1", "20"] + default: "1" concurrency: # Let the active next run finish, including jobs waiting for macOS capacity. @@ -372,7 +377,10 @@ jobs: matrix: os: ${{ fromJSON(needs.changes.outputs.defer_macos == 'true' && '["ubuntu-latest"]' || '["ubuntu-latest","macos-latest"]') }} runs-on: ${{ matrix.os }} - timeout-minutes: 45 + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && inputs.rust_repetitions == '20' && 300 || 45 }} + env: + HACK_RUST_TEST_RUNNER: nextest + HACK_RUST_TEST_REPETITIONS: ${{ matrix.os == 'macos-latest' && github.event_name == 'workflow_dispatch' && inputs.rust_repetitions || '1' }} permissions: contents: read steps: @@ -423,14 +431,30 @@ jobs: echo "$destination/zig-${platform}-0.15.2" >> "$GITHUB_PATH" - name: Check candidate formatting run: cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check + - name: Install pinned process-per-test runner + shell: bash + run: | + bash scripts/install-cargo-nextest.sh "$RUNNER_TEMP/candidate-nextest" + echo "$RUNNER_TEMP/candidate-nextest" >> "$GITHUB_PATH" - name: Lint candidate core run: cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings - name: Test candidate isolation contracts - run: cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --jobs 2 + env: + HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/candidate-rust-results + run: bash scripts/test-runtime-core.sh - name: Lint candidate optional features run: cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --all-features --jobs 2 -- -D warnings - name: Test candidate optional feature contracts - run: cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-features --jobs 2 + env: + HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/candidate-rust-results + run: bash scripts/test-runtime-core.sh --all-features + - name: Preserve per-pass Rust counts and timings + if: always() + uses: actions/upload-artifact@v4 + with: + name: candidate-rust-results-${{ matrix.os }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-rust-results/ + if-no-files-found: ignore - name: Build and inspect local candidate run: | ./scripts/build-hack-local.sh diff --git a/package.json b/package.json index e9bfb445d..773a7c2fc 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build:config-compiler": "bun scripts/build-config-compiler.ts", "check:config-compiler": "cargo fmt --manifest-path packages/config-compiler/Cargo.toml --check && cargo clippy --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --all-targets --jobs 2 -- -D warnings", "test:config-compiler": "cargo test --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --jobs 2", - "test:local": "cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --jobs 2", + "test:local": "bash scripts/test-runtime-core.sh", "check:local": "cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check && cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings", "build:runtime-image": "bun run --cwd packages/cli build:runtime-image", "build:node-runtime-image": "bun run --cwd packages/cli build:node-runtime-image", diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 0365d473c..0439833d0 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -555,6 +555,36 @@ turn on no features and do not need Zig. Root `mise.toml` and `.tool-versions` p Zig 0.15.2. The toolchain image installs the `mise.toml` pins and runs its tasks under `mise exec`. CI downloads a checksum-verified 0.15.2 archive. +### Rust test processes + +Both Candidate core CI hosts use **cargo-nextest 0.9.151**, downloaded from its +pinned release and SHA-256 verified, for default and all-feature tests. Each test +runs in its own process: on macOS the two-call close-on-exec setup briefly permits +another test's spawned child to inherit a socket and delay EOF. Retries are zero. +The admission-pool test group retains the old mutex's serialization of whole-host +`lsof` scans. Ignored helpers remain ignored and their owning tests still invoke +them explicitly; `cargo test --doc` runs separately because nextest omits doctests. + +`bun run test:local` uses the same runner when installed. Install its exact pin with +`cargo install --locked --version 0.9.151 cargo-nextest`. Without nextest the command +prints that instruction and retains stock `cargo test`; the macOS fallback sets +one libtest thread to prevent cross-test descriptor inheritance. CI requires the +pin and never silently falls back. `bun run test:local --all-features` retains the +Zig prerequisite above. The toolchain container's existing `rust` task continues +to use stock Cargo on Linux, where the descriptor creation primitives are atomic. + +For bounded hosted qualification, dispatch `ci.yml` at the exact branch/commit +with `full=true` and `rust_repetitions=20`. Only macOS repeats: 20 default suites +and 20 all-feature suites, each followed by doctests. A failed iteration stops +the job immediately; this is a consecutive-pass experiment, not retry policy. +The `candidate-rust-results-*` artifacts retain each pass's counts and duration. +Discovery JSON and stock libtest lists retain ignored-test counts separately: +nextest's JUnit report contains executed tests, not ignored fixtures. The CI profile +terminates a hung test after ten 60-second intervals, above the existing 500-second +fixture budgets, so a hang fails with evidence. Qualification dispatches allow up +to 300 minutes; ordinary Candidate jobs keep their 45-minute limit. Record measured +test-step times as well as the pass count before treating the loop as qualified. + Source binaries embed their build checkout. If a copied Cargo target directory retains a different checkout identity, use a fresh `--target-dir` for checks and tests; Cargo reporting a cached binary as fresh does not qualify that identity. diff --git a/scripts/install-cargo-nextest.sh b/scripts/install-cargo-nextest.sh new file mode 100755 index 000000000..e8500018c --- /dev/null +++ b/scripts/install-cargo-nextest.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Official stable release, verified against the release asset SHA-256 metadata. +version=0.9.151 +if [[ "${1-}" == --print-version ]]; then + printf '%s\n' "$version" + exit 0 +fi +if [[ "$#" != 1 ]]; then + echo 'Usage: bash scripts/install-cargo-nextest.sh DESTINATION' >&2 + exit 2 +fi +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) + target=x86_64-unknown-linux-gnu + checksum=1d4ce302b820c41f4b89c5fb9bce45f0f0c94882c1cf08fc360caf08be6e7df4 + ;; + Darwin-arm64|Darwin-x86_64) + target=universal-apple-darwin + checksum=d1d7d025d4bfc9fb154cd76aea3949bd78534829a0589abefa6e5dd4edb135dd + ;; + *) echo 'Unsupported nextest release platform.' >&2; exit 2 ;; +esac +archive="cargo-nextest-${version}-${target}.tar.gz" +destination=$1 +mkdir -p "$destination" +curl --fail --location --proto '=https' --proto-redir '=https' --connect-timeout 10 --max-time 120 \ + "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${version}/${archive}" \ + -o "$destination/$archive" +if command -v shasum >/dev/null 2>&1; then + (cd "$destination" && printf '%s %s\n' "$checksum" "$archive" | shasum -a 256 --check) +else + (cd "$destination" && printf '%s %s\n' "$checksum" "$archive" | sha256sum --check) +fi +tar -xzf "$destination/$archive" -C "$destination" cargo-nextest +"$destination/cargo-nextest" --version diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh new file mode 100755 index 000000000..f4311cea6 --- /dev/null +++ b/scripts/test-runtime-core.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P) +cd "$root" +version=$(bash scripts/install-cargo-nextest.sh --print-version) +install="cargo install --locked --version $version cargo-nextest" +runner=${HACK_RUST_TEST_RUNNER:-auto} +repetitions=${HACK_RUST_TEST_REPETITIONS:-1} +case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac +case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac +common=(--locked --manifest-path packages/runtime-core/Cargo.toml --target-dir "${HACK_RUST_TEST_TARGET_DIR:-.hack-local/target}") +if ! command -v cargo-nextest >/dev/null 2>&1; then + if [[ "$runner" == nextest || "$repetitions" != 1 ]]; then + echo "Pinned cargo-nextest $version is required. Install: $install" >&2 + exit 2 + fi + echo "cargo-nextest absent; using cargo test. For process isolation install: $install" >&2 + # A single libtest thread avoids cross-test spawn/close races in the fallback. + if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi + exec cargo test "${common[@]}" --jobs 2 "$@" +fi +actual=$(cargo-nextest --version) +case "$actual" in + "cargo-nextest $version"|"cargo-nextest $version ("*) ;; + *) echo "Pinned cargo-nextest $version is required. Install: $install" >&2; exit 2 ;; +esac +mode=default +for argument in "$@"; do + if [[ "$argument" == --all-features ]]; then mode=all-features; fi +done +if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + mkdir -p "$HACK_RUST_TEST_REPORT_DIR" + # JUnit contains executed tests; keep discovery separately to count ignored + # fixtures and compare names with the stock libtest inventory. + cargo nextest list "${common[@]}" --build-jobs 2 --config-file "$root/.config/nextest.toml" \ + --message-format json "$@" > "$HACK_RUST_TEST_REPORT_DIR/$mode-inventory.json" + cargo nextest show-config test-groups "${common[@]}" --build-jobs 2 \ + --config-file "$root/.config/nextest.toml" --profile ci --groups host-disk-probes "$@" \ + > "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt" + if [[ "$(uname -s)" == Darwin ]] && ! grep -q 'provider::lifecycle::admission_pool::tests::' "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"; then + echo 'The macOS disk-probe tests are missing their serial nextest group.' >&2 + exit 2 + fi + cargo test "${common[@]}" --jobs 2 "$@" -- --list > "$HACK_RUST_TEST_REPORT_DIR/$mode-libtest.txt" + cargo test "${common[@]}" --jobs 2 "$@" -- --list --ignored > "$HACK_RUST_TEST_REPORT_DIR/$mode-ignored.txt" +fi +for ((iteration=1; iteration<=repetitions; iteration++)); do + echo "Rust $mode suite $iteration/$repetitions: process per test, retries=0" + if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + # nextest's store is relative to Cargo's workspace (this standalone crate), + # independently of the shared Cargo --target-dir. + source="$root/packages/runtime-core/target/nextest/ci/junit.xml" + # This is nextest's regenerable result, not a user file. A build failure + # must never copy a previous successful iteration as the failed one's proof. + rm -f -- "$source" + fi + status=0 + cargo nextest run "${common[@]}" --build-jobs 2 \ + --config-file "$root/.config/nextest.toml" --profile ci --retries 0 "$@" || status=$? + if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + if [[ -f "$source" ]]; then + mkdir -p "$HACK_RUST_TEST_REPORT_DIR" + report="$HACK_RUST_TEST_REPORT_DIR/$mode-$iteration.xml" + if [[ -e "$report" ]]; then echo 'Rust report already exists; refusing to replace it.' >&2; exit 2; fi + cp -n "$source" "$report" + elif [[ "$status" == 0 ]]; then + echo 'Nextest succeeded without its required JUnit evidence.' >&2 + exit 2 + fi + fi + # Never reset a failing streak, retry, or continue to another suite after failure. + if [[ "$status" != 0 ]]; then exit "$status"; fi + # Nextest excludes doctests; preserve cargo test's documentation coverage. + cargo test --doc "${common[@]}" --jobs 2 "$@" +done diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts new file mode 100644 index 000000000..f31baae9e --- /dev/null +++ b/tests/runtime-core-test-runner.test.ts @@ -0,0 +1,246 @@ +import { afterEach, expect, test } from "bun:test"; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { privateCliEnv } from "./helpers/private-cli-env.ts"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); +async function fixture( + opts: { + nextest?: boolean; + platform?: string; + fail?: string; + version?: string; + repeats?: string; + required?: boolean; + report?: boolean; + } = {} +) { + const root = await mkdtemp(join(tmpdir(), "hack-rust-runner-")); + roots.push(root); + const tools = join(root, "bin"); + await mkdir(tools); + await mkdir(join(root, "scripts")); + for (const file of ["test-runtime-core.sh", "install-cargo-nextest.sh"]) { + await copyFile(resolve("scripts", file), join(root, "scripts", file)); + } + const log = join(root, "commands"); + const versionChild = Bun.spawn( + [ + "/bin/bash", + join(root, "scripts/install-cargo-nextest.sh"), + "--print-version", + ], + { + env: privateCliEnv({ root }), + stdout: "pipe", + stderr: "ignore", + timeout: 5000, + } + ); + const version = (await new Response(versionChild.stdout).text()).trim(); + expect(await versionChild.exited).toBe(0); + await writeFile( + join(tools, "cargo"), + `#!/bin/sh\nprintf '%s\\n' "$*" >> "$FIXTURE_LOG"\nprintf '%s' "$RUST_TEST_THREADS" > "$FIXTURE_THREADS"\ncase "$1:$2:$FIXTURE_FAIL" in nextest:run:nextest) exit 77;; test:--doc:doc) exit 78;; esac\nif [ "$1:$2" = nextest:run ] && [ -n "$FIXTURE_REPORT" ]; then mkdir -p "$(dirname "$FIXTURE_REPORT")"; printf '\\n' > "$FIXTURE_REPORT"; fi\n`, + { mode: 0o700 } + ); + if (opts.nextest !== false) { + await writeFile( + join(tools, "cargo-nextest"), + `#!/bin/sh\nprintf '%s\\n' 'cargo-nextest ${opts.version ?? version}'\n`, + { mode: 0o700 } + ); + } + await writeFile( + join(tools, "uname"), + `#!/bin/sh\nprintf '%s\\n' '${opts.platform ?? "Linux"}'\n`, + { mode: 0o700 } + ); + const env = privateCliEnv({ + root, + env: { + PATH: `${tools}:/usr/bin:/bin`, + FIXTURE_LOG: log, + FIXTURE_THREADS: join(root, "threads"), + FIXTURE_REPORT: opts.report + ? join(root, "packages/runtime-core/target/nextest/ci/junit.xml") + : undefined, + FIXTURE_FAIL: opts.fail, + HACK_RUST_TEST_RUNNER: opts.required ? "nextest" : "auto", + HACK_RUST_TEST_REPETITIONS: opts.repeats ?? "1", + HACK_RUST_TEST_TARGET_DIR: join(root, "target"), + HACK_RUST_TEST_REPORT_DIR: opts.report + ? join(root, "reports") + : undefined, + }, + }); + const run = async (args: string[] = []) => { + const child = Bun.spawn( + ["/bin/bash", join(root, "scripts/test-runtime-core.sh"), ...args], + { env, stdin: "ignore", stdout: "pipe", stderr: "pipe", timeout: 5000 } + ); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + const commands = await readFile(log, "utf8").catch(() => ""); + return { + code, + stdout, + stderr, + commands: commands.trim() ? commands.trim().split("\n") : [], + version, + threads: await readFile(join(root, "threads"), "utf8").catch(() => ""), + }; + }; + return { run, root, env }; +} + +test("process runner executes nextest with zero retries then retains doctests for feature selection", async () => { + const f = await fixture(); + const r = await f.run(["--all-features"]); + expect(r.code, r.stderr).toBe(0); + expect(r.commands).toHaveLength(2); + expect(r.commands[0]).toContain("nextest run"); + expect(r.commands[0]).toContain("--retries 0"); + expect(r.commands[0]).toContain("--build-jobs 2"); + expect(r.commands[1]).toContain("test --doc"); + for (const command of r.commands) { + expect(command).toContain("--all-features"); + } +}); +for (const fail of ["nextest", "doc"]) { + test(`20-pass qualification stops at first ${fail} failure without retry or fallback`, async () => { + const r = await (await fixture({ fail, repeats: "20" })).run(); + expect(r.code).toBe(fail === "nextest" ? 77 : 78); + expect(r.commands).toHaveLength(fail === "nextest" ? 1 : 2); + }); +} +test("20-pass qualification performs exactly 20 suites and 20 separate doctest steps", async () => { + const r = await (await fixture({ repeats: "20" })).run(); + expect(r.code).toBe(0); + expect(r.commands).toHaveLength(40); + expect(r.commands.filter((c) => c.startsWith("nextest run"))).toHaveLength( + 20 + ); + expect(r.commands.filter((c) => c.startsWith("test --doc"))).toHaveLength(20); +}); +for (const platform of ["Linux", "Darwin"]) { + test(`missing nextest retains the cargo test fallback on ${platform}`, async () => { + const r = await (await fixture({ nextest: false, platform })).run([ + "client_relay", + ]); + expect(r.code).toBe(0); + expect(r.commands).toHaveLength(1); + expect(r.commands[0]).toStartWith("test --locked"); + expect(r.commands[0]).toContain("client_relay"); + expect(r.stderr).toContain( + `cargo install --locked --version ${r.version} cargo-nextest` + ); + expect(r.threads).toBe(platform === "Darwin" ? "1" : ""); + }); +} +test("CI never degrades to libtest when the pinned process runner is missing or mismatched", async () => { + for (const opts of [ + { nextest: false, required: true }, + { version: "0.9.150" }, + { version: "0.9.1512" }, + ]) { + const r = await (await fixture(opts)).run(); + expect(r.code).toBe(2); + expect(r.commands).toEqual([]); + expect(r.stderr).toContain("cargo install --locked --version"); + } +}); + +test("per-pass evidence is read from the Cargo workspace store even with a separate target directory", async () => { + const f = await fixture({ report: true }); + const r = await f.run(); + expect(r.code, r.stderr).toBe(0); + expect(await readFile(join(f.root, "reports/default-1.xml"), "utf8")).toBe( + "\n" + ); + expect(r.commands.some((command) => command.startsWith("nextest list"))).toBe( + true + ); + expect( + r.commands.some((command) => + command.startsWith("nextest show-config test-groups") + ) + ).toBe(true); +}); + +test("a failed nextest invocation cannot publish the previous pass's stale JUnit as fresh evidence", async () => { + const f = await fixture({ report: true, fail: "nextest" }); + const store = join(f.root, "packages/runtime-core/target/nextest/ci"); + await mkdir(store, { recursive: true }); + await writeFile(join(store, "junit.xml"), "stale success"); + const r = await f.run(); + expect(r.code).toBe(77); + expect(await Bun.file(join(f.root, "reports/default-1.xml")).exists()).toBe( + false + ); + expect( + r.commands.filter((command) => command.startsWith("nextest run")) + ).toHaveLength(1); + expect(r.commands.some((command) => command.startsWith("test --doc"))).toBe( + false + ); +}); + +test("the pinned installer refuses corrupt download bytes before extraction or execution", async () => { + const f = await fixture(); + const tools = join(f.root, "bin"); + await writeFile( + join(tools, "uname"), + '#!/bin/sh\nif [ "$1" = -s ]; then echo Linux; else echo x86_64; fi\n', + { mode: 0o700 } + ); + await writeFile( + join(tools, "curl"), + '#!/bin/sh\nwhile [ "$#" -gt 0 ]; do if [ "$1" = -o ]; then shift; printf \'corrupt fixture archive\' > "$1"; exit 0; fi; shift; done\nexit 2\n', + { mode: 0o700 } + ); + const extracted = join(f.root, "extracted"); + await writeFile( + join(tools, "tar"), + "#!/bin/sh\nprintf 'unexpected extraction' > \"$FIXTURE_EXTRACTED\"\nexit 88\n", + { mode: 0o700 } + ); + const child = Bun.spawn( + [ + "/bin/bash", + join(f.root, "scripts/install-cargo-nextest.sh"), + join(f.root, "install"), + ], + { + env: { ...f.env, FIXTURE_EXTRACTED: extracted }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + timeout: 5000, + } + ); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(code).not.toBe(0); + expect(`${stdout}${stderr}`).toContain("FAILED"); + expect(await Bun.file(extracted).exists()).toBe(false); +}); From 8234fc40bc5c2e0475cfd2216bdff257025ba9c6 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:39:03 +0000 Subject: [PATCH 2/3] fix(ci): preserve focused Rust test target selection --- .ai/skills/hack-repo-verify/SKILL.md | 4 ++ packages/runtime-core/README.md | 5 +++ scripts/test-runtime-core.sh | 23 ++++++++++-- tests/runtime-core-test-runner.test.ts | 51 ++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index cbc8c1967..6114107f3 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -65,6 +65,10 @@ changes; the toolchain Rust tasks already enable all features. Keep default cove Candidate CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per test and zero retries, then `cargo test --doc` with the same feature selection. +Explicit target selections (`--lib`, `--test `, etc.) skip doctests, matching +Cargo; explicit `--doc` uses Cargo alone. +Pass Cargo build, feature and target options through this wrapper; use +`cargo nextest` directly for nextest-specific options such as `-E`. This prevents another test's spawned child from inheriting a briefly non-CLOEXEC macOS descriptor and delaying EOF. `bun run test:local` uses that pin when present; otherwise it prints `cargo install --locked --version 0.9.151 cargo-nextest` and diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 0439833d0..b1c294bed 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -564,6 +564,11 @@ another test's spawned child to inherit a socket and delay EOF. Retries are zero The admission-pool test group retains the old mutex's serialization of whole-host `lsof` scans. Ignored helpers remain ignored and their owning tests still invoke them explicitly; `cargo test --doc` runs separately because nextest omits doctests. +Focused target selections such as `bun run test:local --lib` or `--test ` +skip doctests, matching Cargo; explicit `--doc` selection runs through Cargo alone. +The wrapper accepts Cargo build, feature and target options; nextest-specific +options such as `-E` need a direct `cargo nextest` invocation. Documentation-only +selection refuses repetition or report settings intended for nextest qualification. `bun run test:local` uses the same runner when installed. Install its exact pin with `cargo install --locked --version 0.9.151 cargo-nextest`. Without nextest the command diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh index f4311cea6..94ec107d5 100755 --- a/scripts/test-runtime-core.sh +++ b/scripts/test-runtime-core.sh @@ -25,9 +25,25 @@ case "$actual" in *) echo "Pinned cargo-nextest $version is required. Install: $install" >&2; exit 2 ;; esac mode=default +run_doctests=true +doc_only=false for argument in "$@"; do - if [[ "$argument" == --all-features ]]; then mode=all-features; fi + case "$argument" in + --) break ;; + --all-features) mode=all-features ;; + --doc) doc_only=true ;; + --lib|--bins|--bin|--bin=*|--tests|--test|--test=*|--examples|--example|--example=*|--benches|--bench|--bench=*|--all-targets) + run_doctests=false ;; + esac done +# Documentation-only selection has no libtest process for nextest to isolate. +if [[ "$doc_only" == true ]]; then + if [[ "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + echo 'Documentation-only selection cannot produce nextest qualification evidence.' >&2 + exit 2 + fi + exec cargo test "${common[@]}" --jobs 2 "$@" +fi if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then mkdir -p "$HACK_RUST_TEST_REPORT_DIR" # JUnit contains executed tests; keep discovery separately to count ignored @@ -70,6 +86,7 @@ for ((iteration=1; iteration<=repetitions; iteration++)); do fi # Never reset a failing streak, retry, or continue to another suite after failure. if [[ "$status" != 0 ]]; then exit "$status"; fi - # Nextest excludes doctests; preserve cargo test's documentation coverage. - cargo test --doc "${common[@]}" --jobs 2 "$@" + # Nextest excludes doctests. Cargo excludes them for explicit target selection + # too, and rejects --doc combined with those selectors. + if [[ "$run_doctests" == true ]]; then cargo test --doc "${common[@]}" --jobs 2 "$@"; fi done diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts index f31baae9e..6037b3fb8 100644 --- a/tests/runtime-core-test-runner.test.ts +++ b/tests/runtime-core-test-runner.test.ts @@ -123,6 +123,57 @@ test("process runner executes nextest with zero retries then retains doctests fo expect(command).toContain("--all-features"); } }); +test("explicit targets run only the selected nextest suite without conflicting doctest selectors", async () => { + for (const args of [ + ["--lib"], + ["--bins"], + ["--bin", "hack-native"], + ["--bin=hack-native"], + ["--tests"], + ["--test", "stream_relay_contract", "--all-features"], + ["--test=stream_relay_contract", "--all-features"], + ["--examples"], + ["--example", "fixture"], + ["--example=fixture"], + ["--benches"], + ["--bench", "fixture"], + ["--bench=fixture"], + ["--all-targets"], + ]) { + const r = await (await fixture()).run(args); + expect(r.code, r.stderr).toBe(0); + expect(r.commands).toHaveLength(1); + expect(r.commands[0]).toStartWith("nextest run"); + for (const argument of args) { + expect(r.commands[0]).toContain(argument); + } + } +}); +test("explicit documentation selection stays with Cargo and never invokes nextest", async () => { + const r = await (await fixture()).run(["--doc", "--all-features"]); + expect(r.code, r.stderr).toBe(0); + expect(r.commands).toHaveLength(1); + expect(r.commands[0]).toStartWith("test --locked"); + expect(r.commands[0]).toContain("--doc --all-features"); +}); +test("documentation-only selection cannot claim qualification repetitions or nextest reports", async () => { + for (const opts of [{ repeats: "20" }, { report: true }]) { + const r = await (await fixture(opts)).run(["--doc"]); + expect(r.code).toBe(2); + expect(r.commands).toEqual([]); + expect(r.stderr).toContain("cannot produce nextest qualification evidence"); + } +}); +test("arguments after the separator cannot suppress the separate doctest step", async () => { + const r = await (await fixture()).run(["--", "--lib"]); + expect(r.code, r.stderr).toBe(0); + expect(r.commands).toHaveLength(2); + expect(r.commands[0]).toStartWith("nextest run"); + expect(r.commands[1]).toStartWith("test --doc"); + for (const command of r.commands) { + expect(command).toEndWith("-- --lib"); + } +}); for (const fail of ["nextest", "doc"]) { test(`20-pass qualification stops at first ${fail} failure without retry or fallback`, async () => { const r = await (await fixture({ fail, repeats: "20" })).run(); From f230fc98112be43c4b5afb894b448009f01b4bd1 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:14 +0000 Subject: [PATCH 3/3] fix(ci): preserve contributor nextest versions and probe Cargo lookup --- .ai/skills/hack-repo-verify/SKILL.md | 8 ++-- packages/runtime-core/README.md | 13 ++++--- scripts/test-runtime-core.sh | 25 ++++++------ tests/runtime-core-test-runner.test.ts | 54 +++++++++++++++++++++++--- 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index 6114107f3..b8ca1ba9d 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -70,9 +70,11 @@ Cargo; explicit `--doc` uses Cargo alone. Pass Cargo build, feature and target options through this wrapper; use `cargo nextest` directly for nextest-specific options such as `-E`. This prevents another test's spawned child from inheriting a briefly non-CLOEXEC -macOS descriptor and delaying EOF. `bun run test:local` uses that pin when present; -otherwise it prints `cargo install --locked --version 0.9.151 cargo-nextest` and -retains `cargo test` (one libtest thread on macOS). The nextest test group preserves +macOS descriptor and delaying EOF. `bun run test:local` probes `cargo nextest +--version`, matching the subcommand it executes. A missing or different version +prints one fallback line and retains `cargo test` (one libtest thread on macOS), +leaving the installed nextest alone. CI and qualification reports/repetitions +require the exact pin. The nextest test group preserves the admission-pool tests' serialized host-wide disk probes. A manual CI dispatch with `rust_repetitions=20` runs 20 consecutive default and 20 all-feature Mac suites, stopping at the first failure, and preserves per-pass JUnit counts/timings. diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index b1c294bed..677ff280b 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -570,11 +570,14 @@ The wrapper accepts Cargo build, feature and target options; nextest-specific options such as `-E` need a direct `cargo nextest` invocation. Documentation-only selection refuses repetition or report settings intended for nextest qualification. -`bun run test:local` uses the same runner when installed. Install its exact pin with -`cargo install --locked --version 0.9.151 cargo-nextest`. Without nextest the command -prints that instruction and retains stock `cargo test`; the macOS fallback sets -one libtest thread to prevent cross-test descriptor inheritance. CI requires the -pin and never silently falls back. `bun run test:local --all-features` retains the +`bun run test:local` checks `cargo nextest --version`, matching Cargo's lookup for +the test run. If that resolves the pin, it uses nextest; a missing or different +version prints one fallback line and uses stock `cargo test`, leaving the +contributor's installed version alone. The macOS fallback sets one libtest thread +to prevent cross-test descriptor inheritance. To opt into the pinned runner, +install it with `cargo install --locked --version 0.9.151 cargo-nextest`. +CI and qualification reports/repetitions require the pin and never fall back. +`bun run test:local --all-features` retains the Zig prerequisite above. The toolchain container's existing `rust` task continues to use stock Cargo on Linux, where the descriptor creation primitives are atomic. diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh index 94ec107d5..8d617307a 100755 --- a/scripts/test-runtime-core.sh +++ b/scripts/test-runtime-core.sh @@ -9,20 +9,21 @@ repetitions=${HACK_RUST_TEST_REPETITIONS:-1} case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac common=(--locked --manifest-path packages/runtime-core/Cargo.toml --target-dir "${HACK_RUST_TEST_TARGET_DIR:-.hack-local/target}") -if ! command -v cargo-nextest >/dev/null 2>&1; then - if [[ "$runner" == nextest || "$repetitions" != 1 ]]; then - echo "Pinned cargo-nextest $version is required. Install: $install" >&2 - exit 2 - fi - echo "cargo-nextest absent; using cargo test. For process isolation install: $install" >&2 - # A single libtest thread avoids cross-test spawn/close races in the fallback. - if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi - exec cargo test "${common[@]}" --jobs 2 "$@" -fi -actual=$(cargo-nextest --version) +# Cargo's subcommand lookup can differ from a direct PATH lookup. Probe the +# same command used below, and leave a contributor's other nextest version alone. +actual=$(cargo nextest --version 2>/dev/null) || actual="" case "$actual" in "cargo-nextest $version"|"cargo-nextest $version ("*) ;; - *) echo "Pinned cargo-nextest $version is required. Install: $install" >&2; exit 2 ;; + *) + if [[ "$runner" == nextest || "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then + echo "Pinned cargo-nextest $version is required. Install: $install" >&2 + exit 2 + fi + echo "cargo nextest $version unavailable; using cargo test." >&2 + # A single libtest thread avoids cross-test spawn/close races in the fallback. + if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi + exec cargo test "${common[@]}" --jobs 2 "$@" + ;; esac mode=default run_doctests=true diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts index 6037b3fb8..4fc0dbaec 100644 --- a/tests/runtime-core-test-runner.test.ts +++ b/tests/runtime-core-test-runner.test.ts @@ -23,6 +23,7 @@ async function fixture( platform?: string; fail?: string; version?: string; + pathVersion?: string; repeats?: string; required?: boolean; report?: boolean; @@ -54,13 +55,13 @@ async function fixture( expect(await versionChild.exited).toBe(0); await writeFile( join(tools, "cargo"), - `#!/bin/sh\nprintf '%s\\n' "$*" >> "$FIXTURE_LOG"\nprintf '%s' "$RUST_TEST_THREADS" > "$FIXTURE_THREADS"\ncase "$1:$2:$FIXTURE_FAIL" in nextest:run:nextest) exit 77;; test:--doc:doc) exit 78;; esac\nif [ "$1:$2" = nextest:run ] && [ -n "$FIXTURE_REPORT" ]; then mkdir -p "$(dirname "$FIXTURE_REPORT")"; printf '\\n' > "$FIXTURE_REPORT"; fi\n`, + `#!/bin/sh\nif [ "$1:$2" = nextest:--version ]; then [ "$FIXTURE_NEXTEST" = present ] || exit 101; printf 'cargo-nextest %s\\n' "$FIXTURE_CARGO_VERSION"; exit 0; fi\nprintf '%s\\n' "$*" >> "$FIXTURE_LOG"\nprintf '%s' "$RUST_TEST_THREADS" > "$FIXTURE_THREADS"\ncase "$1:$2:$FIXTURE_FAIL" in nextest:run:nextest) exit 77;; test:--doc:doc) exit 78;; esac\nif [ "$1:$2" = nextest:run ] && [ -n "$FIXTURE_REPORT" ]; then mkdir -p "$(dirname "$FIXTURE_REPORT")"; printf '\\n' > "$FIXTURE_REPORT"; fi\n`, { mode: 0o700 } ); if (opts.nextest !== false) { await writeFile( join(tools, "cargo-nextest"), - `#!/bin/sh\nprintf '%s\\n' 'cargo-nextest ${opts.version ?? version}'\n`, + `#!/bin/sh\nprintf 'direct PATH lookup' > "$FIXTURE_DIRECT_LOOKUP"\nprintf '%s\\n' 'cargo-nextest ${opts.pathVersion ?? opts.version ?? version}'\n`, { mode: 0o700 } ); } @@ -75,6 +76,9 @@ async function fixture( PATH: `${tools}:/usr/bin:/bin`, FIXTURE_LOG: log, FIXTURE_THREADS: join(root, "threads"), + FIXTURE_NEXTEST: opts.nextest === false ? "absent" : "present", + FIXTURE_CARGO_VERSION: opts.version ?? version, + FIXTURE_DIRECT_LOOKUP: join(root, "direct-lookup"), FIXTURE_REPORT: opts.report ? join(root, "packages/runtime-core/target/nextest/ci/junit.xml") : undefined, @@ -199,8 +203,8 @@ for (const platform of ["Linux", "Darwin"]) { expect(r.commands).toHaveLength(1); expect(r.commands[0]).toStartWith("test --locked"); expect(r.commands[0]).toContain("client_relay"); - expect(r.stderr).toContain( - `cargo install --locked --version ${r.version} cargo-nextest` + expect(r.stderr.trim()).toBe( + `cargo nextest ${r.version} unavailable; using cargo test.` ); expect(r.threads).toBe(platform === "Darwin" ? "1" : ""); }); @@ -208,8 +212,10 @@ for (const platform of ["Linux", "Darwin"]) { test("CI never degrades to libtest when the pinned process runner is missing or mismatched", async () => { for (const opts of [ { nextest: false, required: true }, - { version: "0.9.150" }, - { version: "0.9.1512" }, + { version: "0.9.150", required: true }, + { version: "0.9.1512", required: true }, + { version: "0.9.150", repeats: "20" }, + { version: "0.9.150", report: true }, ]) { const r = await (await fixture(opts)).run(); expect(r.code).toBe(2); @@ -217,6 +223,42 @@ test("CI never degrades to libtest when the pinned process runner is missing or expect(r.stderr).toContain("cargo install --locked --version"); } }); +test("another contributor nextest version falls back without replacing it on either host", async () => { + for (const platform of ["Linux", "Darwin"]) { + const r = await (await fixture({ version: "0.9.150", platform })).run([ + "--lib", + "--all-features", + ]); + expect(r.code, r.stderr).toBe(0); + expect(r.commands).toHaveLength(1); + expect(r.commands[0]).toStartWith("test --locked"); + expect(r.commands[0]).toEndWith("--lib --all-features"); + expect(r.stderr.trim()).toBe( + `cargo nextest ${r.version} unavailable; using cargo test.` + ); + expect(r.threads).toBe(platform === "Darwin" ? "1" : ""); + } +}); +test("the version check follows Cargo subcommand resolution instead of the PATH binary", async () => { + const rightCargo = await fixture({ + pathVersion: "0.9.150", + required: true, + }); + const accepted = await rightCargo.run(); + expect(accepted.code, accepted.stderr).toBe(0); + expect(accepted.commands[0]).toStartWith("nextest run"); + const wrongCargo = await fixture({ + version: "0.9.150", + pathVersion: accepted.version, + required: true, + }); + const refused = await wrongCargo.run(); + expect(refused.code).toBe(2); + expect(refused.commands).toEqual([]); + for (const f of [wrongCargo, rightCargo]) { + expect(await Bun.file(join(f.root, "direct-lookup")).exists()).toBe(false); + } +}); test("per-pass evidence is read from the Cargo workspace store even with a separate target directory", async () => { const f = await fixture({ report: true });