From d7f0c99de0120d2f2a047132008be3f2bb6508f4 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 22:31:16 +0000
Subject: [PATCH 1/3] fix(ci): isolate Rust tests to prevent macOS descriptor
leaks
---
.ai/skills/hack-repo-verify/SKILL.md | 10 +
.config/nextest.toml | 25 +++
.github/workflows/ci.yml | 30 ++-
package.json | 2 +-
packages/runtime-core/README.md | 30 +++
scripts/install-cargo-nextest.sh | 37 ++++
scripts/test-runtime-core.sh | 75 ++++++++
tests/runtime-core-test-runner.test.ts | 246 +++++++++++++++++++++++++
8 files changed, 451 insertions(+), 4 deletions(-)
create mode 100644 .config/nextest.toml
create mode 100755 scripts/install-cargo-nextest.sh
create mode 100755 scripts/test-runtime-core.sh
create mode 100644 tests/runtime-core-test-runner.test.ts
diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md
index 331e5aed5..cbc8c1967 100644
--- a/.ai/skills/hack-repo-verify/SKILL.md
+++ b/.ai/skills/hack-repo-verify/SKILL.md
@@ -63,6 +63,16 @@ CI also lints those scripts and runs their transport tests; lint changed ones ex
Rust default gates are `bun run check:local` and `bun run test:local`. Add `--all-features` to equivalent Cargo commands for feature
changes; the toolchain Rust tasks already enable all features. Keep default coverage.
+Candidate CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per
+test and zero retries, then `cargo test --doc` with the same feature selection.
+This prevents another test's spawned child from inheriting a briefly non-CLOEXEC
+macOS descriptor and delaying EOF. `bun run test:local` uses that pin when present;
+otherwise it prints `cargo install --locked --version 0.9.151 cargo-nextest` and
+retains `cargo test` (one libtest thread on macOS). The nextest test group preserves
+the admission-pool tests' serialized host-wide disk probes. A manual CI dispatch
+with `rust_repetitions=20` runs 20 consecutive default and 20 all-feature Mac suites,
+stopping at the first failure, and preserves per-pass JUnit counts/timings.
+
`bun run test:local` does not enable `native-stream-relay`, so
`stream_relay_contract` reports zero tests. This default pass does not qualify the
relay contract. With Zig 0.15.2 available, run the local all-features equivalent:
diff --git a/.config/nextest.toml b/.config/nextest.toml
new file mode 100644
index 000000000..64056e6ed
--- /dev/null
+++ b/.config/nextest.toml
@@ -0,0 +1,25 @@
+# Both CI hosts use a separate process for every Rust test. Keep ignored fixtures
+# ignored: their parent tests invoke them explicitly with the stock test binary.
+[profile.default]
+retries = 0
+
+[profile.ci]
+retries = 0
+fail-fast = true
+slow-timeout = { period = "60s", terminate-after = 10 }
+
+[profile.ci.junit]
+path = "junit.xml"
+
+# The old crate mutex cannot serialize separate processes. Preserve its bound on
+# whole-host lsof scans, without serializing unrelated runtime tests.
+[test-groups.host-disk-probes]
+max-threads = 1
+
+[[profile.default.overrides]]
+filter = 'test(provider::lifecycle::admission_pool::tests::)'
+test-group = 'host-disk-probes'
+
+[[profile.ci.overrides]]
+filter = 'test(provider::lifecycle::admission_pool::tests::)'
+test-group = 'host-disk-probes'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ef102ebb6..a0daa7c35 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,6 +16,11 @@ on:
type: boolean
required: true
default: true
+ rust_repetitions:
+ description: Consecutive macOS Rust qualification passes (stops on first failure)
+ type: choice
+ options: ["1", "20"]
+ default: "1"
concurrency:
# Let the active next run finish, including jobs waiting for macOS capacity.
@@ -372,7 +377,10 @@ jobs:
matrix:
os: ${{ fromJSON(needs.changes.outputs.defer_macos == 'true' && '["ubuntu-latest"]' || '["ubuntu-latest","macos-latest"]') }}
runs-on: ${{ matrix.os }}
- timeout-minutes: 45
+ timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && inputs.rust_repetitions == '20' && 300 || 45 }}
+ env:
+ HACK_RUST_TEST_RUNNER: nextest
+ HACK_RUST_TEST_REPETITIONS: ${{ matrix.os == 'macos-latest' && github.event_name == 'workflow_dispatch' && inputs.rust_repetitions || '1' }}
permissions:
contents: read
steps:
@@ -423,14 +431,30 @@ jobs:
echo "$destination/zig-${platform}-0.15.2" >> "$GITHUB_PATH"
- name: Check candidate formatting
run: cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check
+ - name: Install pinned process-per-test runner
+ shell: bash
+ run: |
+ bash scripts/install-cargo-nextest.sh "$RUNNER_TEMP/candidate-nextest"
+ echo "$RUNNER_TEMP/candidate-nextest" >> "$GITHUB_PATH"
- name: Lint candidate core
run: cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings
- name: Test candidate isolation contracts
- run: cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --jobs 2
+ env:
+ HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/candidate-rust-results
+ run: bash scripts/test-runtime-core.sh
- name: Lint candidate optional features
run: cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --all-features --jobs 2 -- -D warnings
- name: Test candidate optional feature contracts
- run: cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-features --jobs 2
+ env:
+ HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/candidate-rust-results
+ run: bash scripts/test-runtime-core.sh --all-features
+ - name: Preserve per-pass Rust counts and timings
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: candidate-rust-results-${{ matrix.os }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/candidate-rust-results/
+ if-no-files-found: ignore
- name: Build and inspect local candidate
run: |
./scripts/build-hack-local.sh
diff --git a/package.json b/package.json
index e9bfb445d..773a7c2fc 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
"build:config-compiler": "bun scripts/build-config-compiler.ts",
"check:config-compiler": "cargo fmt --manifest-path packages/config-compiler/Cargo.toml --check && cargo clippy --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --all-targets --jobs 2 -- -D warnings",
"test:config-compiler": "cargo test --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --jobs 2",
- "test:local": "cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --jobs 2",
+ "test:local": "bash scripts/test-runtime-core.sh",
"check:local": "cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check && cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings",
"build:runtime-image": "bun run --cwd packages/cli build:runtime-image",
"build:node-runtime-image": "bun run --cwd packages/cli build:node-runtime-image",
diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md
index 0365d473c..0439833d0 100644
--- a/packages/runtime-core/README.md
+++ b/packages/runtime-core/README.md
@@ -555,6 +555,36 @@ turn on no features and do not need Zig. Root `mise.toml` and `.tool-versions` p
Zig 0.15.2. The toolchain image installs the `mise.toml` pins and runs its tasks
under `mise exec`. CI downloads a checksum-verified 0.15.2 archive.
+### Rust test processes
+
+Both Candidate core CI hosts use **cargo-nextest 0.9.151**, downloaded from its
+pinned release and SHA-256 verified, for default and all-feature tests. Each test
+runs in its own process: on macOS the two-call close-on-exec setup briefly permits
+another test's spawned child to inherit a socket and delay EOF. Retries are zero.
+The admission-pool test group retains the old mutex's serialization of whole-host
+`lsof` scans. Ignored helpers remain ignored and their owning tests still invoke
+them explicitly; `cargo test --doc` runs separately because nextest omits doctests.
+
+`bun run test:local` uses the same runner when installed. Install its exact pin with
+`cargo install --locked --version 0.9.151 cargo-nextest`. Without nextest the command
+prints that instruction and retains stock `cargo test`; the macOS fallback sets
+one libtest thread to prevent cross-test descriptor inheritance. CI requires the
+pin and never silently falls back. `bun run test:local --all-features` retains the
+Zig prerequisite above. The toolchain container's existing `rust` task continues
+to use stock Cargo on Linux, where the descriptor creation primitives are atomic.
+
+For bounded hosted qualification, dispatch `ci.yml` at the exact branch/commit
+with `full=true` and `rust_repetitions=20`. Only macOS repeats: 20 default suites
+and 20 all-feature suites, each followed by doctests. A failed iteration stops
+the job immediately; this is a consecutive-pass experiment, not retry policy.
+The `candidate-rust-results-*` artifacts retain each pass's counts and duration.
+Discovery JSON and stock libtest lists retain ignored-test counts separately:
+nextest's JUnit report contains executed tests, not ignored fixtures. The CI profile
+terminates a hung test after ten 60-second intervals, above the existing 500-second
+fixture budgets, so a hang fails with evidence. Qualification dispatches allow up
+to 300 minutes; ordinary Candidate jobs keep their 45-minute limit. Record measured
+test-step times as well as the pass count before treating the loop as qualified.
+
Source binaries embed their build checkout. If a copied Cargo target directory
retains a different checkout identity, use a fresh `--target-dir` for checks and
tests; Cargo reporting a cached binary as fresh does not qualify that identity.
diff --git a/scripts/install-cargo-nextest.sh b/scripts/install-cargo-nextest.sh
new file mode 100755
index 000000000..e8500018c
--- /dev/null
+++ b/scripts/install-cargo-nextest.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Official stable release, verified against the release asset SHA-256 metadata.
+version=0.9.151
+if [[ "${1-}" == --print-version ]]; then
+ printf '%s\n' "$version"
+ exit 0
+fi
+if [[ "$#" != 1 ]]; then
+ echo 'Usage: bash scripts/install-cargo-nextest.sh DESTINATION' >&2
+ exit 2
+fi
+case "$(uname -s)-$(uname -m)" in
+ Linux-x86_64)
+ target=x86_64-unknown-linux-gnu
+ checksum=1d4ce302b820c41f4b89c5fb9bce45f0f0c94882c1cf08fc360caf08be6e7df4
+ ;;
+ Darwin-arm64|Darwin-x86_64)
+ target=universal-apple-darwin
+ checksum=d1d7d025d4bfc9fb154cd76aea3949bd78534829a0589abefa6e5dd4edb135dd
+ ;;
+ *) echo 'Unsupported nextest release platform.' >&2; exit 2 ;;
+esac
+archive="cargo-nextest-${version}-${target}.tar.gz"
+destination=$1
+mkdir -p "$destination"
+curl --fail --location --proto '=https' --proto-redir '=https' --connect-timeout 10 --max-time 120 \
+ "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${version}/${archive}" \
+ -o "$destination/$archive"
+if command -v shasum >/dev/null 2>&1; then
+ (cd "$destination" && printf '%s %s\n' "$checksum" "$archive" | shasum -a 256 --check)
+else
+ (cd "$destination" && printf '%s %s\n' "$checksum" "$archive" | sha256sum --check)
+fi
+tar -xzf "$destination/$archive" -C "$destination" cargo-nextest
+"$destination/cargo-nextest" --version
diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh
new file mode 100755
index 000000000..f4311cea6
--- /dev/null
+++ b/scripts/test-runtime-core.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -euo pipefail
+root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)
+cd "$root"
+version=$(bash scripts/install-cargo-nextest.sh --print-version)
+install="cargo install --locked --version $version cargo-nextest"
+runner=${HACK_RUST_TEST_RUNNER:-auto}
+repetitions=${HACK_RUST_TEST_REPETITIONS:-1}
+case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac
+case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac
+common=(--locked --manifest-path packages/runtime-core/Cargo.toml --target-dir "${HACK_RUST_TEST_TARGET_DIR:-.hack-local/target}")
+if ! command -v cargo-nextest >/dev/null 2>&1; then
+ if [[ "$runner" == nextest || "$repetitions" != 1 ]]; then
+ echo "Pinned cargo-nextest $version is required. Install: $install" >&2
+ exit 2
+ fi
+ echo "cargo-nextest absent; using cargo test. For process isolation install: $install" >&2
+ # A single libtest thread avoids cross-test spawn/close races in the fallback.
+ if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi
+ exec cargo test "${common[@]}" --jobs 2 "$@"
+fi
+actual=$(cargo-nextest --version)
+case "$actual" in
+ "cargo-nextest $version"|"cargo-nextest $version ("*) ;;
+ *) echo "Pinned cargo-nextest $version is required. Install: $install" >&2; exit 2 ;;
+esac
+mode=default
+for argument in "$@"; do
+ if [[ "$argument" == --all-features ]]; then mode=all-features; fi
+done
+if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+ mkdir -p "$HACK_RUST_TEST_REPORT_DIR"
+ # JUnit contains executed tests; keep discovery separately to count ignored
+ # fixtures and compare names with the stock libtest inventory.
+ cargo nextest list "${common[@]}" --build-jobs 2 --config-file "$root/.config/nextest.toml" \
+ --message-format json "$@" > "$HACK_RUST_TEST_REPORT_DIR/$mode-inventory.json"
+ cargo nextest show-config test-groups "${common[@]}" --build-jobs 2 \
+ --config-file "$root/.config/nextest.toml" --profile ci --groups host-disk-probes "$@" \
+ > "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"
+ if [[ "$(uname -s)" == Darwin ]] && ! grep -q 'provider::lifecycle::admission_pool::tests::' "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"; then
+ echo 'The macOS disk-probe tests are missing their serial nextest group.' >&2
+ exit 2
+ fi
+ cargo test "${common[@]}" --jobs 2 "$@" -- --list > "$HACK_RUST_TEST_REPORT_DIR/$mode-libtest.txt"
+ cargo test "${common[@]}" --jobs 2 "$@" -- --list --ignored > "$HACK_RUST_TEST_REPORT_DIR/$mode-ignored.txt"
+fi
+for ((iteration=1; iteration<=repetitions; iteration++)); do
+ echo "Rust $mode suite $iteration/$repetitions: process per test, retries=0"
+ if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+ # nextest's store is relative to Cargo's workspace (this standalone crate),
+ # independently of the shared Cargo --target-dir.
+ source="$root/packages/runtime-core/target/nextest/ci/junit.xml"
+ # This is nextest's regenerable result, not a user file. A build failure
+ # must never copy a previous successful iteration as the failed one's proof.
+ rm -f -- "$source"
+ fi
+ status=0
+ cargo nextest run "${common[@]}" --build-jobs 2 \
+ --config-file "$root/.config/nextest.toml" --profile ci --retries 0 "$@" || status=$?
+ if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+ if [[ -f "$source" ]]; then
+ mkdir -p "$HACK_RUST_TEST_REPORT_DIR"
+ report="$HACK_RUST_TEST_REPORT_DIR/$mode-$iteration.xml"
+ if [[ -e "$report" ]]; then echo 'Rust report already exists; refusing to replace it.' >&2; exit 2; fi
+ cp -n "$source" "$report"
+ elif [[ "$status" == 0 ]]; then
+ echo 'Nextest succeeded without its required JUnit evidence.' >&2
+ exit 2
+ fi
+ fi
+ # Never reset a failing streak, retry, or continue to another suite after failure.
+ if [[ "$status" != 0 ]]; then exit "$status"; fi
+ # Nextest excludes doctests; preserve cargo test's documentation coverage.
+ cargo test --doc "${common[@]}" --jobs 2 "$@"
+done
diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts
new file mode 100644
index 000000000..f31baae9e
--- /dev/null
+++ b/tests/runtime-core-test-runner.test.ts
@@ -0,0 +1,246 @@
+import { afterEach, expect, test } from "bun:test";
+import {
+ copyFile,
+ mkdir,
+ mkdtemp,
+ readFile,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { privateCliEnv } from "./helpers/private-cli-env.ts";
+
+const roots: string[] = [];
+afterEach(async () => {
+ for (const root of roots.splice(0)) {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+async function fixture(
+ opts: {
+ nextest?: boolean;
+ platform?: string;
+ fail?: string;
+ version?: string;
+ repeats?: string;
+ required?: boolean;
+ report?: boolean;
+ } = {}
+) {
+ const root = await mkdtemp(join(tmpdir(), "hack-rust-runner-"));
+ roots.push(root);
+ const tools = join(root, "bin");
+ await mkdir(tools);
+ await mkdir(join(root, "scripts"));
+ for (const file of ["test-runtime-core.sh", "install-cargo-nextest.sh"]) {
+ await copyFile(resolve("scripts", file), join(root, "scripts", file));
+ }
+ const log = join(root, "commands");
+ const versionChild = Bun.spawn(
+ [
+ "/bin/bash",
+ join(root, "scripts/install-cargo-nextest.sh"),
+ "--print-version",
+ ],
+ {
+ env: privateCliEnv({ root }),
+ stdout: "pipe",
+ stderr: "ignore",
+ timeout: 5000,
+ }
+ );
+ const version = (await new Response(versionChild.stdout).text()).trim();
+ expect(await versionChild.exited).toBe(0);
+ await writeFile(
+ join(tools, "cargo"),
+ `#!/bin/sh\nprintf '%s\\n' "$*" >> "$FIXTURE_LOG"\nprintf '%s' "$RUST_TEST_THREADS" > "$FIXTURE_THREADS"\ncase "$1:$2:$FIXTURE_FAIL" in nextest:run:nextest) exit 77;; test:--doc:doc) exit 78;; esac\nif [ "$1:$2" = nextest:run ] && [ -n "$FIXTURE_REPORT" ]; then mkdir -p "$(dirname "$FIXTURE_REPORT")"; printf '\\n' > "$FIXTURE_REPORT"; fi\n`,
+ { mode: 0o700 }
+ );
+ if (opts.nextest !== false) {
+ await writeFile(
+ join(tools, "cargo-nextest"),
+ `#!/bin/sh\nprintf '%s\\n' 'cargo-nextest ${opts.version ?? version}'\n`,
+ { mode: 0o700 }
+ );
+ }
+ await writeFile(
+ join(tools, "uname"),
+ `#!/bin/sh\nprintf '%s\\n' '${opts.platform ?? "Linux"}'\n`,
+ { mode: 0o700 }
+ );
+ const env = privateCliEnv({
+ root,
+ env: {
+ PATH: `${tools}:/usr/bin:/bin`,
+ FIXTURE_LOG: log,
+ FIXTURE_THREADS: join(root, "threads"),
+ FIXTURE_REPORT: opts.report
+ ? join(root, "packages/runtime-core/target/nextest/ci/junit.xml")
+ : undefined,
+ FIXTURE_FAIL: opts.fail,
+ HACK_RUST_TEST_RUNNER: opts.required ? "nextest" : "auto",
+ HACK_RUST_TEST_REPETITIONS: opts.repeats ?? "1",
+ HACK_RUST_TEST_TARGET_DIR: join(root, "target"),
+ HACK_RUST_TEST_REPORT_DIR: opts.report
+ ? join(root, "reports")
+ : undefined,
+ },
+ });
+ const run = async (args: string[] = []) => {
+ const child = Bun.spawn(
+ ["/bin/bash", join(root, "scripts/test-runtime-core.sh"), ...args],
+ { env, stdin: "ignore", stdout: "pipe", stderr: "pipe", timeout: 5000 }
+ );
+ const [code, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
+ const commands = await readFile(log, "utf8").catch(() => "");
+ return {
+ code,
+ stdout,
+ stderr,
+ commands: commands.trim() ? commands.trim().split("\n") : [],
+ version,
+ threads: await readFile(join(root, "threads"), "utf8").catch(() => ""),
+ };
+ };
+ return { run, root, env };
+}
+
+test("process runner executes nextest with zero retries then retains doctests for feature selection", async () => {
+ const f = await fixture();
+ const r = await f.run(["--all-features"]);
+ expect(r.code, r.stderr).toBe(0);
+ expect(r.commands).toHaveLength(2);
+ expect(r.commands[0]).toContain("nextest run");
+ expect(r.commands[0]).toContain("--retries 0");
+ expect(r.commands[0]).toContain("--build-jobs 2");
+ expect(r.commands[1]).toContain("test --doc");
+ for (const command of r.commands) {
+ expect(command).toContain("--all-features");
+ }
+});
+for (const fail of ["nextest", "doc"]) {
+ test(`20-pass qualification stops at first ${fail} failure without retry or fallback`, async () => {
+ const r = await (await fixture({ fail, repeats: "20" })).run();
+ expect(r.code).toBe(fail === "nextest" ? 77 : 78);
+ expect(r.commands).toHaveLength(fail === "nextest" ? 1 : 2);
+ });
+}
+test("20-pass qualification performs exactly 20 suites and 20 separate doctest steps", async () => {
+ const r = await (await fixture({ repeats: "20" })).run();
+ expect(r.code).toBe(0);
+ expect(r.commands).toHaveLength(40);
+ expect(r.commands.filter((c) => c.startsWith("nextest run"))).toHaveLength(
+ 20
+ );
+ expect(r.commands.filter((c) => c.startsWith("test --doc"))).toHaveLength(20);
+});
+for (const platform of ["Linux", "Darwin"]) {
+ test(`missing nextest retains the cargo test fallback on ${platform}`, async () => {
+ const r = await (await fixture({ nextest: false, platform })).run([
+ "client_relay",
+ ]);
+ expect(r.code).toBe(0);
+ expect(r.commands).toHaveLength(1);
+ expect(r.commands[0]).toStartWith("test --locked");
+ expect(r.commands[0]).toContain("client_relay");
+ expect(r.stderr).toContain(
+ `cargo install --locked --version ${r.version} cargo-nextest`
+ );
+ expect(r.threads).toBe(platform === "Darwin" ? "1" : "");
+ });
+}
+test("CI never degrades to libtest when the pinned process runner is missing or mismatched", async () => {
+ for (const opts of [
+ { nextest: false, required: true },
+ { version: "0.9.150" },
+ { version: "0.9.1512" },
+ ]) {
+ const r = await (await fixture(opts)).run();
+ expect(r.code).toBe(2);
+ expect(r.commands).toEqual([]);
+ expect(r.stderr).toContain("cargo install --locked --version");
+ }
+});
+
+test("per-pass evidence is read from the Cargo workspace store even with a separate target directory", async () => {
+ const f = await fixture({ report: true });
+ const r = await f.run();
+ expect(r.code, r.stderr).toBe(0);
+ expect(await readFile(join(f.root, "reports/default-1.xml"), "utf8")).toBe(
+ "\n"
+ );
+ expect(r.commands.some((command) => command.startsWith("nextest list"))).toBe(
+ true
+ );
+ expect(
+ r.commands.some((command) =>
+ command.startsWith("nextest show-config test-groups")
+ )
+ ).toBe(true);
+});
+
+test("a failed nextest invocation cannot publish the previous pass's stale JUnit as fresh evidence", async () => {
+ const f = await fixture({ report: true, fail: "nextest" });
+ const store = join(f.root, "packages/runtime-core/target/nextest/ci");
+ await mkdir(store, { recursive: true });
+ await writeFile(join(store, "junit.xml"), "stale success");
+ const r = await f.run();
+ expect(r.code).toBe(77);
+ expect(await Bun.file(join(f.root, "reports/default-1.xml")).exists()).toBe(
+ false
+ );
+ expect(
+ r.commands.filter((command) => command.startsWith("nextest run"))
+ ).toHaveLength(1);
+ expect(r.commands.some((command) => command.startsWith("test --doc"))).toBe(
+ false
+ );
+});
+
+test("the pinned installer refuses corrupt download bytes before extraction or execution", async () => {
+ const f = await fixture();
+ const tools = join(f.root, "bin");
+ await writeFile(
+ join(tools, "uname"),
+ '#!/bin/sh\nif [ "$1" = -s ]; then echo Linux; else echo x86_64; fi\n',
+ { mode: 0o700 }
+ );
+ await writeFile(
+ join(tools, "curl"),
+ '#!/bin/sh\nwhile [ "$#" -gt 0 ]; do if [ "$1" = -o ]; then shift; printf \'corrupt fixture archive\' > "$1"; exit 0; fi; shift; done\nexit 2\n',
+ { mode: 0o700 }
+ );
+ const extracted = join(f.root, "extracted");
+ await writeFile(
+ join(tools, "tar"),
+ "#!/bin/sh\nprintf 'unexpected extraction' > \"$FIXTURE_EXTRACTED\"\nexit 88\n",
+ { mode: 0o700 }
+ );
+ const child = Bun.spawn(
+ [
+ "/bin/bash",
+ join(f.root, "scripts/install-cargo-nextest.sh"),
+ join(f.root, "install"),
+ ],
+ {
+ env: { ...f.env, FIXTURE_EXTRACTED: extracted },
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ timeout: 5000,
+ }
+ );
+ const [code, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
+ expect(code).not.toBe(0);
+ expect(`${stdout}${stderr}`).toContain("FAILED");
+ expect(await Bun.file(extracted).exists()).toBe(false);
+});
From 8234fc40bc5c2e0475cfd2216bdff257025ba9c6 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 22:39:03 +0000
Subject: [PATCH 2/3] fix(ci): preserve focused Rust test target selection
---
.ai/skills/hack-repo-verify/SKILL.md | 4 ++
packages/runtime-core/README.md | 5 +++
scripts/test-runtime-core.sh | 23 ++++++++++--
tests/runtime-core-test-runner.test.ts | 51 ++++++++++++++++++++++++++
4 files changed, 80 insertions(+), 3 deletions(-)
diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md
index cbc8c1967..6114107f3 100644
--- a/.ai/skills/hack-repo-verify/SKILL.md
+++ b/.ai/skills/hack-repo-verify/SKILL.md
@@ -65,6 +65,10 @@ changes; the toolchain Rust tasks already enable all features. Keep default cove
Candidate CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per
test and zero retries, then `cargo test --doc` with the same feature selection.
+Explicit target selections (`--lib`, `--test `, etc.) skip doctests, matching
+Cargo; explicit `--doc` uses Cargo alone.
+Pass Cargo build, feature and target options through this wrapper; use
+`cargo nextest` directly for nextest-specific options such as `-E`.
This prevents another test's spawned child from inheriting a briefly non-CLOEXEC
macOS descriptor and delaying EOF. `bun run test:local` uses that pin when present;
otherwise it prints `cargo install --locked --version 0.9.151 cargo-nextest` and
diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md
index 0439833d0..b1c294bed 100644
--- a/packages/runtime-core/README.md
+++ b/packages/runtime-core/README.md
@@ -564,6 +564,11 @@ another test's spawned child to inherit a socket and delay EOF. Retries are zero
The admission-pool test group retains the old mutex's serialization of whole-host
`lsof` scans. Ignored helpers remain ignored and their owning tests still invoke
them explicitly; `cargo test --doc` runs separately because nextest omits doctests.
+Focused target selections such as `bun run test:local --lib` or `--test `
+skip doctests, matching Cargo; explicit `--doc` selection runs through Cargo alone.
+The wrapper accepts Cargo build, feature and target options; nextest-specific
+options such as `-E` need a direct `cargo nextest` invocation. Documentation-only
+selection refuses repetition or report settings intended for nextest qualification.
`bun run test:local` uses the same runner when installed. Install its exact pin with
`cargo install --locked --version 0.9.151 cargo-nextest`. Without nextest the command
diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh
index f4311cea6..94ec107d5 100755
--- a/scripts/test-runtime-core.sh
+++ b/scripts/test-runtime-core.sh
@@ -25,9 +25,25 @@ case "$actual" in
*) echo "Pinned cargo-nextest $version is required. Install: $install" >&2; exit 2 ;;
esac
mode=default
+run_doctests=true
+doc_only=false
for argument in "$@"; do
- if [[ "$argument" == --all-features ]]; then mode=all-features; fi
+ case "$argument" in
+ --) break ;;
+ --all-features) mode=all-features ;;
+ --doc) doc_only=true ;;
+ --lib|--bins|--bin|--bin=*|--tests|--test|--test=*|--examples|--example|--example=*|--benches|--bench|--bench=*|--all-targets)
+ run_doctests=false ;;
+ esac
done
+# Documentation-only selection has no libtest process for nextest to isolate.
+if [[ "$doc_only" == true ]]; then
+ if [[ "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+ echo 'Documentation-only selection cannot produce nextest qualification evidence.' >&2
+ exit 2
+ fi
+ exec cargo test "${common[@]}" --jobs 2 "$@"
+fi
if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
mkdir -p "$HACK_RUST_TEST_REPORT_DIR"
# JUnit contains executed tests; keep discovery separately to count ignored
@@ -70,6 +86,7 @@ for ((iteration=1; iteration<=repetitions; iteration++)); do
fi
# Never reset a failing streak, retry, or continue to another suite after failure.
if [[ "$status" != 0 ]]; then exit "$status"; fi
- # Nextest excludes doctests; preserve cargo test's documentation coverage.
- cargo test --doc "${common[@]}" --jobs 2 "$@"
+ # Nextest excludes doctests. Cargo excludes them for explicit target selection
+ # too, and rejects --doc combined with those selectors.
+ if [[ "$run_doctests" == true ]]; then cargo test --doc "${common[@]}" --jobs 2 "$@"; fi
done
diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts
index f31baae9e..6037b3fb8 100644
--- a/tests/runtime-core-test-runner.test.ts
+++ b/tests/runtime-core-test-runner.test.ts
@@ -123,6 +123,57 @@ test("process runner executes nextest with zero retries then retains doctests fo
expect(command).toContain("--all-features");
}
});
+test("explicit targets run only the selected nextest suite without conflicting doctest selectors", async () => {
+ for (const args of [
+ ["--lib"],
+ ["--bins"],
+ ["--bin", "hack-native"],
+ ["--bin=hack-native"],
+ ["--tests"],
+ ["--test", "stream_relay_contract", "--all-features"],
+ ["--test=stream_relay_contract", "--all-features"],
+ ["--examples"],
+ ["--example", "fixture"],
+ ["--example=fixture"],
+ ["--benches"],
+ ["--bench", "fixture"],
+ ["--bench=fixture"],
+ ["--all-targets"],
+ ]) {
+ const r = await (await fixture()).run(args);
+ expect(r.code, r.stderr).toBe(0);
+ expect(r.commands).toHaveLength(1);
+ expect(r.commands[0]).toStartWith("nextest run");
+ for (const argument of args) {
+ expect(r.commands[0]).toContain(argument);
+ }
+ }
+});
+test("explicit documentation selection stays with Cargo and never invokes nextest", async () => {
+ const r = await (await fixture()).run(["--doc", "--all-features"]);
+ expect(r.code, r.stderr).toBe(0);
+ expect(r.commands).toHaveLength(1);
+ expect(r.commands[0]).toStartWith("test --locked");
+ expect(r.commands[0]).toContain("--doc --all-features");
+});
+test("documentation-only selection cannot claim qualification repetitions or nextest reports", async () => {
+ for (const opts of [{ repeats: "20" }, { report: true }]) {
+ const r = await (await fixture(opts)).run(["--doc"]);
+ expect(r.code).toBe(2);
+ expect(r.commands).toEqual([]);
+ expect(r.stderr).toContain("cannot produce nextest qualification evidence");
+ }
+});
+test("arguments after the separator cannot suppress the separate doctest step", async () => {
+ const r = await (await fixture()).run(["--", "--lib"]);
+ expect(r.code, r.stderr).toBe(0);
+ expect(r.commands).toHaveLength(2);
+ expect(r.commands[0]).toStartWith("nextest run");
+ expect(r.commands[1]).toStartWith("test --doc");
+ for (const command of r.commands) {
+ expect(command).toEndWith("-- --lib");
+ }
+});
for (const fail of ["nextest", "doc"]) {
test(`20-pass qualification stops at first ${fail} failure without retry or fallback`, async () => {
const r = await (await fixture({ fail, repeats: "20" })).run();
From f230fc98112be43c4b5afb894b448009f01b4bd1 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 01:05:14 +0000
Subject: [PATCH 3/3] fix(ci): preserve contributor nextest versions and probe
Cargo lookup
---
.ai/skills/hack-repo-verify/SKILL.md | 8 ++--
packages/runtime-core/README.md | 13 ++++---
scripts/test-runtime-core.sh | 25 ++++++------
tests/runtime-core-test-runner.test.ts | 54 +++++++++++++++++++++++---
4 files changed, 74 insertions(+), 26 deletions(-)
diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md
index 6114107f3..b8ca1ba9d 100644
--- a/.ai/skills/hack-repo-verify/SKILL.md
+++ b/.ai/skills/hack-repo-verify/SKILL.md
@@ -70,9 +70,11 @@ Cargo; explicit `--doc` uses Cargo alone.
Pass Cargo build, feature and target options through this wrapper; use
`cargo nextest` directly for nextest-specific options such as `-E`.
This prevents another test's spawned child from inheriting a briefly non-CLOEXEC
-macOS descriptor and delaying EOF. `bun run test:local` uses that pin when present;
-otherwise it prints `cargo install --locked --version 0.9.151 cargo-nextest` and
-retains `cargo test` (one libtest thread on macOS). The nextest test group preserves
+macOS descriptor and delaying EOF. `bun run test:local` probes `cargo nextest
+--version`, matching the subcommand it executes. A missing or different version
+prints one fallback line and retains `cargo test` (one libtest thread on macOS),
+leaving the installed nextest alone. CI and qualification reports/repetitions
+require the exact pin. The nextest test group preserves
the admission-pool tests' serialized host-wide disk probes. A manual CI dispatch
with `rust_repetitions=20` runs 20 consecutive default and 20 all-feature Mac suites,
stopping at the first failure, and preserves per-pass JUnit counts/timings.
diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md
index b1c294bed..677ff280b 100644
--- a/packages/runtime-core/README.md
+++ b/packages/runtime-core/README.md
@@ -570,11 +570,14 @@ The wrapper accepts Cargo build, feature and target options; nextest-specific
options such as `-E` need a direct `cargo nextest` invocation. Documentation-only
selection refuses repetition or report settings intended for nextest qualification.
-`bun run test:local` uses the same runner when installed. Install its exact pin with
-`cargo install --locked --version 0.9.151 cargo-nextest`. Without nextest the command
-prints that instruction and retains stock `cargo test`; the macOS fallback sets
-one libtest thread to prevent cross-test descriptor inheritance. CI requires the
-pin and never silently falls back. `bun run test:local --all-features` retains the
+`bun run test:local` checks `cargo nextest --version`, matching Cargo's lookup for
+the test run. If that resolves the pin, it uses nextest; a missing or different
+version prints one fallback line and uses stock `cargo test`, leaving the
+contributor's installed version alone. The macOS fallback sets one libtest thread
+to prevent cross-test descriptor inheritance. To opt into the pinned runner,
+install it with `cargo install --locked --version 0.9.151 cargo-nextest`.
+CI and qualification reports/repetitions require the pin and never fall back.
+`bun run test:local --all-features` retains the
Zig prerequisite above. The toolchain container's existing `rust` task continues
to use stock Cargo on Linux, where the descriptor creation primitives are atomic.
diff --git a/scripts/test-runtime-core.sh b/scripts/test-runtime-core.sh
index 94ec107d5..8d617307a 100755
--- a/scripts/test-runtime-core.sh
+++ b/scripts/test-runtime-core.sh
@@ -9,20 +9,21 @@ repetitions=${HACK_RUST_TEST_REPETITIONS:-1}
case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac
case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac
common=(--locked --manifest-path packages/runtime-core/Cargo.toml --target-dir "${HACK_RUST_TEST_TARGET_DIR:-.hack-local/target}")
-if ! command -v cargo-nextest >/dev/null 2>&1; then
- if [[ "$runner" == nextest || "$repetitions" != 1 ]]; then
- echo "Pinned cargo-nextest $version is required. Install: $install" >&2
- exit 2
- fi
- echo "cargo-nextest absent; using cargo test. For process isolation install: $install" >&2
- # A single libtest thread avoids cross-test spawn/close races in the fallback.
- if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi
- exec cargo test "${common[@]}" --jobs 2 "$@"
-fi
-actual=$(cargo-nextest --version)
+# Cargo's subcommand lookup can differ from a direct PATH lookup. Probe the
+# same command used below, and leave a contributor's other nextest version alone.
+actual=$(cargo nextest --version 2>/dev/null) || actual=""
case "$actual" in
"cargo-nextest $version"|"cargo-nextest $version ("*) ;;
- *) echo "Pinned cargo-nextest $version is required. Install: $install" >&2; exit 2 ;;
+ *)
+ if [[ "$runner" == nextest || "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+ echo "Pinned cargo-nextest $version is required. Install: $install" >&2
+ exit 2
+ fi
+ echo "cargo nextest $version unavailable; using cargo test." >&2
+ # A single libtest thread avoids cross-test spawn/close races in the fallback.
+ if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi
+ exec cargo test "${common[@]}" --jobs 2 "$@"
+ ;;
esac
mode=default
run_doctests=true
diff --git a/tests/runtime-core-test-runner.test.ts b/tests/runtime-core-test-runner.test.ts
index 6037b3fb8..4fc0dbaec 100644
--- a/tests/runtime-core-test-runner.test.ts
+++ b/tests/runtime-core-test-runner.test.ts
@@ -23,6 +23,7 @@ async function fixture(
platform?: string;
fail?: string;
version?: string;
+ pathVersion?: string;
repeats?: string;
required?: boolean;
report?: boolean;
@@ -54,13 +55,13 @@ async function fixture(
expect(await versionChild.exited).toBe(0);
await writeFile(
join(tools, "cargo"),
- `#!/bin/sh\nprintf '%s\\n' "$*" >> "$FIXTURE_LOG"\nprintf '%s' "$RUST_TEST_THREADS" > "$FIXTURE_THREADS"\ncase "$1:$2:$FIXTURE_FAIL" in nextest:run:nextest) exit 77;; test:--doc:doc) exit 78;; esac\nif [ "$1:$2" = nextest:run ] && [ -n "$FIXTURE_REPORT" ]; then mkdir -p "$(dirname "$FIXTURE_REPORT")"; printf '\\n' > "$FIXTURE_REPORT"; fi\n`,
+ `#!/bin/sh\nif [ "$1:$2" = nextest:--version ]; then [ "$FIXTURE_NEXTEST" = present ] || exit 101; printf 'cargo-nextest %s\\n' "$FIXTURE_CARGO_VERSION"; exit 0; fi\nprintf '%s\\n' "$*" >> "$FIXTURE_LOG"\nprintf '%s' "$RUST_TEST_THREADS" > "$FIXTURE_THREADS"\ncase "$1:$2:$FIXTURE_FAIL" in nextest:run:nextest) exit 77;; test:--doc:doc) exit 78;; esac\nif [ "$1:$2" = nextest:run ] && [ -n "$FIXTURE_REPORT" ]; then mkdir -p "$(dirname "$FIXTURE_REPORT")"; printf '\\n' > "$FIXTURE_REPORT"; fi\n`,
{ mode: 0o700 }
);
if (opts.nextest !== false) {
await writeFile(
join(tools, "cargo-nextest"),
- `#!/bin/sh\nprintf '%s\\n' 'cargo-nextest ${opts.version ?? version}'\n`,
+ `#!/bin/sh\nprintf 'direct PATH lookup' > "$FIXTURE_DIRECT_LOOKUP"\nprintf '%s\\n' 'cargo-nextest ${opts.pathVersion ?? opts.version ?? version}'\n`,
{ mode: 0o700 }
);
}
@@ -75,6 +76,9 @@ async function fixture(
PATH: `${tools}:/usr/bin:/bin`,
FIXTURE_LOG: log,
FIXTURE_THREADS: join(root, "threads"),
+ FIXTURE_NEXTEST: opts.nextest === false ? "absent" : "present",
+ FIXTURE_CARGO_VERSION: opts.version ?? version,
+ FIXTURE_DIRECT_LOOKUP: join(root, "direct-lookup"),
FIXTURE_REPORT: opts.report
? join(root, "packages/runtime-core/target/nextest/ci/junit.xml")
: undefined,
@@ -199,8 +203,8 @@ for (const platform of ["Linux", "Darwin"]) {
expect(r.commands).toHaveLength(1);
expect(r.commands[0]).toStartWith("test --locked");
expect(r.commands[0]).toContain("client_relay");
- expect(r.stderr).toContain(
- `cargo install --locked --version ${r.version} cargo-nextest`
+ expect(r.stderr.trim()).toBe(
+ `cargo nextest ${r.version} unavailable; using cargo test.`
);
expect(r.threads).toBe(platform === "Darwin" ? "1" : "");
});
@@ -208,8 +212,10 @@ for (const platform of ["Linux", "Darwin"]) {
test("CI never degrades to libtest when the pinned process runner is missing or mismatched", async () => {
for (const opts of [
{ nextest: false, required: true },
- { version: "0.9.150" },
- { version: "0.9.1512" },
+ { version: "0.9.150", required: true },
+ { version: "0.9.1512", required: true },
+ { version: "0.9.150", repeats: "20" },
+ { version: "0.9.150", report: true },
]) {
const r = await (await fixture(opts)).run();
expect(r.code).toBe(2);
@@ -217,6 +223,42 @@ test("CI never degrades to libtest when the pinned process runner is missing or
expect(r.stderr).toContain("cargo install --locked --version");
}
});
+test("another contributor nextest version falls back without replacing it on either host", async () => {
+ for (const platform of ["Linux", "Darwin"]) {
+ const r = await (await fixture({ version: "0.9.150", platform })).run([
+ "--lib",
+ "--all-features",
+ ]);
+ expect(r.code, r.stderr).toBe(0);
+ expect(r.commands).toHaveLength(1);
+ expect(r.commands[0]).toStartWith("test --locked");
+ expect(r.commands[0]).toEndWith("--lib --all-features");
+ expect(r.stderr.trim()).toBe(
+ `cargo nextest ${r.version} unavailable; using cargo test.`
+ );
+ expect(r.threads).toBe(platform === "Darwin" ? "1" : "");
+ }
+});
+test("the version check follows Cargo subcommand resolution instead of the PATH binary", async () => {
+ const rightCargo = await fixture({
+ pathVersion: "0.9.150",
+ required: true,
+ });
+ const accepted = await rightCargo.run();
+ expect(accepted.code, accepted.stderr).toBe(0);
+ expect(accepted.commands[0]).toStartWith("nextest run");
+ const wrongCargo = await fixture({
+ version: "0.9.150",
+ pathVersion: accepted.version,
+ required: true,
+ });
+ const refused = await wrongCargo.run();
+ expect(refused.code).toBe(2);
+ expect(refused.commands).toEqual([]);
+ for (const f of [wrongCargo, rightCargo]) {
+ expect(await Bun.file(join(f.root, "direct-lookup")).exists()).toBe(false);
+ }
+});
test("per-pass evidence is read from the Cargo workspace store even with a separate target directory", async () => {
const f = await fixture({ report: true });