fix(ci): isolate Rust tests to prevent macOS descriptor leaks (HACK-2203)

.ai/skills/hack-repo-verify/SKILL.md

@@ -63,6 +63,22 @@ CI also lints those scripts and runs their transport tests; lint changed ones ex
 Rust default gates are `bun run check:local` and `bun run test:local`. Add `--all-features` to equivalent Cargo commands for feature
 changes; the toolchain Rust tasks already enable all features. Keep default coverage.
 
+Candidate CI on Linux and macOS runs pinned cargo-nextest 0.9.151, one process per
+test and zero retries, then `cargo test --doc` with the same feature selection.
+Explicit target selections (`--lib`, `--test <name>`, etc.) skip doctests, matching
+Cargo; explicit `--doc` uses Cargo alone.
+Pass Cargo build, feature and target options through this wrapper; use
+`cargo nextest` directly for nextest-specific options such as `-E`.
+This prevents another test's spawned child from inheriting a briefly non-CLOEXEC
+macOS descriptor and delaying EOF. `bun run test:local` probes `cargo nextest
+--version`, matching the subcommand it executes. A missing or different version
+prints one fallback line and retains `cargo test` (one libtest thread on macOS),
+leaving the installed nextest alone. CI and qualification reports/repetitions
+require the exact pin. The nextest test group preserves
+the admission-pool tests' serialized host-wide disk probes. A manual CI dispatch
+with `rust_repetitions=20` runs 20 consecutive default and 20 all-feature Mac suites,
+stopping at the first failure, and preserves per-pass JUnit counts/timings.
+
 `bun run test:local` does not enable `native-stream-relay`, so
 `stream_relay_contract` reports zero tests. This default pass does not qualify the
 relay contract. With Zig 0.15.2 available, run the local all-features equivalent:

.config/nextest.toml

@@ -0,0 +1,25 @@
+# Both CI hosts use a separate process for every Rust test. Keep ignored fixtures
+# ignored: their parent tests invoke them explicitly with the stock test binary.
+[profile.default]
+retries = 0
+
+[profile.ci]
+retries = 0
+fail-fast = true
+slow-timeout = { period = "60s", terminate-after = 10 }
+
+[profile.ci.junit]
+path = "junit.xml"
+
+# The old crate mutex cannot serialize separate processes. Preserve its bound on
+# whole-host lsof scans, without serializing unrelated runtime tests.
+[test-groups.host-disk-probes]
+max-threads = 1
+
+[[profile.default.overrides]]
+filter = 'test(provider::lifecycle::admission_pool::tests::)'
+test-group = 'host-disk-probes'
+
+[[profile.ci.overrides]]
+filter = 'test(provider::lifecycle::admission_pool::tests::)'
+test-group = 'host-disk-probes'

.github/workflows/ci.yml

@@ -16,6 +16,11 @@ on:
         type: boolean
         required: true
         default: true
+      rust_repetitions:
+        description: Consecutive macOS Rust qualification passes (stops on first failure)
+        type: choice
+        options: ["1", "20"]
+        default: "1"
 
 concurrency:
   # Let the active next run finish, including jobs waiting for macOS capacity.
@@ -372,7 +377,10 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.changes.outputs.defer_macos == 'true' && '["ubuntu-latest"]' || '["ubuntu-latest","macos-latest"]') }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && inputs.rust_repetitions == '20' && 300 || 45 }}
+    env:
+      HACK_RUST_TEST_RUNNER: nextest
+      HACK_RUST_TEST_REPETITIONS: ${{ matrix.os == 'macos-latest' && github.event_name == 'workflow_dispatch' && inputs.rust_repetitions || '1' }}
     permissions:
       contents: read
     steps:
@@ -437,14 +445,30 @@ jobs:
           python3 tests/fixtures/environment-launcher-tty.py "$RUNNER_TEMP/interactive-environment-launcher"
       - name: Check candidate formatting
         run: cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check
+      - name: Install pinned process-per-test runner
+        shell: bash
+        run: |
+          bash scripts/install-cargo-nextest.sh "$RUNNER_TEMP/candidate-nextest"
+          echo "$RUNNER_TEMP/candidate-nextest" >> "$GITHUB_PATH"
       - name: Lint candidate core
         run: cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings
       - name: Test candidate isolation contracts
-        run: cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --jobs 2
+        env:
+          HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/candidate-rust-results
+        run: bash scripts/test-runtime-core.sh
       - name: Lint candidate optional features
         run: cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --all-features --jobs 2 -- -D warnings
       - name: Test candidate optional feature contracts
-        run: cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-features --jobs 2
+        env:
+          HACK_RUST_TEST_REPORT_DIR: ${{ runner.temp }}/candidate-rust-results
+        run: bash scripts/test-runtime-core.sh --all-features
+      - name: Preserve per-pass Rust counts and timings
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: candidate-rust-results-${{ matrix.os }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/candidate-rust-results/
+          if-no-files-found: ignore
       - name: Build and inspect local candidate
         run: |
           ./scripts/build-hack-local.sh

package.json

@@ -19,7 +19,7 @@
     "build:config-compiler": "bun scripts/build-config-compiler.ts",
     "check:config-compiler": "cargo fmt --manifest-path packages/config-compiler/Cargo.toml --check && cargo clippy --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --all-targets --jobs 2 -- -D warnings",
     "test:config-compiler": "cargo test --locked --manifest-path packages/config-compiler/Cargo.toml --target-dir .hack-local/config-compiler-target --jobs 2",
-    "test:local": "cargo test --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --jobs 2",
+    "test:local": "bash scripts/test-runtime-core.sh",
     "check:native-guest": "sh scripts/check-native-guest.sh",
     "check:local": "cargo fmt --manifest-path packages/runtime-core/Cargo.toml --check && cargo clippy --locked --manifest-path packages/runtime-core/Cargo.toml --target-dir .hack-local/target --all-targets --jobs 2 -- -D warnings",
     "build:runtime-image": "bun run --cwd packages/cli build:runtime-image",

packages/runtime-core/README.md

@@ -555,6 +555,44 @@ turn on no features and do not need Zig. Root `mise.toml` and `.tool-versions` p
 Zig 0.15.2. The toolchain image installs the `mise.toml` pins and runs its tasks
 under `mise exec`. CI downloads a checksum-verified 0.15.2 archive.
 
+### Rust test processes
+
+Both Candidate core CI hosts use **cargo-nextest 0.9.151**, downloaded from its
+pinned release and SHA-256 verified, for default and all-feature tests. Each test
+runs in its own process: on macOS the two-call close-on-exec setup briefly permits
+another test's spawned child to inherit a socket and delay EOF. Retries are zero.
+The admission-pool test group retains the old mutex's serialization of whole-host
+`lsof` scans. Ignored helpers remain ignored and their owning tests still invoke
+them explicitly; `cargo test --doc` runs separately because nextest omits doctests.
+Focused target selections such as `bun run test:local --lib` or `--test <name>`
+skip doctests, matching Cargo; explicit `--doc` selection runs through Cargo alone.
+The wrapper accepts Cargo build, feature and target options; nextest-specific
+options such as `-E` need a direct `cargo nextest` invocation. Documentation-only
+selection refuses repetition or report settings intended for nextest qualification.
+
+`bun run test:local` checks `cargo nextest --version`, matching Cargo's lookup for
+the test run. If that resolves the pin, it uses nextest; a missing or different
+version prints one fallback line and uses stock `cargo test`, leaving the
+contributor's installed version alone. The macOS fallback sets one libtest thread
+to prevent cross-test descriptor inheritance. To opt into the pinned runner,
+install it with `cargo install --locked --version 0.9.151 cargo-nextest`.
+CI and qualification reports/repetitions require the pin and never fall back.
+`bun run test:local --all-features` retains the
+Zig prerequisite above. The toolchain container's existing `rust` task continues
+to use stock Cargo on Linux, where the descriptor creation primitives are atomic.
+
+For bounded hosted qualification, dispatch `ci.yml` at the exact branch/commit
+with `full=true` and `rust_repetitions=20`. Only macOS repeats: 20 default suites
+and 20 all-feature suites, each followed by doctests. A failed iteration stops
+the job immediately; this is a consecutive-pass experiment, not retry policy.
+The `candidate-rust-results-*` artifacts retain each pass's counts and duration.
+Discovery JSON and stock libtest lists retain ignored-test counts separately:
+nextest's JUnit report contains executed tests, not ignored fixtures. The CI profile
+terminates a hung test after ten 60-second intervals, above the existing 500-second
+fixture budgets, so a hang fails with evidence. Qualification dispatches allow up
+to 300 minutes; ordinary Candidate jobs keep their 45-minute limit. Record measured
+test-step times as well as the pass count before treating the loop as qualified.
+
 Source binaries embed their build checkout. If a copied Cargo target directory
 retains a different checkout identity, use a fresh `--target-dir` for checks and
 tests; Cargo reporting a cached binary as fresh does not qualify that identity.

scripts/install-cargo-nextest.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Official stable release, verified against the release asset SHA-256 metadata.
+version=0.9.151
+if [[ "${1-}" == --print-version ]]; then
+  printf '%s\n' "$version"
+  exit 0
+fi
+if [[ "$#" != 1 ]]; then
+  echo 'Usage: bash scripts/install-cargo-nextest.sh DESTINATION' >&2
+  exit 2
+fi
+case "$(uname -s)-$(uname -m)" in
+  Linux-x86_64)
+    target=x86_64-unknown-linux-gnu
+    checksum=1d4ce302b820c41f4b89c5fb9bce45f0f0c94882c1cf08fc360caf08be6e7df4
+    ;;
+  Darwin-arm64|Darwin-x86_64)
+    target=universal-apple-darwin
+    checksum=d1d7d025d4bfc9fb154cd76aea3949bd78534829a0589abefa6e5dd4edb135dd
+    ;;
+  *) echo 'Unsupported nextest release platform.' >&2; exit 2 ;;
+esac
+archive="cargo-nextest-${version}-${target}.tar.gz"
+destination=$1
+mkdir -p "$destination"
+curl --fail --location --proto '=https' --proto-redir '=https' --connect-timeout 10 --max-time 120 \
+  "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${version}/${archive}" \
+  -o "$destination/$archive"
+if command -v shasum >/dev/null 2>&1; then
+  (cd "$destination" && printf '%s  %s\n' "$checksum" "$archive" | shasum -a 256 --check)
+else
+  (cd "$destination" && printf '%s  %s\n' "$checksum" "$archive" | sha256sum --check)
+fi
+tar -xzf "$destination/$archive" -C "$destination" cargo-nextest
+"$destination/cargo-nextest" --version

scripts/test-runtime-core.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)
+cd "$root"
+version=$(bash scripts/install-cargo-nextest.sh --print-version)
+install="cargo install --locked --version $version cargo-nextest"
+runner=${HACK_RUST_TEST_RUNNER:-auto}
+repetitions=${HACK_RUST_TEST_REPETITIONS:-1}
+case "$runner" in auto|nextest) ;; *) echo 'HACK_RUST_TEST_RUNNER must be auto or nextest.' >&2; exit 2 ;; esac
+case "$repetitions" in 1|20) ;; *) echo 'HACK_RUST_TEST_REPETITIONS must be 1 or 20.' >&2; exit 2 ;; esac
+common=(--locked --manifest-path packages/runtime-core/Cargo.toml --target-dir "${HACK_RUST_TEST_TARGET_DIR:-.hack-local/target}")
+# Cargo's subcommand lookup can differ from a direct PATH lookup. Probe the
+# same command used below, and leave a contributor's other nextest version alone.
+actual=$(cargo nextest --version 2>/dev/null) || actual=""
+case "$actual" in
+  "cargo-nextest $version"|"cargo-nextest $version ("*) ;;
+  *)
+    if [[ "$runner" == nextest || "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+      echo "Pinned cargo-nextest $version is required. Install: $install" >&2
+      exit 2
+    fi
+    echo "cargo nextest $version unavailable; using cargo test." >&2
+    # A single libtest thread avoids cross-test spawn/close races in the fallback.
+    if [[ "$(uname -s)" == Darwin ]]; then export RUST_TEST_THREADS=1; fi
+    exec cargo test "${common[@]}" --jobs 2 "$@"
+    ;;
+esac
+mode=default
+run_doctests=true
+doc_only=false
+for argument in "$@"; do
+  case "$argument" in
+    --) break ;;
+    --all-features) mode=all-features ;;
+    --doc) doc_only=true ;;
+    --lib|--bins|--bin|--bin=*|--tests|--test|--test=*|--examples|--example|--example=*|--benches|--bench|--bench=*|--all-targets)
+      run_doctests=false ;;
+  esac
+done
+# Documentation-only selection has no libtest process for nextest to isolate.
+if [[ "$doc_only" == true ]]; then
+  if [[ "$repetitions" != 1 || -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+    echo 'Documentation-only selection cannot produce nextest qualification evidence.' >&2
+    exit 2
+  fi
+  exec cargo test "${common[@]}" --jobs 2 "$@"
+fi
+if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+  mkdir -p "$HACK_RUST_TEST_REPORT_DIR"
+  # JUnit contains executed tests; keep discovery separately to count ignored
+  # fixtures and compare names with the stock libtest inventory.
+  cargo nextest list "${common[@]}" --build-jobs 2 --config-file "$root/.config/nextest.toml" \
+    --message-format json "$@" > "$HACK_RUST_TEST_REPORT_DIR/$mode-inventory.json"
+  cargo nextest show-config test-groups "${common[@]}" --build-jobs 2 \
+    --config-file "$root/.config/nextest.toml" --profile ci --groups host-disk-probes "$@" \
+    > "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"
+  if [[ "$(uname -s)" == Darwin ]] && ! grep -q 'provider::lifecycle::admission_pool::tests::' "$HACK_RUST_TEST_REPORT_DIR/$mode-groups.txt"; then
+    echo 'The macOS disk-probe tests are missing their serial nextest group.' >&2
+    exit 2
+  fi
+  cargo test "${common[@]}" --jobs 2 "$@" -- --list > "$HACK_RUST_TEST_REPORT_DIR/$mode-libtest.txt"
+  cargo test "${common[@]}" --jobs 2 "$@" -- --list --ignored > "$HACK_RUST_TEST_REPORT_DIR/$mode-ignored.txt"
+fi
+for ((iteration=1; iteration<=repetitions; iteration++)); do
+  echo "Rust $mode suite $iteration/$repetitions: process per test, retries=0"
+  if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+    # nextest's store is relative to Cargo's workspace (this standalone crate),
+    # independently of the shared Cargo --target-dir.
+    source="$root/packages/runtime-core/target/nextest/ci/junit.xml"
+    # This is nextest's regenerable result, not a user file. A build failure
+    # must never copy a previous successful iteration as the failed one's proof.
+    rm -f -- "$source"
+  fi
+  status=0
+  cargo nextest run "${common[@]}" --build-jobs 2 \
+    --config-file "$root/.config/nextest.toml" --profile ci --retries 0 "$@" || status=$?
+  if [[ -n "${HACK_RUST_TEST_REPORT_DIR:-}" ]]; then
+    if [[ -f "$source" ]]; then
+      mkdir -p "$HACK_RUST_TEST_REPORT_DIR"
+      report="$HACK_RUST_TEST_REPORT_DIR/$mode-$iteration.xml"
+      if [[ -e "$report" ]]; then echo 'Rust report already exists; refusing to replace it.' >&2; exit 2; fi
+      cp -n "$source" "$report"
+    elif [[ "$status" == 0 ]]; then
+      echo 'Nextest succeeded without its required JUnit evidence.' >&2
+      exit 2
+    fi
+  fi
+  # Never reset a failing streak, retry, or continue to another suite after failure.
+  if [[ "$status" != 0 ]]; then exit "$status"; fi
+  # Nextest excludes doctests. Cargo excludes them for explicit target selection
+  # too, and rejects --doc combined with those selectors.
+  if [[ "$run_doctests" == true ]]; then cargo test --doc "${common[@]}" --jobs 2 "$@"; fi
+done