From 0c7acc00d3831a7456ab6074512d5f9fac1ec4e9 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 19:03:30 -0500
Subject: [PATCH 01/45] feat(lifecycle): add standalone Rust process supervisor
---
.github/workflows/lifecycle-supervisor.yml | 46 +
docs/diagrams/lifecycle-supervisor.mmd | 27 +
docs/diagrams/lifecycle-supervisor.svg | 66 +
docs/lifecycle-supervisor.md | 169 +++
docs/lifecycle.md | 4 +
docs/reference/v5-model-coverage.md | 2 +
packages/lifecycle-supervisor/Cargo.lock | 114 ++
packages/lifecycle-supervisor/Cargo.toml | 15 +
packages/lifecycle-supervisor/README.md | 37 +
packages/lifecycle-supervisor/src/main.rs | 42 +
packages/lifecycle-supervisor/src/os.rs | 559 +++++++++
packages/lifecycle-supervisor/src/protocol.rs | 235 ++++
packages/lifecycle-supervisor/src/storage.rs | 218 ++++
.../lifecycle-supervisor/src/supervisor.rs | 523 ++++++++
.../lifecycle-supervisor/tests/fake_caller.rs | 1084 +++++++++++++++++
scripts/lib/tla-runtime-models.ts | 8 +
.../tla/lifecycle-supervisor/COVERAGE.md | 38 +
.../LifecycleSupervisor.tla | 85 ++
.../tla/lifecycle-supervisor/negative.cfg | 4 +
.../tla/lifecycle-supervisor/positive.cfg | 4 +
20 files changed, 3280 insertions(+)
create mode 100644 .github/workflows/lifecycle-supervisor.yml
create mode 100644 docs/diagrams/lifecycle-supervisor.mmd
create mode 100644 docs/diagrams/lifecycle-supervisor.svg
create mode 100644 docs/lifecycle-supervisor.md
create mode 100644 packages/lifecycle-supervisor/Cargo.lock
create mode 100644 packages/lifecycle-supervisor/Cargo.toml
create mode 100644 packages/lifecycle-supervisor/README.md
create mode 100644 packages/lifecycle-supervisor/src/main.rs
create mode 100644 packages/lifecycle-supervisor/src/os.rs
create mode 100644 packages/lifecycle-supervisor/src/protocol.rs
create mode 100644 packages/lifecycle-supervisor/src/storage.rs
create mode 100644 packages/lifecycle-supervisor/src/supervisor.rs
create mode 100644 packages/lifecycle-supervisor/tests/fake_caller.rs
create mode 100644 tests/models/tla/lifecycle-supervisor/COVERAGE.md
create mode 100644 tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
create mode 100644 tests/models/tla/lifecycle-supervisor/negative.cfg
create mode 100644 tests/models/tla/lifecycle-supervisor/positive.cfg
diff --git a/.github/workflows/lifecycle-supervisor.yml b/.github/workflows/lifecycle-supervisor.yml
new file mode 100644
index 000000000..54bd02643
--- /dev/null
+++ b/.github/workflows/lifecycle-supervisor.yml
@@ -0,0 +1,46 @@
+name: Lifecycle supervisor
+
+on:
+ pull_request:
+ paths:
+ - packages/lifecycle-supervisor/**
+ - .github/workflows/lifecycle-supervisor.yml
+ push:
+ branches: [next, main]
+ paths:
+ - packages/lifecycle-supervisor/**
+ - .github/workflows/lifecycle-supervisor.yml
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: lifecycle-supervisor-${{ github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ contract:
+ name: Supervisor (${{ matrix.os }})
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-15, macos-15-intel]
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 15
+ env:
+ HACK_DAEMON_AUTOSTART_DISABLE: '1'
+ CARGO_TARGET_DIR: ${{ github.workspace }}/.hack-local/target/lifecycle-supervisor
+ steps:
+ - uses: actions/checkout@v4
+ - uses: dtolnay/rust-toolchain@1.97.1
+ with:
+ components: rustfmt, clippy
+ - name: Check formatting
+ run: cargo fmt --manifest-path packages/lifecycle-supervisor/Cargo.toml --check
+ - name: Lint standalone supervisor
+ run: cargo clippy --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --all-targets -- -D warnings
+ - name: Test standalone supervisor with private fake callers
+ run: cargo test --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml -- --test-threads=4
+ - name: Build standalone release executable
+ run: cargo build --locked --release --manifest-path packages/lifecycle-supervisor/Cargo.toml
diff --git a/docs/diagrams/lifecycle-supervisor.mmd b/docs/diagrams/lifecycle-supervisor.mmd
new file mode 100644
index 000000000..06cb2bd80
--- /dev/null
+++ b/docs/diagrams/lifecycle-supervisor.mmd
@@ -0,0 +1,27 @@
+---
+title: Lifecycle supervisor · ownership and recovery
+---
+stateDiagram-v2
+ direction TB
+ [*] --> Absent
+ Absent --> Starting : Reserve [locks acquired]
+ Starting --> Running : Spawn [identity durable]
+ Starting --> Exited : StartFailure
+ Running --> Recoverable : SupervisorDeath
+ Recoverable --> Recoverable : Retry [duplicate refused]
+ Running --> Orphaned : GuardDeath [members live]
+ Recoverable --> Orphaned : GuardDeath [members live]
+ Recoverable --> Stopping : Stop [identity matches]
+ Running --> Stopping : Stop [identity matches]
+ Running --> Exited : Reap [no live members]
+ Stopping --> Exited : Reap [no live members]
+ Stopping --> KillPending : Kill [grace elapsed]
+ KillPending --> Exited : Reap [no live members]
+ KillPending --> StopUnknown : StopUnknown [members live]
+ note right of Starting
+ INV identity precedes command
+ end note
+ note right of Orphaned
+ Unknown identity refuses stop
+ Live session refuses duplicate
+ end note
diff --git a/docs/diagrams/lifecycle-supervisor.svg b/docs/diagrams/lifecycle-supervisor.svg
new file mode 100644
index 000000000..32561f1ad
--- /dev/null
+++ b/docs/diagrams/lifecycle-supervisor.svg
@@ -0,0 +1,66 @@
+
\ No newline at end of file
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
new file mode 100644
index 000000000..f9158d940
--- /dev/null
+++ b/docs/lifecycle-supervisor.md
@@ -0,0 +1,169 @@
+# Lifecycle supervisor: standalone contract
+
+This first slice builds a standalone Rust supervisor with a fake caller. It does
+not change `hack up`, configuration, or the default runner. CLI integration follows
+lifecycle retirement changes; packaging is a separate change. The current default
+continues to use the lifecycle mux session described in [lifecycle](./lifecycle.md).
+
+## Artifact choice
+
+`packages/lifecycle-supervisor` builds `hack-lifecycle-supervisor`. A separate crate
+adds one executable, but compiles only its small process-management dependency set.
+A config-compiler subcommand would save that artifact while mixing pure compilation
+with process, signal and recovery code. A runtime-core subcommand would bring the
+broader native runtime dependency graph into Compose installations. The separate
+crate keeps those build costs and failure boundaries independent.
+
+The supported build targets are `aarch64-apple-darwin`, `x86_64-apple-darwin`, and
+`x86_64-unknown-linux-gnu`. A successful Linux run does not qualify macOS behavior.
+
+## Protocol and private state
+
+The binary accepts `run`, `stop`, and `status`. Input and output are one four-byte
+big-endian length followed by that many JSON bytes. Frames are bounded to 1 MiB;
+each frame read or write has one 10-second deadline. A complete frame, rather than descriptor close,
+commits delivery. Environment values travel only in the input frame and process
+memory. They are never serialized into state, command-line arguments, or errors.
+Authored commands can themselves print values; output remains unredacted and private.
+
+`run` accepts `{version: 1, root, ownerToken, processes}`. `root` is an absolute,
+existing, owner-only directory for one project instance. `ownerToken` is a fresh
+opaque ownership token supplied by the caller. Each process has `name`, `argv`,
+absolute `cwd`, and `env`. Names are safe file identifiers and must also be unique
+without regard to ASCII case, including on case-sensitive filesystems. Commands execute argv
+directly. A shell-authored command can explicitly use `sh -c`; the supervisor adds
+no shell wrapper. The full environment is explicit, with no ambient inheritance.
+
+`stop` and `status` accept `{version: 1, root, ownerToken}`. Read-only `status`
+returns persisted receipts; a receipt is not a health check. It creates no files,
+processes or ownership. Stop
+requires matching ownership and a live identity before sending a signal. Unknown
+identity is a refusal, never proof that a process is absent. A different boot or a
+leader with a different start identity proves that the old session ended; it
+grants no authority over the replacement. Old terminal receipts are retained and
+ignored only after native absence is proved.
+
+Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
+`observed`, `stopped`, or `unknown`). A stop whose confirmation times out emits
+`unknown` and exits nonzero. Callers must inspect both the receipt and exit status.
+`run` must be spawned as a fresh child without making it a process-group leader:
+the supervisor calls `setsid` itself. A caller using `detached: true` or
+`setpgid(0, 0)` is refused before authored code starts.
+
+The root contains per-command state and lock files, an instance lock and
+`lifecycle.log`. Directories are mode 0700; files are mode 0600. Symlinks,
+hard-linked files, foreign owners and broad permissions are refused. State contains
+name, ownership token, guard PID, process group, OS start identity, boot identity,
+UID, command PID when known, phase, and the direct command's exit code or signal
+when observed. It contains no command or environment.
+State is atomically published and synchronized before authored code can run.
+The guard leads a new session in its own process group. The authored command
+starts in a separate group within that session. Cleanup covers every live group
+in the session, so a command moving into another process group stays owned.
+
+The log uses the existing lifecycle reader contract:
+`UTC timestampservicestdout|stderr|metamessage`.
+Stdout and stderr drain independently; tabs in messages become four spaces. Lines
+longer than 64 KiB become consecutive records so memory remains bounded. An
+unterminated final line is retained. Records from different streams can interleave;
+there is no total ordering guarantee between stdout and stderr.
+
+All descriptors the supervisor opens are close-on-exec. The command receives only
+its intended standard descriptors: null stdin, piped stdout and stderr. On macOS,
+creation and setting close-on-exec are not always atomic. Delivery uses explicit
+frames with deadlines; process liveness comes from an OS census, and post-exit
+output draining is bounded. None depends on another process closing an accidentally
+inherited descriptor.
+
+## State model
+
+This transition table is the first model. Each row maps to a fake-caller regression
+in `packages/lifecycle-supervisor/tests/fake_caller.rs`. `guard` means a resident Rust
+child that owns the command's session and process group. The project supervisor
+owns and reaps guards; each guard owns and reaps its direct command child.
+The table's labels describe abstract control states and request outcomes. Persisted
+phases use lowercase names; `Orphaned` and `StopUnknown` are recovery observations,
+not claims that a dead or paused guard rewrote its receipt.
+
+| Before | Event and guard | Effect | After | Regression |
+| --- | --- | --- | --- | --- |
+| Absent | Start, instance and command locks acquired | Guard creates session; durable identity precedes command spawn | Starting | `starts_and_records_identity` |
+| Starting | Command spawn succeeds | Publish command PID; deliver ready receipt | Running | `starts_and_records_identity` |
+| Starting | Invalid executable or publication fails | No unrecorded command; terminate owned partial start | FailedStart/refused | `failed_start_runs_no_later_command` |
+| Running | Stop with exact owner, boot and start identity | TERM authored session members | Stopping | `stop_after_caller_closed` |
+| Stopping | No non-zombie members remain | Reap child; retain terminal evidence | Exited | `stop_after_caller_closed` |
+| Stopping | Five-second grace expires, identity still matches | KILL authored session members, keeping the guard alive | KillPending | `term_ignoring_command_is_killed_after_grace` |
+| KillPending | Census confirms no live members | Guard reaps and releases command lock | Exited | `term_ignoring_command_is_killed_after_grace` |
+| Stopping/KillPending | Confirmation deadline expires with live members | Retain identity and command lock; report uncertainty | StopUnknown | `unresponsive_guard_stop_remains_unknown` |
+| Running | Project supervisor dies | Guard and durable identity remain | Running/recoverable | `supervisor_kill_then_recover_stop` |
+| Recoverable | Another start sees held command lock or live old identity | Refuse duplicate; preserve original | Recoverable | `supervisor_kill_then_recover_stop` |
+| Running/recoverable | Guard itself is separately killed with live session members | Retain recorded session, refuse duplicate and unsafe stop | Orphaned | `dead_guard_with_live_session_refuses_duplicate_and_stop` |
+| Any live recorded | Owner differs or identity is unknown | Refuse; signal nothing | Unchanged | `foreign_owner_cannot_stop` |
+| Any recorded | Boot differs or leader PID is reused | Treat old session as absent; signal no replacement | Unchanged | `stale_reused_session_leader_does_not_block_fresh_launch` |
+| Running | Command exits; only zombies remain | Reap direct child; zombies do not hold liveness | Exited | `zombies_do_not_hold_liveness` |
+
+
+
+A resident guard closes the publication gap: command ownership is durable before
+spawn, and the group leader survives the project supervisor's death. The guard
+keeps the command lock until its session has no live members, so a restarted project
+supervisor cannot launch a second copy. If a guard itself is separately killed
+while its command survives, identity becomes unknown; recovery refuses unsafe
+signals and duplicate launch. This is a distinct failure from killing the project
+supervisor and remains visible to callers rather than being reported as a
+successful stop. Launch and stop share a bounded lock so stop cannot race past a
+partially launched plan.
+
+The table models process death and retry, not filesystem hardware failure, hostile
+same-user mutation, commands that deliberately create new sessions, or power-loss
+recovery of live processes. Commands that daemonize out of their session are outside
+this ownership contract. Native tests qualify only their host OS. Darwin has no
+pidfd: its identity-to-signal interval, and numeric group signaling on both
+platforms, retain a small kernel observation/effect race. The model assumes the
+observed identity remains valid across that interval.
+
+## Terminal behavior and later work
+
+The current mux runner supplies a terminal, though its FIFO wrapper redirects
+command output. A command that prompts or requires a controlling terminal can
+therefore depend on that mux environment today. The standalone supervisor provides
+no PTY: stdin is `/dev/null`, output is piped, and there is no controlling terminal.
+Such a command sees EOF or its own non-TTY failure. It must use a noninteractive
+mode or remain in an interactive workspace until PTY support exists.
+
+Later work can add an explicit PTY mode and a small attach client so a persistent
+interactive workspace can exist without tmux. An outside-tool provider interface
+should deliver environment on stdin, with explicit framing and launch receipts.
+Neither feature is part of this unit; interactive workspaces keep tmux by default.
+
+## Delivery gates
+
+The CLI opt-in is planned as `lifecycle.runner = "mux" | "supervisor"`, default
+`mux`, plus a test environment override. Both runners must pass existing lifecycle
+unit and e2e scenarios before that integration is ready. Packaging must include the
+binary wherever the Compose CLI installs: stable archives, prerelease bundles,
+Homebrew, self-upgrade and supported container distributions. Existing prerelease
+packaging is macOS arm64 only; Intel/Linux prerelease support needs its own explicit
+qualification rather than assuming the stable archive matrix covers it.
+
+The packaging PR inventory is:
+
+- New `scripts/build-lifecycle-supervisor.ts`, root `package.json`, and CLI build
+ wiring in `packages/cli/package.json` where needed.
+- `scripts/build-release.ts`, `scripts/update-homebrew-tap.ts`,
+ `src/lib/release-install.ts`, and `src/lib/release-staging.ts`.
+- `scripts/build-native-candidate.sh`, `scripts/prerelease-plan.ts`,
+ `scripts/installer/integrity.ts`, `scripts/installer/bundle.ts`, and retained
+ `scripts/install-prerelease.py` compatibility inventories.
+- `.github/workflows/ci.yml`, `.github/workflows/release.yml`,
+ `.github/workflows/prerelease.yml`, and `docker/slim-runtime/Dockerfile`.
+- `.hack/toolchain/run.sh`, `.hack/README.md`, and install/prerelease docs.
+- Archive, Homebrew, upgrade and prerelease inventory regression tests, plus an
+ installed supervisor artifact check analogous to the config compiler check.
+
+Default migration removes the Python/FIFO wrapper in
+`src/lib/project-lifecycle-controller.ts` and the shell execution gate in
+`src/lib/shared-host-lifecycle.ts`. Finite shell-authored hooks continue to use
+POSIX `sh -c` intentionally. Interactive tmux environment delivery stays separate.
+Zellij removal, its breaking-change notice, and the workspace-only `MuxBackend`
+contract belong to the third unit.
diff --git a/docs/lifecycle.md b/docs/lifecycle.md
index 37e052814..c30bce50b 100644
--- a/docs/lifecycle.md
+++ b/docs/lifecycle.md
@@ -4,6 +4,10 @@ Many projects need host-side setup around `hack up` (auth, local proxies, tunnel
Lifecycle is configured in `.hack/hack.config.json` under `lifecycle`.
+The [standalone Rust supervisor contract](./lifecycle-supervisor.md) describes
+the separately tested supervisor work. It does not change the current mux runner
+or expose a new configuration setting in this slice.
+
With the Compose backend, the first Ctrl-C during foreground `hack up` asks
Compose to stop the project containers. Hack waits for Compose to finish stopping
them and drains its logs before exiting with code 130. A second Ctrl-C is forwarded
diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md
index d96eb13a3..d55c0653b 100644
--- a/docs/reference/v5-model-coverage.md
+++ b/docs/reference/v5-model-coverage.md
@@ -36,6 +36,7 @@ must be attached to the eventual exact integration revision.
| Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification |
| --- | --- | --- | --- |
+| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown identities, arbitrary process exit and supervisor/guard loss; 129 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
| Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. |
| Guest control ready before exposed bridge sockets bind; identity changes or the startup deadline expires | No new TLA+ model; deterministic clock schedules both readiness orders against real Unix socket nodes | `provider/bridge.rs` waits only in fresh `finish_boot`, before adoption, and keeps running audits immediate. `bridge_startup_tests.rs` covers two slots, late creation and mode tightening, permanent absence/unsafe modes, type/UID/link refusals, provider loss before/after observation and late success rejection. No wait mutates sockets or grants cleanup authority; the existing verified-provider stop handles failure. `lifecycle/boot_sequence_tests.rs` drives the real waiter through the production fresh-boot sequence and rejects audit/adoption on timeout, unsafe sockets or provider loss. `native-runtime-client.test.ts` checks the fixed runtime recover command and identical-root hint reach the frontend | Two sockets, finite scheduled events, injected provider verification and monotonic clock. Kernel process identity, provider scheduling, disk durability and successful recovery remain host boundaries. Require ten consecutive real-VM Mac Ctrl-C scenarios with pool stop/start; portable tests are not that evidence. |
| CI cancellation must not lose merged-change coverage; only exact-head full evidence may qualify a prerelease | No maintained GitHub scheduler model. Finite fixture interleavings cover predecessor success/failure/cancellation, two competing full/scoped runs and two rerun attempts | `ci.yml` serializes next pushes in one non-cancelling group (GitHub cancels only the replaced pending run), cancels stale PR runs, and requires a successful before run for selection. `selectPrereleaseCiRun` reads current attempts; `requireBranchChecks` binds suite and job check IDs. `ci-trusted-policy`, `prerelease-plan` and `release-prepare-gate` regressions reject cancelled predecessors, green subsets, stale checks, skipped hosts and changed attempts | API snapshots and GitHub group/event semantics are assumptions; races after the last read remain possible. Publication rechecks the frozen protected head and full evidence but does not transactionally lock GitHub. Hosted dispatch/cancellation checks qualify wiring; no scheduler liveness or formal refinement claim. |
@@ -79,6 +80,7 @@ liveness/reachability controls have their own documented assumptions.
| Model | Positive distinct states | Configurations |
| --- | ---: | ---: |
+| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 129 | 2 |
| [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 |
| [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 |
| [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 |
diff --git a/packages/lifecycle-supervisor/Cargo.lock b/packages/lifecycle-supervisor/Cargo.lock
new file mode 100644
index 000000000..665862df1
--- /dev/null
+++ b/packages/lifecycle-supervisor/Cargo.lock
@@ -0,0 +1,114 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "hack-lifecycle-supervisor"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
diff --git a/packages/lifecycle-supervisor/Cargo.toml b/packages/lifecycle-supervisor/Cargo.toml
new file mode 100644
index 000000000..1fdb83721
--- /dev/null
+++ b/packages/lifecycle-supervisor/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "hack-lifecycle-supervisor"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.97.1"
+publish = false
+
+[dependencies]
+libc = "0.2"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+[profile.release]
+lto = "thin"
+strip = true
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
new file mode 100644
index 000000000..3fc78f6d8
--- /dev/null
+++ b/packages/lifecycle-supervisor/README.md
@@ -0,0 +1,37 @@
+# Standalone lifecycle supervisor
+
+This private crate builds `hack-lifecycle-supervisor`, an internal host-process
+owner for Linux and macOS. It has no dependency on runtime-core, a terminal
+multiplexer, Python, or a shell wrapper. Authored shell commands can still
+explicitly select a shell in their argv.
+
+The first slice is exercised by a fake caller. It does not select a new CLI runner
+or install the executable. See [the protocol, state model and delivery gates](../../docs/lifecycle-supervisor.md).
+
+Build and verify from the repository root with the pinned toolchain:
+
+```sh
+cargo +1.97.1 fmt --manifest-path packages/lifecycle-supervisor/Cargo.toml --check
+cargo +1.97.1 clippy --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor --all-targets -- -D warnings
+cargo +1.97.1 test --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor
+cargo +1.97.1 build --locked --release --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor
+```
+
+Tests create private temporary roots and explicit fake command environments. They
+need no network, Docker, user registry or running Hack daemon. Each test owns its
+processes and cleans them up. CI runs the same crate on Linux x86_64, macOS arm64,
+and macOS x86_64. A cross-target `cargo check` checks compilation only; it does not
+qualify platform signal, process identity, output or recovery behavior.
+
+`run` must be spawned as a fresh child without the caller making it a process-group
+leader. The supervisor calls `setsid` itself and detaches its standard descriptors
+after the framed ready receipt. `stop` and `status` are finite framed requests.
+Frames, not pipe closure, delimit delivery. The per-command guard publishes kernel
+identity before executing authored argv and retains ownership after the project
+supervisor dies. A killed guard with surviving session members is an unknown
+ownership state: retry and unsafe stop are refused. A command that creates another
+session with `setsid` leaves this ownership boundary and is unsupported.
+
+The caller supplies a fresh ownership token and an existing private instance root.
+No command/environment is persisted. Log output is unredacted and mode 0600.
+There is no PTY: stdin is null and stdout/stderr are pipes.
diff --git a/packages/lifecycle-supervisor/src/main.rs b/packages/lifecycle-supervisor/src/main.rs
new file mode 100644
index 000000000..41a8850f9
--- /dev/null
+++ b/packages/lifecycle-supervisor/src/main.rs
@@ -0,0 +1,42 @@
+//! Standalone lifecycle owner; no terminal, shell wrapper, or environment file.
+//!
+//! Transition table (specified before implementation):
+//! | From | Guard / action | To | Durable consequence |
+//! | absent | command lock acquired, record guard identity | starting | identity before execution |
+//! | starting | authored spawn succeeds | running | command PID, no environment |
+//! | starting | authored spawn fails | exited | no authored process |
+//! | running | verified owner requests TERM | stopping | retained ownership |
+//! | stopping | no live authored session members | exited | retained terminal receipt |
+//! | stopping | five-second grace expires | kill-pending | KILL authored session groups |
+//! | kill-pending | no live authored session members | killed | confirmed terminal receipt |
+//! | kill-pending | live members remain | kill-pending | retain guard and lock; stop unknown |
+//! | starting/running | project owner dies | same | resident guard retains lock and identity |
+//! | any live | duplicate command / lock busy | refused | no second execution |
+//! | any | token or identity mismatch | refused | no signal |
+//! | any | guard dead and authored session live | orphaned | duplicate and stop refused |
+//!
+//! A resident guard is the session leader in its own process group. Authored
+//! commands receive a different group in that session. It publishes its kernel
+//! identity before starting authored code, and remains present while any live
+//! session member remains. A project-owner crash therefore leaves a verifiable
+//! leader. Killing that guard separately is an unknown-ownership outcome and
+//! fails closed when a live group remains.
+
+mod os;
+mod protocol;
+mod storage;
+mod supervisor;
+
+fn main() {
+ if supervisor::main().is_err() {
+ // Inputs can contain credentials. Never include parser or OS error text.
+ eprintln!("Lifecycle supervisor refused or failed; values omitted.");
+ std::process::exit(1);
+ }
+}
+
+type Result = std::result::Result;
+
+fn refused() -> std::io::Error {
+ std::io::Error::other("Lifecycle operation refused; values omitted")
+}
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
new file mode 100644
index 000000000..98bc95f9d
--- /dev/null
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -0,0 +1,559 @@
+use crate::{Result, refused};
+#[cfg(target_os = "linux")]
+use std::fs;
+use std::{
+ io,
+ os::fd::AsRawFd,
+ sync::atomic::{AtomicBool, Ordering},
+};
+
+pub static STOP: AtomicBool = AtomicBool::new(false);
+
+extern "C" fn stop_signal(_: i32) {
+ STOP.store(true, Ordering::Relaxed);
+}
+
+pub fn signals() -> Result<()> {
+ // SAFETY: sigaction is initialized, handler has C ABI and only stores an
+ // atomic flag; the kernel copies the action and retains no Rust pointer.
+ unsafe {
+ let mut action: libc::sigaction = std::mem::zeroed();
+ action.sa_sigaction = libc::SIG_DFL;
+ libc::sigemptyset(&mut action.sa_mask);
+ if libc::sigaction(libc::SIGCHLD, &action, std::ptr::null_mut()) != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ // A caller's blocked signal mask survives exec. Supervision must still
+ // receive stop requests and must be able to reap its direct children.
+ if libc::sigprocmask(libc::SIG_SETMASK, &action.sa_mask, std::ptr::null_mut()) != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ action.sa_sigaction = stop_signal as *const () as usize;
+ libc::sigemptyset(&mut action.sa_mask);
+ for signal in [libc::SIGTERM, libc::SIGINT, libc::SIGHUP] {
+ if libc::sigaction(signal, &action, std::ptr::null_mut()) != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ }
+ }
+ Ok(())
+}
+
+pub fn session() -> Result<()> {
+ // SAFETY: setsid has no pointer preconditions; this process is the caller's
+ // freshly spawned child, never an interactive terminal owner.
+ if unsafe { libc::setsid() } < 0 {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(())
+}
+
+#[derive(Clone, Debug)]
+pub struct Identity {
+ pub pid: i32,
+ pub group: i32,
+ pub session: i32,
+ pub start: u64,
+ pub uid: u32,
+ pub zombie: bool,
+}
+
+#[cfg(target_os = "linux")]
+pub fn inspect(pid: i32) -> Result> {
+ use std::os::unix::fs::MetadataExt;
+ let path = format!("/proc/{pid}/stat");
+ let text = match fs::read_to_string(&path) {
+ Ok(value) => value,
+ Err(error) if absent_error(&error) => return Ok(None),
+ Err(error) => return Err(error),
+ };
+ let tail = text.rsplit_once(") ").ok_or_else(refused)?.1;
+ let fields: Vec<_> = tail.split_whitespace().collect();
+ let group = fields
+ .get(2)
+ .and_then(|v| v.parse().ok())
+ .ok_or_else(refused)?;
+ let start = fields
+ .get(19)
+ .and_then(|v| v.parse().ok())
+ .ok_or_else(refused)?;
+ let session = fields
+ .get(3)
+ .and_then(|v| v.parse().ok())
+ .ok_or_else(refused)?;
+ let metadata = match fs::metadata(&path) {
+ Ok(value) => value,
+ Err(error) if absent_error(&error) => return Ok(None),
+ Err(error) => return Err(error),
+ };
+ Ok(Some(Identity {
+ pid,
+ group,
+ session,
+ start,
+ uid: metadata.uid(),
+ zombie: fields.first() == Some(&"Z") || fields.first() == Some(&"X"),
+ }))
+}
+
+#[cfg(target_os = "linux")]
+pub fn census() -> Result> {
+ let mut result = Vec::new();
+ for entry in fs::read_dir("/proc")? {
+ let entry = entry?;
+ if let Some(pid) = entry.file_name().to_str().and_then(|v| v.parse().ok())
+ && let Some(identity) = inspect(pid)?
+ {
+ result.push(identity);
+ }
+ }
+ Ok(result)
+}
+
+#[cfg(target_os = "linux")]
+pub fn boot() -> Result {
+ Ok(fs::read_to_string("/proc/sys/kernel/random/boot_id")?
+ .trim()
+ .into())
+}
+
+#[cfg(target_os = "macos")]
+pub fn inspect(pid: i32) -> Result> {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: buffer is writable for its exact size, and no pointer is retained.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ if size as usize != std::mem::size_of::() {
+ let error = io::Error::last_os_error();
+ if size == 0 && error.raw_os_error() == Some(libc::ESRCH) {
+ return Ok(None);
+ }
+ if size == 0 && error.raw_os_error() == Some(libc::EPERM) {
+ return Err(error);
+ }
+ return Err(refused());
+ }
+ // SAFETY: the kernel initialized exactly the expected struct size above.
+ let info = unsafe { info.assume_init() };
+ // SAFETY: getsid only observes the named PID. ESRCH is an exit race.
+ let session = unsafe { libc::getsid(pid) };
+ if session < 0 {
+ let error = io::Error::last_os_error();
+ if error.raw_os_error() == Some(libc::ESRCH) {
+ return Ok(None);
+ }
+ return Err(error);
+ }
+ Ok(Some(Identity {
+ pid,
+ group: info.pbi_pgid as i32,
+ session,
+ start: info
+ .pbi_start_tvsec
+ .checked_mul(1_000_000)
+ .and_then(|v| v.checked_add(info.pbi_start_tvusec))
+ .ok_or_else(refused)?,
+ uid: info.pbi_uid,
+ zombie: info.pbi_status == libc::SZOMB as u32,
+ }))
+}
+
+#[cfg(target_os = "macos")]
+pub fn census() -> Result> {
+ const PROC_ALL_PIDS: u32 = 1;
+ // SAFETY: querying required bytes uses a null pointer and zero buffer size.
+ let size = unsafe { libc::proc_listpids(PROC_ALL_PIDS, 0, std::ptr::null_mut(), 0) };
+ if size <= 0 {
+ return Err(refused());
+ }
+ let mut pids = vec![0i32; size as usize / 4 + 1024];
+ // SAFETY: vector is initialized and live for the byte size supplied.
+ let written = unsafe {
+ libc::proc_listpids(
+ PROC_ALL_PIDS,
+ 0,
+ pids.as_mut_ptr().cast(),
+ (pids.len() * 4) as i32,
+ )
+ };
+ if written <= 0 || written as usize >= pids.len() * 4 {
+ return Err(refused());
+ }
+ let mut result = Vec::new();
+ for pid in pids
+ .into_iter()
+ .take(written as usize / 4)
+ .filter(|pid| *pid > 0)
+ {
+ match inspect(pid) {
+ Ok(Some(identity)) => result.push(identity),
+ Ok(None) => (),
+ // Unrelated protected processes cannot belong to this user's
+ // authored session. Exact guard reads remain strict in inspect.
+ Err(error) if error.raw_os_error() == Some(libc::EPERM) => (),
+ Err(error) => return Err(error),
+ }
+ }
+ Ok(result)
+}
+
+#[cfg(target_os = "macos")]
+pub fn boot() -> Result {
+ let mut bytes = [0u8; 64];
+ let mut length = bytes.len();
+ // SAFETY: read-only sysctl with writable bounded buffer, no retained pointer.
+ if unsafe {
+ libc::sysctlbyname(
+ c"kern.bootsessionuuid".as_ptr(),
+ bytes.as_mut_ptr().cast(),
+ &mut length,
+ std::ptr::null_mut(),
+ 0,
+ )
+ } != 0
+ {
+ return Err(refused());
+ }
+ if length == 0 || length > bytes.len() {
+ return Err(refused());
+ }
+ Ok(std::str::from_utf8(&bytes[..length])
+ .map_err(|_| refused())?
+ .trim_end_matches('\0')
+ .to_ascii_lowercase())
+}
+
+pub fn same(state: &crate::storage::State, identity: &Identity) -> Result<()> {
+ if state.guard_pid != identity.pid
+ || state.process_group_id != identity.group
+ || state.guard_pid != identity.session
+ || state.guard_start_time != identity.start
+ || state.uid != identity.uid
+ || state.boot_id != boot()?
+ || identity.zombie
+ {
+ return Err(refused());
+ }
+ Ok(())
+}
+
+pub fn live_group(group: i32, excluding: Option) -> Result {
+ Ok(census()?
+ .iter()
+ .any(|p| p.session == group && !p.zombie && Some(p.pid) != excluding))
+}
+
+pub fn absent_error(error: &io::Error) -> bool {
+ error.kind() == io::ErrorKind::NotFound
+ || matches!(error.raw_os_error(), Some(libc::ESRCH) | Some(libc::ENOENT))
+}
+
+/// A boot change or a reused leader PID proves the old session is gone. A
+/// missing leader does not: descendants retain its SID and require a census.
+pub fn recorded_live(state: &crate::storage::State, excluding_guard: bool) -> Result {
+ if state.boot_id != boot()? {
+ return Ok(false);
+ }
+ if let Some(identity) = inspect(state.guard_pid)?
+ && (identity.start != state.guard_start_time
+ || identity.uid != state.uid
+ || identity.session != state.guard_pid)
+ {
+ return Ok(false);
+ }
+ live_group(state.guard_pid, excluding_guard.then_some(state.guard_pid))
+}
+
+/// The guard is never a member of an authored process group. It retains the
+/// command lock while KILL is confirmed, including uninterruptible processes.
+pub fn session_signal(session: i32, signal: i32) -> Result<()> {
+ let members: Vec<_> = census()?
+ .into_iter()
+ .filter(|p| p.session == session && p.pid != session && !p.zombie)
+ .collect();
+ let groups: std::collections::BTreeSet<_> = members
+ .iter()
+ .filter(|p| p.group != session)
+ .map(|p| p.group)
+ .collect();
+ for member in members.iter().filter(|p| p.group == session) {
+ signal_member(member, signal)?;
+ }
+ for group in groups {
+ group_signal(group, signal)?;
+ }
+ Ok(())
+}
+
+fn signal_member(expected: &Identity, signal: i32) -> Result<()> {
+ #[cfg(target_os = "linux")]
+ {
+ use std::os::fd::{FromRawFd, OwnedFd};
+ // SAFETY: pidfd_open returns a newly owned CLOEXEC descriptor.
+ let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, expected.pid, 0u32) };
+ if raw < 0 {
+ let error = io::Error::last_os_error();
+ if absent_error(&error) {
+ return Ok(());
+ }
+ return Err(error);
+ }
+ let fd = unsafe { OwnedFd::from_raw_fd(raw as i32) };
+ let Some(current) = inspect(expected.pid)? else {
+ return Ok(());
+ };
+ if current.start != expected.start
+ || current.session != expected.session
+ || current.uid != expected.uid
+ {
+ return Ok(());
+ }
+ // SAFETY: the pidfd pins the verified process; null siginfo is ordinary delivery.
+ if unsafe {
+ libc::syscall(
+ libc::SYS_pidfd_send_signal,
+ fd.as_raw_fd(),
+ signal,
+ std::ptr::null::(),
+ 0u32,
+ )
+ } < 0
+ {
+ let error = io::Error::last_os_error();
+ if !absent_error(&error) {
+ return Err(error);
+ }
+ }
+ Ok(())
+ }
+ #[cfg(target_os = "macos")]
+ {
+ let Some(current) = inspect(expected.pid)? else {
+ return Ok(());
+ };
+ if current.start != expected.start
+ || current.session != expected.session
+ || current.uid != expected.uid
+ {
+ return Ok(());
+ }
+ // SAFETY: exact incarnation and session were rechecked immediately above.
+ if unsafe { libc::kill(expected.pid, signal) } != 0 {
+ let error = io::Error::last_os_error();
+ if !absent_error(&error) {
+ return Err(error);
+ }
+ }
+ Ok(())
+ }
+}
+
+/// Authored commands start a separate group and restore inherited ignored
+/// dispositions, which exec itself is not required to reset.
+pub fn command_setup() -> Result<()> {
+ // SAFETY: async-signal-safe calls in the post-fork, single-threaded child.
+ unsafe {
+ if libc::setpgid(0, 0) != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ let mut action: libc::sigaction = std::mem::zeroed();
+ action.sa_sigaction = libc::SIG_DFL;
+ libc::sigemptyset(&mut action.sa_mask);
+ #[cfg(target_os = "linux")]
+ let highest = libc::SIGRTMAX();
+ #[cfg(target_os = "macos")]
+ let highest = 31;
+ for signal in 1..=highest {
+ if signal != libc::SIGKILL
+ && signal != libc::SIGSTOP
+ && libc::sigaction(signal, &action, std::ptr::null_mut()) != 0
+ {
+ #[cfg(target_os = "linux")]
+ if (signal == 32 || signal == 33)
+ && io::Error::last_os_error().raw_os_error() == Some(libc::EINVAL)
+ {
+ continue;
+ }
+ return Err(io::Error::last_os_error());
+ }
+ }
+ let mut mask = std::mem::zeroed();
+ libc::sigemptyset(&mut mask);
+ if libc::sigprocmask(libc::SIG_SETMASK, &mask, std::ptr::null_mut()) != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ }
+ Ok(())
+}
+
+pub fn subreaper() -> Result<()> {
+ #[cfg(target_os = "linux")]
+ // SAFETY: process-local prctl enabling orphan adoption, integer options only.
+ if unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) } != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(())
+}
+
+pub fn reap() -> Result> {
+ use std::os::unix::process::ExitStatusExt;
+ let mut reaped = Vec::new();
+ loop {
+ let mut status = 0;
+ // SAFETY: waitpid writes one live integer; WNOHANG never blocks.
+ let pid = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) };
+ if pid == 0 {
+ return Ok(reaped);
+ }
+ if pid < 0 {
+ let error = io::Error::last_os_error();
+ if error.raw_os_error() == Some(libc::ECHILD) {
+ return Ok(reaped);
+ }
+ if error.kind() == io::ErrorKind::Interrupted {
+ continue;
+ }
+ return Err(error);
+ }
+ reaped.push((pid, std::process::ExitStatus::from_raw(status)));
+ }
+}
+
+pub fn close_inherited() -> Result<()> {
+ #[cfg(target_os = "linux")]
+ {
+ // SAFETY: called before opening application descriptors. Only inherited
+ // descriptors >=3 are closed; the three protocol/std descriptors remain.
+ if unsafe { libc::syscall(libc::SYS_close_range, 3u32, u32::MAX, 0u32) } == 0 {
+ return Ok(());
+ }
+ }
+ // Portable fallback is bounded by the process's descriptor limit and does
+ // not allocate or retain descriptor references while closing them.
+ let mut limit: libc::rlimit = unsafe { std::mem::zeroed() };
+ // SAFETY: getrlimit initializes one live rlimit value.
+ if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) } != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ if limit.rlim_cur > 1_048_576 {
+ return Err(refused());
+ }
+ for fd in 3..limit.rlim_cur as i32 {
+ // SAFETY: startup owns no fd >=3; failed closes are harmless gaps.
+ unsafe {
+ libc::close(fd);
+ }
+ }
+ Ok(())
+}
+
+pub fn signal_guard(state: &crate::storage::State) -> Result<()> {
+ #[cfg(target_os = "linux")]
+ {
+ use std::os::fd::{FromRawFd, OwnedFd};
+ // SAFETY: pidfd_open pins this PID incarnation and returns a fresh CLOEXEC fd.
+ let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, state.guard_pid, 0u32) };
+ if raw < 0 {
+ return Err(io::Error::last_os_error());
+ }
+ let fd = unsafe { OwnedFd::from_raw_fd(raw as i32) };
+ same(state, &inspect(state.guard_pid)?.ok_or_else(refused)?)?;
+ // SAFETY: pidfd is live; null siginfo requests ordinary signal delivery.
+ if unsafe {
+ libc::syscall(
+ libc::SYS_pidfd_send_signal,
+ fd.as_raw_fd(),
+ libc::SIGTERM,
+ std::ptr::null::(),
+ 0u32,
+ )
+ } < 0
+ {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(())
+ }
+ #[cfg(target_os = "macos")]
+ {
+ same(state, &inspect(state.guard_pid)?.ok_or_else(refused)?)?;
+ // SAFETY: guarded by the exact kernel identity check above. Darwin has no pidfd.
+ if unsafe { libc::kill(state.guard_pid, libc::SIGTERM) } != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(())
+ }
+}
+
+pub fn group_signal(group: i32, signal: i32) -> Result<()> {
+ if group <= 1 {
+ return Err(refused());
+ }
+ // SAFETY: validated positive group number is negated for group delivery.
+ if unsafe { libc::kill(-group, signal) } != 0 {
+ let error = io::Error::last_os_error();
+ if error.raw_os_error() != Some(libc::ESRCH) {
+ return Err(error);
+ }
+ }
+ Ok(())
+}
+
+pub fn nonblocking(fd: &impl AsRawFd) -> Result<()> {
+ // SAFETY: fcntl borrows a live descriptor and supplies integer flags only.
+ let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) };
+ if flags < 0
+ || unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0
+ {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(())
+}
+
+pub fn timestamp() -> Result {
+ // SAFETY: time returns current seconds; gmtime_r writes one live tm struct,
+ // strftime writes at most the supplied buffer length and retains no pointer.
+ unsafe {
+ let now = libc::time(std::ptr::null_mut());
+ let mut tm: libc::tm = std::mem::zeroed();
+ if libc::gmtime_r(&now, &mut tm).is_null() {
+ return Err(refused());
+ }
+ let mut bytes = [0u8; 32];
+ let count = libc::strftime(
+ bytes.as_mut_ptr().cast(),
+ bytes.len(),
+ c"%Y-%m-%dT%H:%M:%SZ".as_ptr(),
+ &tm,
+ );
+ if count == 0 {
+ return Err(refused());
+ }
+ Ok(std::str::from_utf8(&bytes[..count])
+ .map_err(|_| refused())?
+ .into())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn proc_exit_during_open_or_read_is_absent_not_census_failure() {
+ for errno in [libc::ENOENT, libc::ESRCH] {
+ assert!(super::absent_error(&std::io::Error::from_raw_os_error(
+ errno
+ )));
+ }
+ for errno in [libc::EPERM, libc::EIO] {
+ assert!(!super::absent_error(&std::io::Error::from_raw_os_error(
+ errno
+ )));
+ }
+ }
+}
diff --git a/packages/lifecycle-supervisor/src/protocol.rs b/packages/lifecycle-supervisor/src/protocol.rs
new file mode 100644
index 000000000..0a795a3e2
--- /dev/null
+++ b/packages/lifecycle-supervisor/src/protocol.rs
@@ -0,0 +1,235 @@
+use crate::{Result, refused};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+use std::{
+ collections::BTreeMap,
+ io::{Read, Write},
+ os::fd::AsRawFd,
+ path::PathBuf,
+ time::{Duration, Instant},
+};
+
+pub const LIMIT: usize = 1024 * 1024;
+pub const DEADLINE: Duration = Duration::from_secs(10);
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Launch {
+ pub version: u32,
+ pub root: PathBuf,
+ pub owner_token: String,
+ pub processes: Vec,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Process {
+ pub name: String,
+ pub argv: Vec,
+ pub cwd: PathBuf,
+ pub env: BTreeMap,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct Control {
+ pub version: u32,
+ pub root: PathBuf,
+ pub owner_token: String,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct GuardLaunch {
+ pub version: u32,
+ pub root: PathBuf,
+ pub owner_token: String,
+ pub process: Process,
+}
+
+pub fn valid_name(name: &str) -> bool {
+ !name.is_empty()
+ && name.as_bytes()[0].is_ascii_alphanumeric()
+ && name.len() <= 96
+ && name
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b))
+ && name != "."
+ && name != ".."
+}
+
+pub fn validate_control(version: u32, root: &std::path::Path, token: &str) -> Result<()> {
+ if version != 1
+ || !root.is_absolute()
+ || token.is_empty()
+ || token.len() > 256
+ || token.contains('\0')
+ {
+ return Err(refused());
+ }
+ Ok(())
+}
+
+impl Process {
+ pub fn validate(&self) -> Result<()> {
+ if !valid_name(&self.name)
+ || !self.cwd.is_absolute()
+ || self.argv.is_empty()
+ || self.argv[0].is_empty()
+ || self.argv.iter().any(|v| v.contains('\0'))
+ || self.env.iter().any(|(key, value)| {
+ let mut bytes = key.bytes();
+ !bytes
+ .next()
+ .is_some_and(|b| b.is_ascii_alphabetic() || b == b'_')
+ || !bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_')
+ || value.contains('\0')
+ })
+ {
+ return Err(refused());
+ }
+ Ok(())
+ }
+}
+
+/// Length and deadline, never descriptor closure, delimit a handoff. Polling
+/// handles Darwin's brief inheritable-descriptor window without waiting on EOF.
+pub fn read(input: &mut (impl Read + AsRawFd)) -> Result {
+ let deadline = Instant::now() + DEADLINE;
+ let mut header = [0; 4];
+ exact(input, &mut header, deadline)?;
+ let length = u32::from_be_bytes(header) as usize;
+ if length == 0 || length > LIMIT {
+ return Err(refused());
+ }
+ let mut bytes = vec![0; length];
+ exact(input, &mut bytes, deadline)?;
+ let parsed = serde_json::from_slice(&bytes).map_err(|_| refused());
+ bytes.fill(0);
+ parsed
+}
+
+fn exact(input: &mut (impl Read + AsRawFd), bytes: &mut [u8], deadline: Instant) -> Result<()> {
+ let mut offset = 0;
+ while offset < bytes.len() {
+ let remaining = deadline
+ .checked_duration_since(Instant::now())
+ .ok_or_else(refused)?;
+ let mut fd = libc::pollfd {
+ fd: input.as_raw_fd(),
+ events: libc::POLLIN,
+ revents: 0,
+ };
+ // SAFETY: poll borrows one initialized descriptor record for this call.
+ let result = unsafe {
+ libc::poll(
+ &mut fd,
+ 1,
+ remaining.as_millis().min(i32::MAX as u128) as i32,
+ )
+ };
+ if result < 0 && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted {
+ continue;
+ }
+ if result <= 0 {
+ return Err(refused());
+ }
+ // Read exactly the polled kernel descriptor. Stdin's high-level Read
+ // buffers ahead: polling after a four-byte header would then wait for
+ // data already held inside its user-space buffer.
+ // SAFETY: the destination slice is writable for its supplied length;
+ // read borrows a live descriptor and retains no pointer.
+ let count = unsafe {
+ libc::read(
+ input.as_raw_fd(),
+ bytes[offset..].as_mut_ptr().cast(),
+ bytes.len() - offset,
+ )
+ };
+ let result = if count < 0 {
+ Err(std::io::Error::last_os_error())
+ } else {
+ Ok(count as usize)
+ };
+ match result {
+ Ok(0) => return Err(refused()),
+ Ok(count) => offset += count,
+ Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
+ Err(error) => return Err(error),
+ }
+ }
+ Ok(())
+}
+
+pub fn write(output: &mut (impl Write + AsRawFd), value: &impl Serialize) -> Result<()> {
+ let mut bytes = serde_json::to_vec(value).map_err(|_| refused())?;
+ if bytes.is_empty() || bytes.len() > LIMIT {
+ return Err(refused());
+ }
+ let fd = output.as_raw_fd();
+ // SAFETY: fcntl borrows this live, exclusively used protocol descriptor.
+ let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
+ if flags < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ // SAFETY: only integer descriptor flags are changed for bounded writes.
+ if unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ let deadline = Instant::now() + DEADLINE;
+ let result = send(fd, &(bytes.len() as u32).to_be_bytes(), deadline)
+ .and_then(|()| send(fd, &bytes, deadline));
+ // SAFETY: restore the original flags before returning the borrowed fd.
+ let restore = unsafe { libc::fcntl(fd, libc::F_SETFL, flags) };
+ bytes.fill(0);
+ result?;
+ if restore < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+}
+
+fn send(fd: i32, bytes: &[u8], deadline: Instant) -> Result<()> {
+ let mut offset = 0;
+ while offset < bytes.len() {
+ let remaining = deadline
+ .checked_duration_since(Instant::now())
+ .ok_or_else(refused)?;
+ let mut record = libc::pollfd {
+ fd,
+ events: libc::POLLOUT,
+ revents: 0,
+ };
+ // SAFETY: poll borrows one initialized descriptor record.
+ let ready = unsafe {
+ libc::poll(
+ &mut record,
+ 1,
+ remaining.as_millis().min(i32::MAX as u128) as i32,
+ )
+ };
+ if ready < 0 && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted {
+ continue;
+ }
+ if ready <= 0 {
+ return Err(refused());
+ }
+ // SAFETY: bytes remain live for the supplied length; fd is nonblocking.
+ let count =
+ unsafe { libc::write(fd, bytes[offset..].as_ptr().cast(), bytes.len() - offset) };
+ if count < 0 {
+ let error = std::io::Error::last_os_error();
+ if matches!(
+ error.kind(),
+ std::io::ErrorKind::Interrupted | std::io::ErrorKind::WouldBlock
+ ) {
+ continue;
+ }
+ return Err(error);
+ }
+ if count == 0 {
+ return Err(refused());
+ }
+ offset += count as usize;
+ }
+ Ok(())
+}
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
new file mode 100644
index 000000000..593560a0c
--- /dev/null
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -0,0 +1,218 @@
+use crate::{Result, refused};
+use serde::{Deserialize, Serialize};
+use std::{
+ fs::{File, OpenOptions},
+ io::{Read, Write},
+ os::{
+ fd::{AsRawFd, FromRawFd, OwnedFd},
+ unix::fs::{MetadataExt, OpenOptionsExt},
+ },
+ path::{Path, PathBuf},
+};
+
+#[derive(Deserialize, Serialize, Clone, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct State {
+ pub version: u32,
+ pub name: String,
+ pub owner_token: String,
+ pub guard_pid: i32,
+ pub process_group_id: i32,
+ pub guard_start_time: u64,
+ pub boot_id: String,
+ pub uid: u32,
+ pub command_pid: Option,
+ pub phase: String,
+ #[serde(default)]
+ pub exit_code: Option,
+ #[serde(default)]
+ pub signal: Option,
+}
+
+/// Operations use an opened O_NOFOLLOW directory and relative openat/renameat;
+/// changing a pathname after admission cannot redirect state/log writes.
+pub struct Root {
+ directory: File,
+ pub path: PathBuf,
+}
+
+impl Root {
+ pub fn open(path: &Path) -> Result {
+ let directory = OpenOptions::new()
+ .read(true)
+ .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(path)?;
+ let metadata = directory.metadata()?;
+ // SAFETY: geteuid has no preconditions.
+ if !metadata.is_dir()
+ || metadata.uid() != unsafe { libc::geteuid() }
+ || metadata.mode() & 0o777 != 0o700
+ {
+ return Err(refused());
+ }
+ Ok(Self {
+ directory,
+ path: path.into(),
+ })
+ }
+
+ pub fn file(&self, name: &str, flags: i32) -> Result {
+ let name = std::ffi::CString::new(name).map_err(|_| refused())?;
+ // SAFETY: directory descriptor is live; name is NUL-terminated; returned
+ // descriptor is newly owned and immediately wrapped in RAII.
+ let fd = unsafe {
+ libc::openat(
+ self.directory.as_raw_fd(),
+ name.as_ptr(),
+ flags | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK,
+ 0o600,
+ )
+ };
+ if fd < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ // SAFETY: fd was just returned by openat and has no other owner.
+ let file = File::from(unsafe { OwnedFd::from_raw_fd(fd) });
+ let metadata = file.metadata()?;
+ // SAFETY: geteuid has no preconditions.
+ if !metadata.is_file()
+ || metadata.uid() != unsafe { libc::geteuid() }
+ || metadata.nlink() != 1
+ || metadata.mode() & 0o777 != 0o600
+ {
+ return Err(refused());
+ }
+ Ok(file)
+ }
+
+ pub fn lock(&self, name: &str) -> Result {
+ let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?;
+ // SAFETY: flock operates only on this live descriptor.
+ if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 {
+ return Err(refused());
+ }
+ Ok(file)
+ }
+
+ pub fn lock_wait(&self, name: &str) -> Result {
+ let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?;
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
+ loop {
+ // SAFETY: flock only borrows the live owned descriptor.
+ if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
+ return Ok(file);
+ }
+ let error = std::io::Error::last_os_error();
+ if !matches!(
+ error.kind(),
+ std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
+ ) {
+ return Err(error);
+ }
+ if std::time::Instant::now() >= deadline {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::TimedOut,
+ "Lifecycle lock wait expired",
+ ));
+ }
+ std::thread::sleep(std::time::Duration::from_millis(20));
+ }
+ }
+
+ pub fn save(&self, state: &State) -> Result<()> {
+ let final_name = format!("{}.json", state.name);
+ // Refuse unsafe pre-existing targets before atomically replacing an owned
+ // regular receipt. Never truncate through a link.
+ match self.file(&final_name, libc::O_RDONLY) {
+ Ok(_) => (),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
+ Err(error) => return Err(error),
+ }
+ static SERIAL: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+ let mut attempts = 0;
+ let (temporary_name, mut temporary) = loop {
+ attempts += 1;
+ if attempts > 16 {
+ return Err(refused());
+ }
+ let clock = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map_err(|_| refused())?
+ .as_nanos();
+ let serial = SERIAL.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ let name = format!(
+ ".{}.{}.{clock}.{serial}.tmp",
+ state.name,
+ std::process::id()
+ );
+ match self.file(&name, libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL) {
+ Ok(file) => break (name, file),
+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
+ Err(error) => return Err(error),
+ }
+ };
+ let bytes = serde_json::to_vec(state).map_err(|_| refused())?;
+ temporary.write_all(&bytes)?;
+ temporary.sync_all()?;
+ let temporary_name = std::ffi::CString::new(temporary_name).map_err(|_| refused())?;
+ let final_name = std::ffi::CString::new(final_name).map_err(|_| refused())?;
+ // SAFETY: both names are NUL terminated and relative to this live root.
+ if unsafe {
+ libc::renameat(
+ self.directory.as_raw_fd(),
+ temporary_name.as_ptr(),
+ self.directory.as_raw_fd(),
+ final_name.as_ptr(),
+ )
+ } != 0
+ {
+ return Err(std::io::Error::last_os_error());
+ }
+ self.directory.sync_all()
+ }
+
+ pub fn load(&self, name: &str) -> Result> {
+ let mut file = match self.file(&format!("{name}.json"), libc::O_RDONLY) {
+ Ok(file) => file,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+ Err(error) => return Err(error),
+ };
+ if file.metadata()?.len() > 8192 {
+ return Err(refused());
+ }
+ let mut bytes = Vec::new();
+ Read::by_ref(&mut file).take(8193).read_to_end(&mut bytes)?;
+ let state: State = serde_json::from_slice(&bytes).map_err(|_| refused())?;
+ if state.version != 1
+ || state.name != name
+ || state.guard_pid <= 1
+ || state.process_group_id != state.guard_pid
+ || state.guard_start_time == 0
+ {
+ return Err(refused());
+ }
+ Ok(Some(state))
+ }
+
+ pub fn states(&self) -> Result> {
+ let mut states = Vec::new();
+ // Enumeration is followed by descriptor-relative checked reads. Root path
+ // replacement can at most change the candidate names, never read location.
+ for entry in std::fs::read_dir(&self.path)? {
+ let entry = entry?;
+ if let Some(name) = entry
+ .file_name()
+ .to_str()
+ .and_then(|v| v.strip_suffix(".json"))
+ {
+ if !crate::protocol::valid_name(name) {
+ return Err(refused());
+ }
+ if let Some(state) = self.load(name)? {
+ states.push(state);
+ }
+ }
+ }
+ Ok(states)
+ }
+}
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
new file mode 100644
index 000000000..d52babf29
--- /dev/null
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -0,0 +1,523 @@
+use crate::{
+ Result, os,
+ protocol::{self, Control, GuardLaunch, Launch},
+ refused,
+ storage::{Root, State},
+};
+use serde::Serialize;
+use std::{
+ fs::File,
+ io::{Read, Write},
+ os::{
+ fd::AsRawFd,
+ unix::process::{CommandExt, ExitStatusExt},
+ },
+ process::{Child, Command, Stdio},
+ sync::atomic::Ordering,
+ thread,
+ time::{Duration, Instant},
+};
+
+const GRACE: Duration = Duration::from_secs(5);
+const TICK: Duration = Duration::from_millis(20);
+const LINE_LIMIT: usize = 64 * 1024;
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct Receipt {
+ version: u32,
+ supervisor_pid: u32,
+ outcome: &'static str,
+ processes: Vec,
+}
+
+pub fn main() -> Result<()> {
+ os::close_inherited()?;
+ let args: Vec<_> = std::env::args_os().skip(1).collect();
+ if args.len() != 1 {
+ return Err(refused());
+ }
+ match args[0].to_str() {
+ Some("run") => run(protocol::read(&mut std::io::stdin())?),
+ Some("guard") => guard(protocol::read(&mut std::io::stdin())?),
+ Some("stop") | Some("status") => {
+ let request: Control = protocol::read(&mut std::io::stdin())?;
+ protocol::validate_control(request.version, &request.root, &request.owner_token)?;
+ let root = Root::open(&request.root)?;
+ let _stop_lock = if args[0] == "stop" {
+ Some(root.lock_wait(".stop.lock")?)
+ } else {
+ None
+ };
+ let mut states = Vec::new();
+ for state in root.states()? {
+ if state.owner_token == request.owner_token {
+ states.push(state);
+ } else if os::recorded_live(&state, false)? {
+ // Old terminal receipts survive renames and new generations.
+ // Only a kernel census can prove they no longer hold work;
+ // a persisted terminal phase alone is never sufficient.
+ return Err(refused());
+ }
+ }
+ let result = if args[0] == "stop" {
+ stop(&states)
+ } else {
+ Ok(())
+ };
+ protocol::write(
+ &mut std::io::stdout(),
+ &Receipt {
+ version: 1,
+ supervisor_pid: 0,
+ outcome: if result.is_err() {
+ "unknown"
+ } else if args[0] == "stop" {
+ "stopped"
+ } else {
+ "observed"
+ },
+ processes: root
+ .states()?
+ .into_iter()
+ .filter(|state| state.owner_token == request.owner_token)
+ .collect(),
+ },
+ )?;
+ result
+ }
+ _ => Err(refused()),
+ }
+}
+
+fn run(request: Launch) -> Result<()> {
+ protocol::validate_control(request.version, &request.root, &request.owner_token)?;
+ if request.processes.is_empty() || request.processes.len() > 128 {
+ return Err(refused());
+ }
+ let mut names = std::collections::BTreeSet::new();
+ for process in &request.processes {
+ process.validate()?;
+ if !names.insert(process.name.to_ascii_lowercase()) {
+ return Err(refused());
+ }
+ }
+ let root = Root::open(&request.root)?;
+ let _instance_lock = root.lock(".instance.lock")?;
+ let launch_lock = root.lock_wait(".stop.lock")?;
+ // Validate every old claim before starting any new command. A guard survives
+ // project-owner death; its command lock also closes concurrent restart races.
+ for state in root.states()? {
+ if os::recorded_live(&state, false)? {
+ return Err(refused());
+ }
+ }
+ os::session()?;
+ os::signals()?;
+ let mut children = Vec::new();
+ let outcome = (|| {
+ let mut states = Vec::new();
+ for process in request.processes {
+ if os::STOP.load(Ordering::Relaxed) {
+ return Err(refused());
+ }
+ let mut command = Command::new(std::env::current_exe()?);
+ command
+ .arg("guard")
+ .env_clear()
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null());
+ // SAFETY: setsid is async-signal-safe, and this closure allocates no
+ // application state. It runs in the freshly forked child only.
+ unsafe {
+ command.pre_exec(os::session);
+ }
+ let child = command.spawn()?;
+ children.push(child);
+ let child = children.last_mut().ok_or_else(refused)?;
+ let plan = GuardLaunch {
+ version: 1,
+ root: request.root.clone(),
+ owner_token: request.owner_token.clone(),
+ process,
+ };
+ protocol::write(&mut child.stdin.take().ok_or_else(refused)?, &plan)?;
+ let state: State = protocol::read(&mut child.stdout.take().ok_or_else(refused)?)?;
+ if state.guard_pid != child.id() as i32 || state.owner_token != request.owner_token {
+ return Err(refused());
+ }
+ if state.phase == "failed-start" {
+ return Err(refused());
+ }
+ states.push(state);
+ }
+ protocol::write(
+ &mut std::io::stdout(),
+ &Receipt {
+ version: 1,
+ supervisor_pid: std::process::id(),
+ outcome: "ready",
+ processes: states,
+ },
+ )?;
+ drop(launch_lock);
+ detach_stdio()?;
+ loop {
+ if os::STOP.load(Ordering::Relaxed) {
+ match root.lock_wait(".stop.lock") {
+ Ok(_lock) => {
+ stop(&root.states()?)?;
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::TimedOut => (),
+ Err(error) => return Err(error),
+ }
+ }
+ let mut running = false;
+ for child in &mut children {
+ if child.try_wait()?.is_none() {
+ running = true;
+ }
+ }
+ if !running {
+ return Ok(());
+ }
+ thread::sleep(TICK);
+ }
+ })();
+ if outcome.is_err() {
+ // Rollback only this start's unreaped children. Never infer authority from
+ // another root entry while handling a launch failure.
+ rollback(&mut children);
+ }
+ outcome
+}
+
+fn rollback(children: &mut [Child]) {
+ for child in children.iter_mut() {
+ if child.try_wait().ok().flatten().is_some() {
+ continue;
+ }
+ let pid = child.id() as i32;
+ // An unreaped direct Child pins the PID; only the guard receives TERM.
+ // SAFETY: the child has not been reaped, so this PID cannot be reused.
+ unsafe {
+ libc::kill(pid, libc::SIGTERM);
+ }
+ }
+ let deadline = Instant::now() + GRACE + Duration::from_secs(2);
+ while Instant::now() < deadline {
+ if children
+ .iter_mut()
+ .all(|child| child.try_wait().ok().flatten().is_some())
+ {
+ break;
+ }
+ thread::sleep(TICK);
+ }
+ // A guard still alive after KILL confirmation remains the owner of unknown
+ // or uninterruptible work. Do not kill it and discard its lock.
+}
+
+fn guard(request: GuardLaunch) -> Result<()> {
+ protocol::validate_control(request.version, &request.root, &request.owner_token)?;
+ request.process.validate()?;
+ let root = Root::open(&request.root)?;
+ let _command_lock = root.lock(&format!("{}.lock", request.process.name))?;
+ if let Some(previous) = root.load(&request.process.name)?
+ && os::recorded_live(&previous, false)?
+ {
+ return Err(refused());
+ }
+ let pid = std::process::id() as i32;
+ let identity = os::inspect(pid)?.ok_or_else(refused)?;
+ if identity.group != pid {
+ return Err(refused());
+ }
+ os::signals()?;
+ os::subreaper()?;
+ let mut state = State {
+ version: 1,
+ name: request.process.name.clone(),
+ owner_token: request.owner_token,
+ guard_pid: pid,
+ process_group_id: pid,
+ guard_start_time: identity.start,
+ boot_id: os::boot()?,
+ uid: identity.uid,
+ command_pid: None,
+ phase: "starting".into(),
+ exit_code: None,
+ signal: None,
+ };
+ let mut log = root.file(
+ "lifecycle.log",
+ libc::O_WRONLY | libc::O_APPEND | libc::O_CREAT,
+ )?;
+ root.save(&state)?;
+ let outcome = guard_command(&request.process, &root, &mut state, &mut log);
+ if outcome.is_err() && state.command_pid.is_some() {
+ // Keep ownership while confirming internal-error cleanup. If even the
+ // census fails, retain the guard and lock rather than silently orphan.
+ let _ = os::session_signal(pid, libc::SIGTERM);
+ thread::sleep(GRACE);
+ loop {
+ let _ = os::session_signal(pid, libc::SIGKILL);
+ let _ = os::reap();
+ if matches!(os::live_group(pid, Some(pid)), Ok(false)) {
+ break;
+ }
+ thread::sleep(TICK);
+ }
+ }
+ outcome
+}
+
+fn guard_command(
+ process: &protocol::Process,
+ root: &Root,
+ state: &mut State,
+ log: &mut File,
+) -> Result<()> {
+ let mut command = Command::new(&process.argv[0]);
+ command
+ .args(&process.argv[1..])
+ .current_dir(&process.cwd)
+ .env_clear()
+ .envs(&process.env)
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ // SAFETY: command_setup uses only async-signal-safe operations after fork.
+ unsafe {
+ command.pre_exec(os::command_setup);
+ }
+ let mut child = match command.spawn() {
+ Ok(child) => child,
+ Err(_) => {
+ state.phase = "failed-start".into();
+ root.save(state)?;
+ protocol::write(&mut std::io::stdout(), state)?;
+ return Err(refused());
+ }
+ };
+ state.command_pid = Some(child.id() as i32);
+ state.phase = "running".into();
+ root.save(state)?;
+ let mut stdout = child.stdout.take().ok_or_else(refused)?;
+ let mut stderr = child.stderr.take().ok_or_else(refused)?;
+ os::nonblocking(&stdout)?;
+ os::nonblocking(&stderr)?;
+ protocol::write(&mut std::io::stdout(), state)?;
+ detach_stdio()?;
+ let mut out = Lines::default();
+ let mut err = Lines::default();
+ let mut stopping = None;
+ let mut killed = false;
+ let mut direct_exited = false;
+ let mut drained_after_exit = None;
+ loop {
+ // Alternate bounded reads so stdout bursts cannot starve stderr/stop.
+ out.drain(&mut stdout, log, &state.name, "stdout")?;
+ err.drain(&mut stderr, log, &state.name, "stderr")?;
+ if !direct_exited && let Some(status) = child.try_wait()? {
+ direct_exited = true;
+ state.exit_code = status.code();
+ state.signal = status.signal();
+ }
+ // Linux's subreaper may adopt exited grandchildren while the direct
+ // command remains alive. Reap continuously, retaining the direct child's
+ // wait status if it exits between try_wait and this raw waitpid census.
+ for (pid, status) in os::reap()? {
+ if pid == child.id() as i32 {
+ direct_exited = true;
+ state.exit_code = status.code();
+ state.signal = status.signal();
+ }
+ }
+ let live = os::live_group(state.process_group_id, Some(state.guard_pid))?;
+ if os::STOP.swap(false, Ordering::Relaxed) && stopping.is_none() {
+ state.phase = "stopping".into();
+ root.save(state)?;
+ stopping = Some(Instant::now());
+ os::session_signal(state.guard_pid, libc::SIGTERM)?;
+ }
+ if !live && direct_exited {
+ let since = *drained_after_exit.get_or_insert_with(Instant::now);
+ // Never wait on EOF: foreign fork/exec activity can transiently
+ // inherit a pipe on Darwin. Drain a bounded final interval instead.
+ if since.elapsed() >= Duration::from_millis(100) {
+ if !out.pending.is_empty() {
+ out.flush(log, &state.name, "stdout", true)?;
+ }
+ if !err.pending.is_empty() {
+ err.flush(log, &state.name, "stderr", true)?;
+ }
+ state.phase = if killed { "killed" } else { "exited" }.into();
+ root.save(state)?;
+ return Ok(());
+ }
+ }
+ if stopping.is_some_and(|start| start.elapsed() >= GRACE) {
+ state.phase = "kill-pending".into();
+ if !killed {
+ root.save(state)?;
+ killed = true;
+ }
+ os::session_signal(state.guard_pid, libc::SIGKILL)?;
+ }
+ thread::sleep(TICK);
+ }
+}
+
+fn stop(states: &[State]) -> Result<()> {
+ let mut active = Vec::new();
+ // Validate the full set before taking the first effect.
+ for state in states {
+ if !os::recorded_live(state, false)? {
+ continue;
+ }
+ let identity = os::inspect(state.guard_pid)?.ok_or_else(refused)?;
+ os::same(state, &identity)?;
+ active.push(state.clone());
+ }
+ for state in &active {
+ if let Err(error) = os::signal_guard(state) {
+ // Natural exit can race between admission and pidfd delivery.
+ // Only a new kernel absence proof turns that race into success.
+ if os::recorded_live(state, false)? {
+ return Err(error);
+ }
+ }
+ }
+ let deadline = Instant::now() + GRACE + Duration::from_secs(2);
+ while Instant::now() < deadline {
+ let mut live = false;
+ for state in &active {
+ live |= os::recorded_live(state, false)?;
+ }
+ if !live {
+ return Ok(());
+ }
+ thread::sleep(TICK);
+ }
+ // Unknown outcome: the guard retains its command lock while any live
+ // member remains. Neither timeout nor zombie-only existence implies alive.
+ Err(refused())
+}
+
+fn detach_stdio() -> Result<()> {
+ let null = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open("/dev/null")?;
+ for fd in 0..=2 {
+ // SAFETY: dup2 replaces only this process's standard descriptors with
+ // the borrowed live /dev/null descriptor; no pointer is retained.
+ if unsafe { libc::dup2(null.as_raw_fd(), fd) } < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ }
+ Ok(())
+}
+
+#[derive(Default)]
+struct Lines {
+ pending: Vec,
+}
+
+impl Lines {
+ fn drain(
+ &mut self,
+ input: &mut impl Read,
+ log: &mut File,
+ name: &str,
+ stream: &str,
+ ) -> Result<()> {
+ let mut buffer = [0u8; 8192];
+ for _ in 0..32 {
+ match input.read(&mut buffer) {
+ Ok(0) => break,
+ Ok(count) => {
+ for byte in &buffer[..count] {
+ if *byte == b'\n' {
+ self.flush(log, name, stream, true)?;
+ } else {
+ self.pending.push(*byte);
+ if self.pending.len() == LINE_LIMIT {
+ let tail_length = incomplete_utf8_tail(&self.pending);
+ let tail = self.pending.split_off(self.pending.len() - tail_length);
+ self.flush(log, name, stream, false)?;
+ self.pending = tail;
+ }
+ }
+ }
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break,
+ Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
+ Err(error) => return Err(error),
+ }
+ }
+ Ok(())
+ }
+
+ fn flush(&mut self, log: &mut File, name: &str, stream: &str, trim_cr: bool) -> Result<()> {
+ let bytes = if trim_cr {
+ self.pending.strip_suffix(b"\r").unwrap_or(&self.pending)
+ } else {
+ &self.pending
+ };
+ let message = String::from_utf8_lossy(bytes).replace('\t', " ");
+ let record = format!("{}\t{name}\t{stream}\t{message}\n", os::timestamp()?);
+ // One append per complete record; concurrent guards share the same
+ // flock so records larger than PIPE_BUF never interleave.
+ // SAFETY: flock borrows this live file descriptor until LOCK_UN.
+ if unsafe { libc::flock(log.as_raw_fd(), libc::LOCK_EX) } != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ let result = log.write_all(record.as_bytes());
+ // SAFETY: unlock the same live descriptor after the write completes.
+ let unlocked = unsafe { libc::flock(log.as_raw_fd(), libc::LOCK_UN) };
+ result?;
+ if unlocked != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ self.pending.clear();
+ Ok(())
+ }
+}
+
+fn incomplete_utf8_tail(bytes: &[u8]) -> usize {
+ for index in (bytes.len().saturating_sub(4)..bytes.len()).rev() {
+ let byte = bytes[index];
+ if byte & 0xc0 == 0x80 {
+ continue;
+ }
+ let expected = match byte {
+ 0xc2..=0xdf => 2,
+ 0xe0..=0xef => 3,
+ 0xf0..=0xf4 => 4,
+ _ => 1,
+ };
+ let available = bytes.len() - index;
+ return if expected > available { available } else { 0 };
+ }
+ 0
+}
+
+#[cfg(test)]
+mod tests {
+ use super::incomplete_utf8_tail;
+
+ #[test]
+ fn chunk_boundary_retains_only_incomplete_codepoints() {
+ for text in ["a", "é", "€", "🦀"] {
+ let bytes = text.as_bytes();
+ assert_eq!(incomplete_utf8_tail(bytes), 0);
+ for length in 1..bytes.len() {
+ assert_eq!(incomplete_utf8_tail(&bytes[..length]), length);
+ }
+ }
+ assert_eq!(incomplete_utf8_tail(b"abc\xff"), 0);
+ }
+}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
new file mode 100644
index 000000000..bb2683c59
--- /dev/null
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -0,0 +1,1084 @@
+//! Real-process qualification through the framed public protocol. No shell,
+//! network, daemon, user HOME or registry is involved.
+use serde_json::{Value, json};
+use std::{
+ cell::RefCell,
+ fs,
+ io::{Read, Write},
+ os::{fd::AsRawFd, unix::fs::MetadataExt},
+ path::{Path, PathBuf},
+ process::{Child, Command, Stdio},
+ sync::atomic::{AtomicU64, Ordering},
+ thread,
+ time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
+
+static SEQUENCE: AtomicU64 = AtomicU64::new(0);
+const TOKEN: &str = "synthetic-owner-token";
+const TIMEOUT: Duration = Duration::from_secs(15);
+
+struct Fixture {
+ root: PathBuf,
+ owned: RefCell>,
+}
+
+impl Fixture {
+ fn new() -> Self {
+ let parent = std::env::var_os("TMPDIR")
+ .map(PathBuf::from)
+ .unwrap_or_else(std::env::temp_dir);
+ let nonce = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ let root = parent.join(format!(
+ "supervisor-test-{}-{nonce}-{}",
+ std::process::id(),
+ SEQUENCE.fetch_add(1, Ordering::Relaxed)
+ ));
+ fs::DirBuilder::new().mode(0o700).create(&root).unwrap();
+ Self {
+ root,
+ owned: RefCell::new(Vec::new()),
+ }
+ }
+
+ fn process(&self, name: &str, mode: &str) -> Value {
+ json!({"name":name,"argv":[std::env::current_exe().unwrap(),"--exact","fixture_process","--nocapture"],"cwd":self.root,"env":{"HOME":self.root,"PATH":"/usr/bin:/bin","HACK_DAEMON_AUTOSTART_DISABLE":"1","SUPERVISOR_FIXTURE":mode}})
+ }
+
+ fn launch(&self, processes: Vec) -> Value {
+ json!({"version":1,"root":self.root,"ownerToken":TOKEN,"processes":processes})
+ }
+
+ fn control(&self, token: &str) -> Value {
+ json!({"version":1,"root":self.root,"ownerToken":token})
+ }
+
+ fn request(&self, action: &str, request: &Value) -> (Child, Value) {
+ let mut child = command(&self.root, action).spawn().unwrap();
+ send(child.stdin.as_mut().unwrap(), request);
+ // Deliberately retain stdin: frame receipt must not depend on EOF.
+ let reply = receive(child.stdout.as_mut().unwrap());
+ self.remember(&reply);
+ (child, reply)
+ }
+
+ fn remember(&self, reply: &Value) {
+ for state in reply["processes"].as_array().into_iter().flatten() {
+ for field in ["guardPid", "commandPid"] {
+ if let Some(id) = state[field].as_i64().and_then(|id| identity(id as i32)) {
+ self.owned.borrow_mut().push(id);
+ }
+ }
+ }
+ }
+
+ fn start(&self, processes: Vec) -> (Child, Value) {
+ self.request("run", &self.launch(processes))
+ }
+
+ fn stop(&self) {
+ let mut child = command(&self.root, "stop").spawn().unwrap();
+ send(child.stdin.as_mut().unwrap(), &self.control(TOKEN));
+ drop(child.stdin.take());
+ assert!(wait(&mut child, TIMEOUT).success(), "owned stop failed");
+ }
+
+ fn state(&self, name: &str) -> Value {
+ eventually(|| {
+ fs::read(self.root.join(format!("{name}.json")))
+ .ok()
+ .and_then(|b| serde_json::from_slice(&b).ok())
+ })
+ }
+
+ fn log(&self) -> String {
+ fs::read_to_string(self.root.join("lifecycle.log")).unwrap_or_default()
+ }
+
+ fn await_log(&self, needle: &str) {
+ eventually(|| self.log().contains(needle).then_some(()));
+ }
+}
+
+impl Drop for Fixture {
+ fn drop(&mut self) {
+ // Ask the normal owner first, including after an assertion panic. Stop
+ // has a bounded wait and all inputs refer only to this fixture root.
+ if let Ok(mut stop) = command(&self.root, "stop").spawn() {
+ if let Some(mut stdin) = stop.stdin.take() {
+ let bytes = serde_json::to_vec(&self.control(TOKEN)).unwrap_or_default();
+ let _ = stdin.write_all(&(bytes.len() as u32).to_be_bytes());
+ let _ = stdin.write_all(&bytes);
+ }
+ let deadline = Instant::now() + Duration::from_secs(8);
+ while stop.try_wait().ok().flatten().is_none() && Instant::now() < deadline {
+ thread::sleep(Duration::from_millis(20));
+ }
+ let _ = stop.kill();
+ let _ = stop.wait();
+ }
+ // Receipt copies are captured before tests deliberately corrupt state.
+ // A still-matching guard or command birth proves the original session
+ // still exists. Recheck each enumerated PID's birth before its signal.
+ for anchor in self.owned.borrow().iter() {
+ if identity(anchor.pid).as_ref() != Some(anchor) {
+ continue;
+ }
+ let mut members: Vec<_> = census()
+ .into_iter()
+ .filter(|p| p.session == anchor.session)
+ .collect();
+ members.sort_by_key(|p| p.pid == anchor.session);
+ for member in members {
+ if identity(member.pid).as_ref() == Some(&member) {
+ // SAFETY: matched kernel birth and session from a live
+ // captured launch identity; no user-owned state is used.
+ unsafe {
+ libc::kill(member.pid, libc::SIGKILL);
+ }
+ }
+ }
+ }
+ let _ = fs::remove_dir_all(&self.root);
+ }
+}
+
+#[derive(Clone, PartialEq, Eq)]
+struct Identity {
+ pid: i32,
+ session: i32,
+ birth: u64,
+}
+
+#[cfg(target_os = "linux")]
+fn identity(pid: i32) -> Option {
+ let text = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
+ let fields: Vec<_> = text.rsplit_once(") ")?.1.split_whitespace().collect();
+ Some(Identity {
+ pid,
+ session: fields.get(3)?.parse().ok()?,
+ birth: fields.get(19)?.parse().ok()?,
+ })
+}
+
+#[cfg(target_os = "macos")]
+fn identity(pid: i32) -> Option {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: kernel writes at most the supplied live buffer size.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ if size as usize != std::mem::size_of::() {
+ return None;
+ }
+ let info = unsafe { info.assume_init() };
+ let session = unsafe { libc::getsid(pid) };
+ (session > 1).then_some(Identity {
+ pid,
+ session,
+ birth: info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec,
+ })
+}
+
+#[cfg(target_os = "linux")]
+fn census() -> Vec {
+ fs::read_dir("/proc")
+ .into_iter()
+ .flatten()
+ .flatten()
+ .filter_map(|e| e.file_name().to_str()?.parse().ok())
+ .filter_map(identity)
+ .collect()
+}
+
+#[cfg(target_os = "macos")]
+fn census() -> Vec {
+ let mut pids = vec![0i32; 65536];
+ // SAFETY: initialized vector provides the supplied writable byte count.
+ let count =
+ unsafe { libc::proc_listpids(1, 0, pids.as_mut_ptr().cast(), (pids.len() * 4) as i32) };
+ if count <= 0 {
+ return Vec::new();
+ }
+ pids.into_iter()
+ .take(count as usize / 4)
+ .filter(|p| *p > 1)
+ .filter_map(identity)
+ .collect()
+}
+
+fn command(root: &Path, action: &str) -> Command {
+ let mut command = Command::new(env!("CARGO_BIN_EXE_hack-lifecycle-supervisor"));
+ command
+ .arg(action)
+ .env_clear()
+ .env("HOME", root)
+ .env("PATH", "/usr/bin:/bin")
+ .env("HACK_DAEMON_AUTOSTART_DISABLE", "1")
+ .env("SUPERVISOR_PARENT_ONLY", "synthetic-parent-only")
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null());
+ command
+}
+
+fn send(writer: &mut impl Write, value: &Value) {
+ let bytes = serde_json::to_vec(value).unwrap();
+ writer
+ .write_all(&(bytes.len() as u32).to_be_bytes())
+ .unwrap();
+ writer.write_all(&bytes).unwrap();
+ writer.flush().unwrap();
+}
+
+fn exact(reader: &mut (impl Read + AsRawFd), bytes: &mut [u8]) {
+ let end = Instant::now() + TIMEOUT;
+ let mut offset = 0;
+ while offset < bytes.len() {
+ assert!(Instant::now() < end, "framed response timed out");
+ let mut poll = libc::pollfd {
+ fd: reader.as_raw_fd(),
+ events: libc::POLLIN,
+ revents: 0,
+ };
+ // SAFETY: one initialized pollfd is writable and alive for this call.
+ let n = unsafe { libc::poll(&mut poll, 1, 100) };
+ if n == 0 {
+ continue;
+ }
+ assert!(n > 0, "poll failed");
+ let n = reader.read(&mut bytes[offset..]).unwrap();
+ assert!(n > 0, "response closed before a complete frame");
+ offset += n;
+ }
+}
+
+fn receive(reader: &mut (impl Read + AsRawFd)) -> Value {
+ let mut header = [0; 4];
+ exact(reader, &mut header);
+ let length = u32::from_be_bytes(header) as usize;
+ assert!((1..=1024 * 1024).contains(&length));
+ let mut bytes = vec![0; length];
+ exact(reader, &mut bytes);
+ serde_json::from_slice(&bytes).unwrap()
+}
+
+fn wait(child: &mut Child, timeout: Duration) -> std::process::ExitStatus {
+ let end = Instant::now() + timeout;
+ loop {
+ if let Some(status) = child.try_wait().unwrap() {
+ return status;
+ }
+ if Instant::now() >= end {
+ let _ = child.kill();
+ let _ = child.wait();
+ panic!("owned subprocess exceeded deadline");
+ }
+ thread::sleep(Duration::from_millis(20));
+ }
+}
+
+fn eventually(mut predicate: impl FnMut() -> Option) -> T {
+ let end = Instant::now() + TIMEOUT;
+ loop {
+ if let Some(value) = predicate() {
+ return value;
+ }
+ assert!(
+ Instant::now() < end,
+ "expected fixture observation did not arrive"
+ );
+ thread::sleep(Duration::from_millis(20));
+ }
+}
+
+fn pid(state: &Value, field: &str) -> i32 {
+ state[field].as_i64().unwrap() as i32
+}
+
+struct ForeignChild(Child);
+impl ForeignChild {
+ fn spawn() -> Self {
+ use std::os::unix::process::CommandExt;
+ let mut command = Command::new(std::env::current_exe().unwrap());
+ command
+ .args(["--exact", "fixture_process", "--nocapture"])
+ .env_clear()
+ .env("SUPERVISOR_FIXTURE", "hold")
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null());
+ // SAFETY: changes only the new child, with no allocated state.
+ unsafe {
+ command.pre_exec(|| {
+ if libc::setsid() < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
+ }
+ Self(command.spawn().unwrap())
+ }
+}
+impl Drop for ForeignChild {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+}
+
+fn refused(fixture: &Fixture, action: &str, request: Value) {
+ let mut child = command(&fixture.root, action).spawn().unwrap();
+ send(child.stdin.as_mut().unwrap(), &request);
+ drop(child.stdin.take());
+ assert!(
+ !wait(&mut child, TIMEOUT).success(),
+ "unsafe operation was accepted"
+ );
+}
+
+#[test]
+fn starts_and_records_identity() {
+ let f = Fixture::new();
+ let (mut caller, receipt) = f.start(vec![f.process("worker", "hold")]);
+ assert_eq!(receipt["version"], 1);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ assert_eq!(state["ownerToken"], TOKEN);
+ assert_eq!(state["guardPid"], state["processGroupId"]);
+ assert!(state["guardStartTime"].as_u64().unwrap() > 0);
+ assert!(state["bootId"].as_str().is_some_and(|v| !v.is_empty()));
+ assert!(pid(&state, "commandPid") > 1);
+ assert_eq!(
+ fs::metadata(f.root.join("worker.json")).unwrap().mode() & 0o777,
+ 0o600
+ );
+ assert_eq!(
+ fs::metadata(f.root.join("lifecycle.log")).unwrap().mode() & 0o777,
+ 0o600
+ );
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn stop_after_caller_closed() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ drop(caller.stdin.take());
+ drop(caller.stdout.take());
+ f.await_log("fixture-ready");
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn supervisor_kill_then_recover_stop() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ caller.kill().unwrap();
+ caller.wait().unwrap();
+ refused(&f, "run", f.launch(vec![f.process("worker", "hold")]));
+ f.stop();
+}
+
+#[test]
+fn term_ignoring_command_is_killed_after_grace() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]);
+ f.await_log("fixture-ready");
+ let start = Instant::now();
+ f.stop();
+ assert!(
+ start.elapsed() >= Duration::from_secs(5),
+ "TERM grace was shortened"
+ );
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn foreign_owner_cannot_stop() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ refused(&f, "stop", f.control("different-owner"));
+ let state = f.state("worker");
+ // SAFETY: signal zero observes this fixture's process without delivering a signal.
+ assert_eq!(unsafe { libc::kill(pid(&state, "commandPid"), 0) }, 0);
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn failed_start_runs_no_later_command() {
+ let f = Fixture::new();
+ let broken =
+ json!({"name":"broken","argv":["/nonexistent/synthetic-program"],"cwd":f.root,"env":{}});
+ refused(
+ &f,
+ "run",
+ f.launch(vec![broken, f.process("later", "hold")]),
+ );
+ assert!(
+ !f.root.join("later.json").exists(),
+ "later command started after failure"
+ );
+ assert!(!f.log().contains("fixture-ready"));
+}
+
+#[test]
+fn output_bursts_and_long_lines_include_partial_tail() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "burst")]);
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let text = f.log();
+ assert!(text.contains("\tstderr\tfixture-error"));
+ assert!(text.contains("partial-tail"));
+ assert_eq!(text.matches("burst-record-").count(), 2000);
+ assert_eq!(text.bytes().filter(|v| *v == b'X').count(), 300_000);
+ for line in text.lines() {
+ let fields: Vec<_> = line.splitn(4, '\t').collect();
+ assert_eq!(fields.len(), 4);
+ assert_eq!(fields[1], "worker");
+ assert!(fields[0].ends_with('Z'));
+ }
+}
+
+#[test]
+fn zombies_do_not_hold_liveness() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "zombie")]);
+ assert!(
+ wait(&mut caller, TIMEOUT).success(),
+ "zombie-only session remained alive"
+ );
+}
+
+#[test]
+fn sentinel_environment_never_enters_argv_or_state() {
+ let f = Fixture::new();
+ let mut process = f.process("worker", "sentinel");
+ let sentinel = format!(
+ "synthetic-credential-{}-{}",
+ std::process::id(),
+ SEQUENCE.fetch_add(1, Ordering::Relaxed)
+ );
+ process["env"]["SYNTHETIC_PRIVATE_VALUE"] = json!(sentinel);
+ let (mut caller, _) = f.start(vec![process]);
+ f.await_log("sentinel-received");
+ for entry in fs::read_dir(&f.root).unwrap().flatten() {
+ if entry.file_type().unwrap().is_file() {
+ let bytes = fs::read(entry.path()).unwrap();
+ assert!(
+ !bytes
+ .windows(sentinel.len())
+ .any(|v| v == sentinel.as_bytes()),
+ "private value persisted"
+ );
+ }
+ }
+ let state = f.state("worker");
+ #[cfg(target_os = "linux")]
+ for process_id in [
+ caller.id() as i32,
+ pid(&state, "guardPid"),
+ pid(&state, "commandPid"),
+ ] {
+ let bytes = fs::read(format!("/proc/{process_id}/cmdline")).unwrap();
+ assert!(
+ !bytes
+ .windows(sentinel.len())
+ .any(|v| v == sentinel.as_bytes()),
+ "private value in argv"
+ );
+ }
+ #[cfg(target_os = "linux")]
+ {
+ let bytes = fs::read(format!("/proc/{}/environ", pid(&state, "guardPid"))).unwrap();
+ assert!(bytes.is_empty(), "guard inherited caller environment");
+ }
+ #[cfg(target_os = "macos")]
+ {
+ let output = Command::new("/bin/ps")
+ .args([
+ "-o",
+ "command=",
+ "-p",
+ &format!(
+ "{},{},{}",
+ caller.id(),
+ pid(&state, "guardPid"),
+ pid(&state, "commandPid")
+ ),
+ ])
+ .output()
+ .unwrap();
+ assert!(
+ !output
+ .stdout
+ .windows(sentinel.len())
+ .any(|v| v == sentinel.as_bytes()),
+ "private value in argv"
+ );
+ }
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn child_descriptor_allowlist() {
+ use std::os::fd::{FromRawFd, OwnedFd};
+ use std::os::unix::fs::OpenOptionsExt;
+ let f = Fixture::new();
+ let file = fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(0o600)
+ .open(f.root.join("unexpected-inheritance"))
+ .unwrap();
+ // Deliberately emulate a caller's inheritable descriptor. The launcher's
+ // startup descriptor sweep and each exec boundary must exclude it.
+ let raw = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_DUPFD, 200) };
+ assert!(raw >= 200);
+ let _inheritable = unsafe { OwnedFd::from_raw_fd(raw) };
+ let (mut caller, _) = f.start(vec![f.process("worker", "descriptors")]);
+ assert!(wait(&mut caller, TIMEOUT).success());
+ assert!(f.log().contains("descriptor-allowlist-ok"));
+ assert!(!f.log().contains("descriptor-leak"));
+}
+
+#[test]
+fn concurrent_stop_requests_settle() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let mut first = command(&f.root, "stop").spawn().unwrap();
+ let mut second = command(&f.root, "stop").spawn().unwrap();
+ send(first.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ send(second.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ drop(first.stdin.take());
+ drop(second.stdin.take());
+ let first_ok = wait(&mut first, TIMEOUT).success();
+ let second_ok = wait(&mut second, TIMEOUT).success();
+ assert!(first_ok || second_ok, "neither stop request settled");
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn changed_process_group_remains_in_owned_session() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "change-group")]);
+ f.await_log("changed-group-ready");
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn inherited_ignored_signal_is_reset() {
+ use std::os::unix::process::CommandExt;
+ let f = Fixture::new();
+ let mut command = command(&f.root, "run");
+ // SAFETY: signal changes only the freshly forked test child; it is an
+ // async-signal-safe operation and captures no allocated state.
+ unsafe {
+ command.pre_exec(|| {
+ libc::signal(libc::SIGTERM, libc::SIG_IGN);
+ Ok(())
+ });
+ }
+ let mut caller = command.spawn().unwrap();
+ send(
+ caller.stdin.as_mut().unwrap(),
+ &f.launch(vec![f.process("worker", "signal-default")]),
+ );
+ let receipt = receive(caller.stdout.as_mut().unwrap());
+ f.remember(&receipt);
+ f.await_log("signal-default-ok");
+ let start = Instant::now();
+ f.stop();
+ assert!(
+ start.elapsed() < Duration::from_secs(4),
+ "command retained ignored TERM disposition"
+ );
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn supervisor_killed_during_output_burst_is_recoverable() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "continuous-output")]);
+ f.await_log("continuous-output");
+ caller.kill().unwrap();
+ caller.wait().unwrap();
+ f.stop();
+}
+
+#[test]
+fn dead_guard_with_live_session_refuses_duplicate_and_stop() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ // SAFETY: these PIDs came from this fixture's ready receipt only.
+ unsafe {
+ libc::kill(pid(&state, "guardPid"), libc::SIGKILL);
+ }
+ assert!(wait(&mut caller, TIMEOUT).success());
+ refused(&f, "run", f.launch(vec![f.process("worker", "hold")]));
+ let (mut stop, _) = f.request("stop", &f.control(TOKEN));
+ assert!(!wait(&mut stop, TIMEOUT).success());
+ // The guard cannot prove ownership after its own death. The test itself
+ // retains direct launch provenance and must clean up its authored child.
+ unsafe {
+ libc::kill(pid(&state, "commandPid"), libc::SIGKILL);
+ }
+}
+
+#[test]
+fn stale_receipt_cannot_signal_another_process() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let mut state = f.state("worker");
+ let original = state.clone();
+ // A valid-looking receipt with an incorrect birth time must never become
+ // authority just because its PID currently exists.
+ state["guardStartTime"] = json!(state["guardStartTime"].as_u64().unwrap() + 1);
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&state).unwrap(),
+ )
+ .unwrap();
+ let (mut stop, _) = f.request("stop", &f.control(TOKEN));
+ wait(&mut stop, TIMEOUT);
+ assert_eq!(unsafe { libc::kill(pid(&original, "commandPid"), 0) }, 0);
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&original).unwrap(),
+ )
+ .unwrap();
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn receipt_naming_a_foreign_pid_cannot_stop_it() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let mut foreign = ForeignChild::spawn();
+ let original = f.state("worker");
+ let mut state = original.clone();
+ state["guardPid"] = json!(foreign.0.id());
+ state["processGroupId"] = json!(foreign.0.id());
+ // Linux births have clock-tick precision: two distinct live PIDs can have
+ // equal births. Make this stale identity differ deterministically.
+ let foreign_identity = eventually(|| identity(foreign.0.id() as i32));
+ state["guardStartTime"] = json!(foreign_identity.birth + 1);
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&state).unwrap(),
+ )
+ .unwrap();
+ let (mut stop, _) = f.request("stop", &f.control(TOKEN));
+ wait(&mut stop, TIMEOUT);
+ let foreign_running = foreign.0.try_wait().unwrap().is_none();
+ drop(foreign);
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&original).unwrap(),
+ )
+ .unwrap();
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+ assert!(foreign_running, "stale receipt signalled another process");
+}
+
+#[test]
+fn stale_reused_session_leader_does_not_block_fresh_launch() {
+ let f = Fixture::new();
+ let (mut first, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ f.stop();
+ assert!(wait(&mut first, TIMEOUT).success());
+ let mut foreign = ForeignChild::spawn();
+ let foreign_identity = eventually(|| identity(foreign.0.id() as i32));
+ let mut state = f.state("worker");
+ state["guardPid"] = json!(foreign.0.id());
+ state["processGroupId"] = json!(foreign.0.id());
+ state["guardStartTime"] = json!(foreign_identity.birth + 1);
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&state).unwrap(),
+ )
+ .unwrap();
+ let (mut next, _) = f.start(vec![f.process("worker", "hold")]);
+ f.stop();
+ assert!(wait(&mut next, TIMEOUT).success());
+ assert!(
+ foreign.0.try_wait().unwrap().is_none(),
+ "foreign reused session was signalled"
+ );
+}
+
+#[test]
+fn incomplete_frame_has_one_whole_handoff_deadline() {
+ let f = Fixture::new();
+ let mut caller = command(&f.root, "run").spawn().unwrap();
+ let start = Instant::now();
+ caller.stdin.as_mut().unwrap().write_all(&[0]).unwrap();
+ thread::sleep(Duration::from_secs(6));
+ // Complete the header late, leaving the announced body absent. A reset
+ // deadline per read would let this request survive sixteen seconds.
+ caller
+ .stdin
+ .as_mut()
+ .unwrap()
+ .write_all(&[0, 0, 16])
+ .unwrap();
+ assert!(!wait(&mut caller, Duration::from_secs(6)).success());
+ assert!(start.elapsed() < Duration::from_secs(12));
+ assert!(!f.root.join("lifecycle.log").exists());
+}
+
+#[test]
+fn failure_teardown_stops_separate_command_group() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ let command_pid = pid(&state, "commandPid");
+ let guard_pid = pid(&state, "guardPid");
+ assert_ne!(command_pid, guard_pid);
+ drop(f);
+ assert!(wait(&mut caller, TIMEOUT).success());
+ assert!(identity(command_pid).is_none(), "fixture command leaked");
+ assert!(identity(guard_pid).is_none(), "fixture guard leaked");
+}
+
+#[test]
+fn unresponsive_guard_stop_remains_unknown() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ let guard = pid(&state, "guardPid");
+ // SAFETY: the guard belongs to this fixture and its receipt is immutable.
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert_eq!(receipt["outcome"], "unknown");
+ assert!(!wait(&mut stop, TIMEOUT).success());
+ assert_eq!(unsafe { libc::kill(pid(&state, "commandPid"), 0) }, 0);
+ refused(&f, "run", f.launch(vec![f.process("worker", "hold")]));
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGCONT) }, 0);
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn new_generation_ignores_retained_terminal_receipts() {
+ let f = Fixture::new();
+ let (mut first, _) = f.start(vec![f.process("old-name", "hold")]);
+ f.await_log("fixture-ready");
+ f.stop();
+ assert!(wait(&mut first, TIMEOUT).success());
+ let mut launch = f.launch(vec![f.process("renamed", "hold")]);
+ launch["ownerToken"] = json!("synthetic-next-generation");
+ let (mut next, _) = f.request("run", &launch);
+ let (mut stop, receipt) = f.request("stop", &f.control("synthetic-next-generation"));
+ assert!(wait(&mut stop, TIMEOUT).success());
+ assert!(wait(&mut next, TIMEOUT).success());
+ assert_eq!(receipt["processes"].as_array().unwrap().len(), 1);
+ assert!(
+ f.root.join("old-name.json").exists(),
+ "terminal receipt was discarded"
+ );
+}
+
+#[test]
+fn command_joining_guard_group_still_receives_kill() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "join-guard-group")]);
+ f.await_log("joined-guard-group");
+ let start = Instant::now();
+ f.stop();
+ assert!(start.elapsed() >= Duration::from_secs(5));
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn process_group_leader_caller_refuses_without_launch() {
+ use std::os::unix::process::CommandExt;
+ let f = Fixture::new();
+ let mut command = command(&f.root, "run");
+ // SAFETY: changes only the newly forked child, using no allocated state.
+ unsafe {
+ command.pre_exec(|| {
+ if libc::setpgid(0, 0) != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
+ }
+ let mut caller = command.spawn().unwrap();
+ send(
+ caller.stdin.as_mut().unwrap(),
+ &f.launch(vec![f.process("worker", "hold")]),
+ );
+ drop(caller.stdin.take());
+ assert!(!wait(&mut caller, TIMEOUT).success());
+ assert!(!f.root.join("worker.json").exists());
+ assert!(!f.root.join("lifecycle.log").exists());
+}
+
+#[test]
+fn stop_serializes_with_multiple_process_launches() {
+ let f = Fixture::new();
+ let processes: Vec<_> = (0..16)
+ .map(|i| f.process(&format!("worker-{i}"), "hold"))
+ .collect();
+ let mut caller = command(&f.root, "run").spawn().unwrap();
+ send(caller.stdin.as_mut().unwrap(), &f.launch(processes));
+ f.state("worker-0");
+ let mut stop = command(&f.root, "stop").spawn().unwrap();
+ send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ let ready = receive(caller.stdout.as_mut().unwrap());
+ f.remember(&ready);
+ let receipt = receive(stop.stdout.as_mut().unwrap());
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(wait(&mut stop, TIMEOUT).success());
+ assert!(wait(&mut caller, TIMEOUT).success());
+ for state in ready["processes"].as_array().unwrap() {
+ assert!(
+ identity(pid(state, "commandPid")).is_none(),
+ "a launch escaped concurrent stop"
+ );
+ }
+}
+
+#[test]
+fn unicode_codepoint_crossing_chunk_boundary_is_preserved() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "unicode")]);
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let log = f.log();
+ assert_eq!(log.matches('🦀').count(), 1);
+ assert!(
+ !log.contains('\u{fffd}'),
+ "UTF-8 chunk boundary corrupted a codepoint"
+ );
+}
+
+#[test]
+fn case_insensitive_process_name_collision_refuses_all_launches() {
+ let f = Fixture::new();
+ refused(
+ &f,
+ "run",
+ f.launch(vec![
+ f.process("Worker", "hold"),
+ f.process("worker", "hold"),
+ ]),
+ );
+ assert!(!f.root.join("Worker.json").exists());
+ assert!(!f.root.join("worker.json").exists());
+ assert!(!f.root.join("lifecycle.log").exists());
+}
+
+#[test]
+fn status_observes_without_creating_files() {
+ let f = Fixture::new();
+ let (mut status, receipt) = f.request("status", &f.control(TOKEN));
+ assert!(wait(&mut status, TIMEOUT).success());
+ assert_eq!(receipt["outcome"], "observed");
+ assert!(receipt["processes"].as_array().unwrap().is_empty());
+ assert_eq!(fs::read_dir(&f.root).unwrap().count(), 0);
+}
+
+#[cfg(target_os = "linux")]
+#[test]
+fn adopted_orphan_is_reaped_while_direct_command_lives() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "orphan-exits")]);
+ f.await_log("orphan-ready");
+ let orphan = eventually(|| {
+ f.log().lines().find_map(|line| {
+ line.split("orphan-ready ")
+ .nth(1)
+ .and_then(|value| value.parse::().ok())
+ })
+ });
+ eventually(|| identity(orphan).is_none().then_some(()));
+ let state = f.state("worker");
+ assert!(identity(pid(&state, "commandPid")).is_some());
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn fixture_process() {
+ let Ok(mode) = std::env::var("SUPERVISOR_FIXTURE") else {
+ return;
+ };
+ assert!(
+ std::env::var_os("SUPERVISOR_PARENT_ONLY").is_none(),
+ "command inherited caller environment"
+ );
+ match mode.as_str() {
+ "burst" => {
+ let mut out = std::io::stdout().lock();
+ for i in 0..2000 {
+ writeln!(out, "burst-record-{i}").unwrap();
+ }
+ out.write_all(&vec![b'X'; 300_000]).unwrap();
+ out.write_all(b"\npartial-tail").unwrap();
+ out.flush().unwrap();
+ eprintln!("fixture-error");
+ std::process::exit(0);
+ }
+ "descriptors" => {
+ assert_eq!(unsafe { libc::isatty(0) }, 0);
+ let mut byte = [0u8];
+ assert_eq!(
+ std::io::stdin().read(&mut byte).unwrap(),
+ 0,
+ "command inherited private protocol input"
+ );
+ let mut leaked = false;
+ for fd in 3..1024 {
+ // SAFETY: F_GETFD observes descriptor state without mutation.
+ if unsafe { libc::fcntl(fd, libc::F_GETFD) } >= 0 {
+ leaked = true;
+ }
+ }
+ println!(
+ "{}",
+ if leaked {
+ "descriptor-leak"
+ } else {
+ "descriptor-allowlist-ok"
+ }
+ );
+ std::process::exit(if leaked { 1 } else { 0 });
+ }
+ "zombie" => {
+ // SAFETY: fixture immediately exits in the fork child; it invokes
+ // no allocator or Rust destructors after fork in that branch.
+ let child = unsafe { libc::fork() };
+ assert!(child >= 0);
+ if child == 0 {
+ unsafe {
+ libc::_exit(0);
+ }
+ }
+ thread::sleep(Duration::from_millis(100));
+ std::process::exit(0);
+ }
+ #[cfg(target_os = "linux")]
+ "orphan-exits" => {
+ let (mut input, output) = std::os::unix::net::UnixStream::pair().unwrap();
+ // SAFETY: both fork descendants use only async-signal-safe libc
+ // calls and immediately exit; no Rust allocator state is reused.
+ let intermediary = unsafe { libc::fork() };
+ assert!(intermediary >= 0);
+ if intermediary == 0 {
+ unsafe {
+ let orphan = libc::fork();
+ if orphan == 0 {
+ libc::usleep(300_000);
+ libc::_exit(0);
+ }
+ let bytes = orphan.to_be_bytes();
+ libc::write(output.as_raw_fd(), bytes.as_ptr().cast(), bytes.len());
+ libc::_exit(if orphan < 0 { 98 } else { 0 });
+ }
+ }
+ let mut status = 0;
+ assert_eq!(
+ unsafe { libc::waitpid(intermediary, &mut status, 0) },
+ intermediary
+ );
+ let mut bytes = [0; 4];
+ input.read_exact(&mut bytes).unwrap();
+ let orphan = i32::from_be_bytes(bytes);
+ assert!(orphan > 1);
+ println!("orphan-ready {orphan}");
+ }
+ "change-group" => {
+ // SAFETY: fork child uses only async-signal-safe calls and never
+ // returns to Rust, so the libtest thread state is not reused.
+ let child = unsafe { libc::fork() };
+ assert!(child >= 0);
+ if child == 0 {
+ unsafe {
+ if libc::setpgid(0, 0) != 0 {
+ libc::_exit(98);
+ }
+ libc::write(1, c"changed-group-ready\n".as_ptr().cast(), 20);
+ loop {
+ libc::pause();
+ }
+ }
+ }
+ }
+ "join-guard-group" => {
+ // SAFETY: fixture joins its existing session leader's group; only
+ // this authored test process's group/disposition are changed.
+ unsafe {
+ let session = libc::getsid(0);
+ assert!(session > 1);
+ assert_eq!(libc::setpgid(0, session), 0);
+ libc::signal(libc::SIGTERM, libc::SIG_IGN);
+ }
+ println!("joined-guard-group");
+ }
+ "unicode" => {
+ let mut out = std::io::stdout().lock();
+ out.write_all(&vec![b'a'; 65535]).unwrap();
+ out.write_all("🦀\n".as_bytes()).unwrap();
+ out.flush().unwrap();
+ std::process::exit(0);
+ }
+ "signal-default" => {
+ let mut disposition: libc::sigaction = unsafe { std::mem::zeroed() };
+ assert_eq!(
+ unsafe { libc::sigaction(libc::SIGTERM, std::ptr::null(), &mut disposition) },
+ 0
+ );
+ assert_eq!(disposition.sa_sigaction, libc::SIG_DFL);
+ println!("signal-default-ok");
+ }
+ "continuous-output" => loop {
+ println!("continuous-output {}", "x".repeat(4096));
+ thread::sleep(Duration::from_millis(1));
+ },
+ "sentinel" => {
+ assert!(std::env::var_os("SYNTHETIC_PRIVATE_VALUE").is_some());
+ println!("sentinel-received");
+ }
+ "ignore-term" => {
+ // SAFETY: SIG_IGN is a valid process-local disposition.
+ unsafe {
+ libc::signal(libc::SIGTERM, libc::SIG_IGN);
+ }
+ }
+ "hold" => (),
+ _ => std::process::exit(99),
+ }
+ println!("fixture-ready");
+ std::io::stdout().flush().unwrap();
+ loop {
+ thread::sleep(Duration::from_secs(1));
+ }
+}
+
+use std::os::unix::fs::DirBuilderExt;
diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts
index 61f47515d..aed6fa011 100644
--- a/scripts/lib/tla-runtime-models.ts
+++ b/scripts/lib/tla-runtime-models.ts
@@ -141,6 +141,14 @@ const restartPolicyStaleUses = [
{ engine: "exited", bound: 3 },
] as const;
const contracts: readonly ModelContract[] = [
+ {
+ name: "lifecycle-supervisor",
+ module: "LifecycleSupervisor",
+ states: 129,
+ invariant: "NoDuplicate",
+ action: "Retry",
+ fields: ["duplicate = TRUE", "live = TRUE", "durable = TRUE"],
+ },
{
name: "shutdown-rebase",
module: "ShutdownRebase",
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
new file mode 100644
index 000000000..5416ba319
--- /dev/null
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -0,0 +1,38 @@
+# Lifecycle supervisor ownership model
+
+`LifecycleSupervisor.tla` models one configured command name and at most two
+start attempts. The model precedes implementation. The first artifact is a
+standalone Rust binary exercised by a fake caller; the CLI remains unchanged.
+
+| Action/property | Source transition | Regression boundary |
+| --- | --- | --- |
+| `Reserve`, `Publish`, `Spawn`; `DurableBeforeLaunch` | Per-command flock, guard session creation, synced identity before argv spawn | `starts_and_records_identity`; refused publication/spawn cannot run a later command |
+| `SupervisorDeath`; `GuardRetainsLock` | Resident guard retains flock and stdout/stderr readers after project supervisor loss | `supervisor_kill_then_recover_stop`; output continues after supervisor SIGKILL |
+| `GuardDeath`, `Retry`; `NoDuplicate` | Retained session census refuses a retry even if separately killed guard released its flock | Guard-loss retry and identity refusal regression |
+| `Stop`, `Kill`; `NoForeignSignal` | Exact owner/boot/start identity before TERM and KILL; guard remains outside command groups | Wrong-owner/stale-identity refusal; TERM-ignore escalation |
+| `ProcessExit`, `Reap` | Session census ignores zombies; reap direct/adopted children; release lock after no live members | Zombie liveness; child changes process group within session |
+| `StopUnknown` | Bounded confirmation can report uncertainty while live members remain | Honest unknown recovery outcome, never successful absence |
+
+Positive TLC exploration: 129 distinct states, queue exhausted, depth 11. Negative
+control sets `SkipLiveFence = TRUE`; `Retry` then violates `NoDuplicate` with live,
+durable and duplicate all true in the same state. Both guard-loss and project-loss
+traces can witness this violation; the registry requires the named action and
+same-state fields without depending on worker scheduling.
+
+Checker: checksum-pinned TLA+ 1.7.4 artifact (TLC reports version 2.19), Java 17,
+two workers and a bounded heap. The maintained runner pins the positive count and
+validates the negative invariant/action/witness. Apalache is not qualified here.
+
+Assumptions: flock serialization, atomic/synchronized publication, reliable native
+identity and session census, and rechecking identity before effects. Signal delivery
+does not imply process exit: `KillPending` can retain live processes, and
+`StopUnknown` is distinct from `Exited`. No fairness or eventual exit is assumed.
+An identity change blocks signaling. A separately killed guard leaves an orphaned
+session; it never grants new launch authority simply because its lock was released.
+
+Omissions: actual filesystem durability, command contents, multi-command rollback,
+pipe buffers, descriptor creation races, scheduler latency, resource use, power loss,
+PID reuse between observation and signaling, hostile same-user mutation, and
+commands escaping their session with `setsid`. Those remain native implementation
+or explicitly unsupported boundaries. This finite model does not qualify a shipped
+artifact, macOS behavior, CLI integration or the default runner.
diff --git a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
new file mode 100644
index 000000000..0fc117f65
--- /dev/null
+++ b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
@@ -0,0 +1,85 @@
+------------------------ MODULE LifecycleSupervisor ------------------------
+EXTENDS Naturals
+CONSTANT SkipLiveFence
+VARIABLES phase, durable, guard, locked, live, supervisor, attempts,
+ identity, unsafeSignal, duplicate
+vars == <>
+
+Init == /\ phase = "Absent"
+ /\ durable = FALSE /\ guard = FALSE /\ locked = FALSE
+ /\ live = FALSE /\ supervisor = FALSE /\ attempts = 0
+ /\ identity = "match" /\ unsafeSignal = FALSE /\ duplicate = FALSE
+
+Reserve == /\ phase \in {"Absent", "Exited"} /\ ~live /\ attempts < 2
+ /\ phase' = "Starting" /\ guard' = TRUE /\ locked' = TRUE
+ /\ supervisor' = TRUE /\ durable' = FALSE
+ /\ attempts' = attempts + 1
+ /\ UNCHANGED <>
+Publish == /\ phase = "Starting" /\ ~durable
+ /\ durable' = TRUE
+ /\ UNCHANGED <>
+Spawn == /\ phase = "Starting" /\ durable /\ guard /\ locked
+ /\ phase' = "Running" /\ live' = TRUE
+ /\ UNCHANGED <>
+StartFailure == /\ phase = "Starting"
+ /\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE
+ /\ supervisor' = FALSE
+ /\ UNCHANGED <>
+SupervisorDeath == /\ supervisor /\ phase = "Running"
+ /\ supervisor' = FALSE /\ phase' = "Recoverable"
+ /\ UNCHANGED <>
+GuardDeath == /\ guard /\ live
+ /\ guard' = FALSE /\ locked' = FALSE
+ /\ phase' = "Orphaned" /\ supervisor' = FALSE
+ /\ UNCHANGED <>
+Retry == /\ ~supervisor /\ live /\ attempts < 2
+ /\ SkipLiveFence
+ /\ duplicate' = TRUE /\ attempts' = attempts + 1
+ /\ UNCHANGED <>
+ReplaceIdentity == /\ identity = "match" /\ durable
+ /\ identity' \in {"foreign", "unknown"}
+ /\ UNCHANGED <>
+Stop == /\ phase \in {"Running", "Recoverable"}
+ /\ guard /\ durable /\ identity = "match"
+ /\ phase' = "Stopping"
+ /\ unsafeSignal' = unsafeSignal \/ identity # "match"
+ /\ UNCHANGED <>
+Kill == /\ phase = "Stopping" /\ guard /\ identity = "match"
+ /\ phase' = "KillPending"
+ /\ unsafeSignal' = unsafeSignal \/ identity # "match"
+ /\ UNCHANGED <>
+StopUnknown == /\ phase \in {"Stopping", "KillPending"} /\ live
+ /\ phase' = "StopUnknown"
+ /\ UNCHANGED <>
+ProcessExit == /\ live /\ live' = FALSE
+ /\ UNCHANGED <>
+Reap == /\ ~live /\ phase \notin {"Absent", "Starting", "Exited"}
+ /\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE
+ /\ supervisor' = FALSE
+ /\ UNCHANGED <>
+Next == Reserve \/ Publish \/ Spawn \/ StartFailure \/ SupervisorDeath
+ \/ GuardDeath \/ Retry \/ ReplaceIdentity \/ Stop \/ Kill
+ \/ StopUnknown \/ ProcessExit \/ Reap
+Spec == Init /\ [][Next]_vars
+TypeOK == /\ phase \in {"Absent", "Starting", "Running", "Recoverable",
+ "Orphaned", "Stopping", "KillPending", "StopUnknown", "Exited"}
+ /\ attempts \in 0..2 /\ identity \in {"match", "foreign", "unknown"}
+ /\ \A b \in {durable, guard, locked, live, supervisor,
+ unsafeSignal, duplicate} : b \in BOOLEAN
+DurableBeforeLaunch == live => durable
+GuardRetainsLock == guard => locked
+NoForeignSignal == ~unsafeSignal
+NoDuplicate == ~duplicate
+=============================================================================
diff --git a/tests/models/tla/lifecycle-supervisor/negative.cfg b/tests/models/tla/lifecycle-supervisor/negative.cfg
new file mode 100644
index 000000000..a53d6e384
--- /dev/null
+++ b/tests/models/tla/lifecycle-supervisor/negative.cfg
@@ -0,0 +1,4 @@
+SPECIFICATION Spec
+CONSTANT SkipLiveFence = TRUE
+INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
+CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-supervisor/positive.cfg b/tests/models/tla/lifecycle-supervisor/positive.cfg
new file mode 100644
index 000000000..58bdd8c68
--- /dev/null
+++ b/tests/models/tla/lifecycle-supervisor/positive.cfg
@@ -0,0 +1,4 @@
+SPECIFICATION Spec
+CONSTANT SkipLiveFence = FALSE
+INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
+CHECK_DEADLOCK FALSE
From 6a57cdb009d9996a5b1362fdcad8826f10cc9d07 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 20:16:43 -0500
Subject: [PATCH 02/45] fix(lifecycle): pass macOS supervisor lint without
redundant cast
---
packages/lifecycle-supervisor/src/os.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 98bc95f9d..7ec5c168e 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -161,7 +161,7 @@ pub fn inspect(pid: i32) -> Result> {
.and_then(|v| v.checked_add(info.pbi_start_tvusec))
.ok_or_else(refused)?,
uid: info.pbi_uid,
- zombie: info.pbi_status == libc::SZOMB as u32,
+ zombie: info.pbi_status == libc::SZOMB,
}))
}
From 55aec84e21f30b68cd570003b27866df037a913f Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 20:46:32 -0500
Subject: [PATCH 03/45] fix(lifecycle): recover receipts across case-only
command renames
---
packages/lifecycle-supervisor/src/os.rs | 24 +++++++++++++++++--
packages/lifecycle-supervisor/src/storage.rs | 4 +++-
.../lifecycle-supervisor/tests/fake_caller.rs | 15 ++++++++++++
3 files changed, 40 insertions(+), 3 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 7ec5c168e..126ae4a43 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -62,12 +62,12 @@ pub struct Identity {
pub fn inspect(pid: i32) -> Result> {
use std::os::unix::fs::MetadataExt;
let path = format!("/proc/{pid}/stat");
- let text = match fs::read_to_string(&path) {
+ let bytes = match fs::read(&path) {
Ok(value) => value,
Err(error) if absent_error(&error) => return Ok(None),
Err(error) => return Err(error),
};
- let tail = text.rsplit_once(") ").ok_or_else(refused)?.1;
+ let tail = proc_stat_tail(&bytes)?;
let fields: Vec<_> = tail.split_whitespace().collect();
let group = fields
.get(2)
@@ -96,6 +96,17 @@ pub fn inspect(pid: i32) -> Result> {
}))
}
+#[cfg(target_os = "linux")]
+fn proc_stat_tail(bytes: &[u8]) -> Result<&str> {
+ // comm is arbitrary bytes and may itself contain parentheses. Only the
+ // fields following its final delimiter have an ASCII/UTF-8 contract.
+ let end = bytes
+ .windows(2)
+ .rposition(|v| v == b") ")
+ .ok_or_else(refused)?;
+ std::str::from_utf8(&bytes[end + 2..]).map_err(|_| refused())
+}
+
#[cfg(target_os = "linux")]
pub fn census() -> Result> {
let mut result = Vec::new();
@@ -543,6 +554,15 @@ pub fn timestamp() -> Result {
#[cfg(test)]
mod tests {
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn non_utf8_process_name_does_not_invalidate_identity_fields() {
+ assert_eq!(
+ super::proc_stat_tail(b"42 (name\xff) nested) S 1 2 3").unwrap(),
+ "S 1 2 3"
+ );
+ assert!(super::proc_stat_tail(b"42 (name) S \xff").is_err());
+ }
#[test]
fn proc_exit_during_open_or_read_is_absent_not_census_failure() {
for errno in [libc::ENOENT, libc::ESRCH] {
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index 593560a0c..6c28444fa 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -184,7 +184,9 @@ impl Root {
Read::by_ref(&mut file).take(8193).read_to_end(&mut bytes)?;
let state: State = serde_json::from_slice(&bytes).map_err(|_| refused())?;
if state.version != 1
- || state.name != name
+ // Case-insensitive filesystems may return a retained receipt under
+ // its previous spelling after a case-only command rename.
+ || !state.name.eq_ignore_ascii_case(name)
|| state.guard_pid <= 1
|| state.process_group_id != state.guard_pid
|| state.guard_start_time == 0
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index bb2683c59..dd3a3aae7 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -804,6 +804,21 @@ fn new_generation_ignores_retained_terminal_receipts() {
);
}
+#[test]
+fn case_only_rename_accepts_terminal_receipt_with_previous_spelling() {
+ let f = Fixture::new();
+ let (mut first, _) = f.start(vec![f.process("Worker", "hold")]);
+ f.await_log("fixture-ready");
+ f.stop();
+ assert!(wait(&mut first, TIMEOUT).success());
+ // Also emulate the case-insensitive lookup on Linux: the payload retains
+ // its original spelling while the lookup uses the new command name.
+ fs::rename(f.root.join("Worker.json"), f.root.join("worker.json")).unwrap();
+ let (mut next, _) = f.start(vec![f.process("worker", "hold")]);
+ f.stop();
+ assert!(wait(&mut next, TIMEOUT).success());
+}
+
#[test]
fn command_joining_guard_group_still_receives_kill() {
let f = Fixture::new();
From 597eb47788f579b5073a99b3ba70902bbfd4000d Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 20:54:38 -0500
Subject: [PATCH 04/45] fix(lifecycle): accept inherited subprocess environment
names
---
packages/lifecycle-supervisor/src/protocol.rs | 27 ++++++++++++++-----
1 file changed, 21 insertions(+), 6 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/protocol.rs b/packages/lifecycle-supervisor/src/protocol.rs
index 0a795a3e2..1781fa52a 100644
--- a/packages/lifecycle-supervisor/src/protocol.rs
+++ b/packages/lifecycle-supervisor/src/protocol.rs
@@ -77,12 +77,7 @@ impl Process {
|| self.argv[0].is_empty()
|| self.argv.iter().any(|v| v.contains('\0'))
|| self.env.iter().any(|(key, value)| {
- let mut bytes = key.bytes();
- !bytes
- .next()
- .is_some_and(|b| b.is_ascii_alphabetic() || b == b'_')
- || !bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_')
- || value.contains('\0')
+ key.is_empty() || key.contains(['\0', '=']) || value.contains('\0')
})
{
return Err(refused());
@@ -91,6 +86,26 @@ impl Process {
}
}
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn inherited_environment_uses_exec_constraints_not_shell_identifiers() {
+ let mut process: super::Process = serde_json::from_value(serde_json::json!({
+ "name": "worker", "argv": ["/usr/bin/true"], "cwd": "/",
+ "env": {"BASH_FUNC_fixture%%": "() { :; }", "1.name": "synthetic"}
+ }))
+ .unwrap();
+ assert!(process.validate().is_ok());
+ for key in ["", "bad=name", "bad\0name"] {
+ process.env.insert(key.into(), "synthetic".into());
+ assert!(process.validate().is_err());
+ process.env.remove(key);
+ }
+ process.env.insert("VALID".into(), "bad\0value".into());
+ assert!(process.validate().is_err());
+ }
+}
+
/// Length and deadline, never descriptor closure, delimit a handoff. Polling
/// handles Darwin's brief inheritable-descriptor window without waiting on EOF.
pub fn read(input: &mut (impl Read + AsRawFd)) -> Result {
From ff1e481d7a2ac4ec07aa46feecc74a3e74ebe755 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 20:55:16 -0500
Subject: [PATCH 05/45] fix(lifecycle): place protocol regressions after
implementation
---
packages/lifecycle-supervisor/src/protocol.rs | 40 +++++++++----------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/protocol.rs b/packages/lifecycle-supervisor/src/protocol.rs
index 1781fa52a..f12c63fbd 100644
--- a/packages/lifecycle-supervisor/src/protocol.rs
+++ b/packages/lifecycle-supervisor/src/protocol.rs
@@ -86,26 +86,6 @@ impl Process {
}
}
-#[cfg(test)]
-mod tests {
- #[test]
- fn inherited_environment_uses_exec_constraints_not_shell_identifiers() {
- let mut process: super::Process = serde_json::from_value(serde_json::json!({
- "name": "worker", "argv": ["/usr/bin/true"], "cwd": "/",
- "env": {"BASH_FUNC_fixture%%": "() { :; }", "1.name": "synthetic"}
- }))
- .unwrap();
- assert!(process.validate().is_ok());
- for key in ["", "bad=name", "bad\0name"] {
- process.env.insert(key.into(), "synthetic".into());
- assert!(process.validate().is_err());
- process.env.remove(key);
- }
- process.env.insert("VALID".into(), "bad\0value".into());
- assert!(process.validate().is_err());
- }
-}
-
/// Length and deadline, never descriptor closure, delimit a handoff. Polling
/// handles Darwin's brief inheritable-descriptor window without waiting on EOF.
pub fn read(input: &mut (impl Read + AsRawFd)) -> Result {
@@ -248,3 +228,23 @@ fn send(fd: i32, bytes: &[u8], deadline: Instant) -> Result<()> {
}
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn inherited_environment_uses_exec_constraints_not_shell_identifiers() {
+ let mut process: super::Process = serde_json::from_value(serde_json::json!({
+ "name": "worker", "argv": ["/usr/bin/true"], "cwd": "/",
+ "env": {"BASH_FUNC_fixture%%": "() { :; }", "1.name": "synthetic"}
+ }))
+ .unwrap();
+ assert!(process.validate().is_ok());
+ for key in ["", "bad=name", "bad\0name"] {
+ process.env.insert(key.into(), "synthetic".into());
+ assert!(process.validate().is_err());
+ process.env.remove(key);
+ }
+ process.env.insert("VALID".into(), "bad\0value".into());
+ assert!(process.validate().is_err());
+ }
+}
From 766a3afd5f0b76d689d2c5bcd0cf9a35ea62f7e2 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 22:55:57 -0500
Subject: [PATCH 06/45] fix(lifecycle): recover native orphan sessions and
bound idle work
---
packages/lifecycle-supervisor/README.md | 50 ++-
packages/lifecycle-supervisor/src/os.rs | 336 +++++++++++++++-
.../lifecycle-supervisor/src/supervisor.rs | 221 ++++++++---
.../lifecycle-supervisor/tests/fake_caller.rs | 371 +++++++++++++++++-
.../tla/lifecycle-supervisor/COVERAGE.md | 27 +-
5 files changed, 920 insertions(+), 85 deletions(-)
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 3fc78f6d8..03b909e49 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -28,10 +28,54 @@ leader. The supervisor calls `setsid` itself and detaches its standard descripto
after the framed ready receipt. `stop` and `status` are finite framed requests.
Frames, not pipe closure, delimit delivery. The per-command guard publishes kernel
identity before executing authored argv and retains ownership after the project
-supervisor dies. A killed guard with surviving session members is an unknown
-ownership state: retry and unsafe stop are refused. A command that creates another
-session with `setsid` leaves this ownership boundary and is unsupported.
+supervisor dies. A killed guard with surviving session members prevents a fresh
+launch. Stop can recover that session after validating its recorded boot, leader
+birth, UID and SID: TERM, five seconds of grace, then KILL if necessary, with
+individual incarnation checks before delivery. A different boot or reused leader
+proves the old session absent and never authorizes signaling the replacement.
+A command that creates another session with `setsid` leaves this ownership boundary and is unsupported.
The caller supplies a fresh ownership token and an existing private instance root.
No command/environment is persisted. Log output is unredacted and mode 0600.
There is no PTY: stdin is null and stdout/stderr are pipes.
+
+Status returns `liveProcessNames` and `unknownProcessNames` separately from the
+persisted `processes[].phase`. A missing/zombie guard with live descendants is
+unknown until recovery confirms absence; persisted `running` alone proves nothing.
+These are additive version-1 receipt fields; they do not alter state files.
+
+Idle guards poll stdout/stderr and react to SIGCHLD/stop signals. A live direct
+child proves work remains without a machine-wide census. After its exit, a 500 ms
+fallback session census detects surviving descendants. EOF descriptors leave the poll set. A slow census still
+finds descendants whose exit does not generate SIGCHLD for this guard. Stop uses
+20 ms confirmation intervals and keeps its five-second TERM grace. Immediate
+restart retries the instance lock for at most one second. Command-lock admission
+waits at most ten seconds for a previous owner to finish releasing its lock.
+
+The project supervisor passes its launch/stop flock lease to each guard on startup
+fd 2. The guard retains it through durable publication and receipt delivery, then
+replaces stdio with `/dev/null`. Supervisor death therefore cannot let stop pass an
+unpublished guard that might later spawn. Authored children get their own stderr
+pipe and never inherit that lease. Startup closes inherited descriptors with Linux
+`close_range` or enumeration of actual `/proc/self/fd` / `/dev/fd` entries; its cost
+is independent of the descriptor limit. Log flock acquisition and release retry EINTR.
+
+A usable process census is required before authored spawn. Linux preflight rejects
+known `hidepid` mounts and requires a readable self identity in the census. An
+unreadable process is skipped only after proving its UID is foreign; inaccessible
+same-user identities fail closed. Darwin uses the short BSD identity query to
+prove a protected process has a foreign UID; failure to establish that also refuses.
+A post-launch census error retains the guard and command lock during cleanup.
+This is not qualification of hidepid, PID namespaces, sandboxed/filtered process
+views, or a process view that silently omits same-user members. Those views remain
+unsupported. Same-user hostile receipt mutation and commands escaping their SID
+remain outside the contract. Darwin lacks pidfd, so identity recheck and numeric
+signal delivery still have a kernel observation/effect interval.
+
+The ignored `idle_guard_cpu_and_wakeup_measurement` fixture samples one guard and
+its supervisor for three five-second idle intervals. Linux CPU ticks and voluntary
+context switches are process accounting, not hardware wakeup telemetry. Run it
+alone with `--ignored --exact idle_guard_cpu_and_wakeup_measurement --nocapture`.
+The ignored `million_descriptor_limit_qualification` fixture requires a host hard
+RLIMIT_NOFILE of at least 1048576; the ordinary raised-limit fixture uses the host's
+available limit. A host refusing that raise does not qualify the million-fd case.
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 126ae4a43..df436fef5 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -1,5 +1,4 @@
use crate::{Result, refused};
-#[cfg(target_os = "linux")]
use std::fs;
use std::{
io,
@@ -7,6 +6,12 @@ use std::{
sync::atomic::{AtomicBool, Ordering},
};
+pub static CHILD_CHANGED: AtomicBool = AtomicBool::new(false);
+
+extern "C" fn child_signal(_: i32) {
+ CHILD_CHANGED.store(true, Ordering::Relaxed);
+}
+
pub static STOP: AtomicBool = AtomicBool::new(false);
extern "C" fn stop_signal(_: i32) {
@@ -18,7 +23,7 @@ pub fn signals() -> Result<()> {
// atomic flag; the kernel copies the action and retains no Rust pointer.
unsafe {
let mut action: libc::sigaction = std::mem::zeroed();
- action.sa_sigaction = libc::SIG_DFL;
+ action.sa_sigaction = child_signal as *const () as usize;
libc::sigemptyset(&mut action.sa_mask);
if libc::sigaction(libc::SIGCHLD, &action, std::ptr::null_mut()) != 0 {
return Err(io::Error::last_os_error());
@@ -112,12 +117,26 @@ pub fn census() -> Result> {
let mut result = Vec::new();
for entry in fs::read_dir("/proc")? {
let entry = entry?;
- if let Some(pid) = entry.file_name().to_str().and_then(|v| v.parse().ok())
- && let Some(identity) = inspect(pid)?
- {
- result.push(identity);
+ if let Some(pid) = entry.file_name().to_str().and_then(|v| v.parse().ok()) {
+ match inspect(pid) {
+ Ok(Some(identity)) => result.push(identity),
+ Ok(None) => (),
+ Err(error)
+ if matches!(error.raw_os_error(), Some(libc::EPERM) | Some(libc::EACCES)) =>
+ {
+ use std::os::unix::fs::MetadataExt;
+ match entry.metadata() {
+ // SAFETY: geteuid is an observation with no preconditions.
+ Ok(metadata) if metadata.uid() != unsafe { libc::geteuid() } => (),
+ Err(error) if absent_error(&error) => (),
+ _ => return Err(error),
+ }
+ }
+ Err(error) => return Err(error),
+ }
}
}
+ readable_self(&result)?;
Ok(result)
}
@@ -208,13 +227,38 @@ pub fn census() -> Result> {
Ok(None) => (),
// Unrelated protected processes cannot belong to this user's
// authored session. Exact guard reads remain strict in inspect.
- Err(error) if error.raw_os_error() == Some(libc::EPERM) => (),
+ Err(error) if error.raw_os_error() == Some(libc::EPERM) && foreign_uid(pid)? => (),
Err(error) => return Err(error),
}
}
+ readable_self(&result)?;
Ok(result)
}
+#[cfg(target_os = "macos")]
+fn foreign_uid(pid: i32) -> Result {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: bounded writable struct buffer, retained only for this call.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDT_SHORTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ if size as usize != std::mem::size_of::() {
+ let error = io::Error::last_os_error();
+ if absent_error(&error) {
+ return Ok(true);
+ }
+ return Err(refused());
+ }
+ // SAFETY: exact struct size was initialized; geteuid only observes identity.
+ Ok(unsafe { info.assume_init().pbsi_uid != libc::geteuid() })
+}
+
#[cfg(target_os = "macos")]
pub fn boot() -> Result {
let mut bytes = [0u8; 64];
@@ -282,6 +326,41 @@ pub fn recorded_live(state: &crate::storage::State, excluding_guard: bool) -> Re
live_group(state.guard_pid, excluding_guard.then_some(state.guard_pid))
}
+pub enum Observation {
+ Absent,
+ Live,
+ Unknown,
+}
+
+/// Health observation is stricter than a persisted running phase. A lost guard
+/// with remaining session members is unknown, never healthy or absent.
+pub fn observe_state(state: &crate::storage::State) -> Observation {
+ let observed = (|| -> Result {
+ if state.boot_id != boot()? {
+ return Ok(Observation::Absent);
+ }
+ let leader = inspect(state.guard_pid)?;
+ if let Some(identity) = &leader
+ && (identity.start != state.guard_start_time
+ || identity.uid != state.uid
+ || identity.session != state.guard_pid)
+ {
+ return Ok(Observation::Absent);
+ }
+ if !live_group(state.guard_pid, Some(state.guard_pid))? {
+ return Ok(Observation::Absent);
+ }
+ Ok(
+ if leader.is_some_and(|p| !p.zombie && p.group == state.guard_pid) {
+ Observation::Live
+ } else {
+ Observation::Unknown
+ },
+ )
+ })();
+ observed.unwrap_or(Observation::Unknown)
+}
+
/// The guard is never a member of an authored process group. It retains the
/// command lock while KILL is confirmed, including uninterruptible processes.
pub fn session_signal(session: i32, signal: i32) -> Result<()> {
@@ -446,18 +525,24 @@ pub fn close_inherited() -> Result<()> {
return Ok(());
}
}
- // Portable fallback is bounded by the process's descriptor limit and does
- // not allocate or retain descriptor references while closing them.
- let mut limit: libc::rlimit = unsafe { std::mem::zeroed() };
- // SAFETY: getrlimit initializes one live rlimit value.
- if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) } != 0 {
- return Err(io::Error::last_os_error());
- }
- if limit.rlim_cur > 1_048_576 {
- return Err(refused());
- }
- for fd in 3..limit.rlim_cur as i32 {
- // SAFETY: startup owns no fd >=3; failed closes are harmless gaps.
+ // Collect first, so the directory iterator's descriptor is closed before
+ // the sweep. Work is proportional to open descriptors, not RLIMIT_NOFILE.
+ #[cfg(target_os = "linux")]
+ let directory = "/proc/self/fd";
+ #[cfg(target_os = "macos")]
+ let directory = "/dev/fd";
+ let descriptors: Vec = fs::read_dir(directory)?
+ .map(|entry| {
+ entry?
+ .file_name()
+ .to_str()
+ .and_then(|v| v.parse().ok())
+ .ok_or_else(refused)
+ })
+ .collect::>()?;
+ for fd in descriptors.into_iter().filter(|fd| *fd >= 3) {
+ // SAFETY: startup owns no application descriptors >=3. The iterator's
+ // former descriptor may already be closed; that gap is harmless.
unsafe {
libc::close(fd);
}
@@ -509,13 +594,134 @@ pub fn group_signal(group: i32, signal: i32) -> Result<()> {
// SAFETY: validated positive group number is negated for group delivery.
if unsafe { libc::kill(-group, signal) } != 0 {
let error = io::Error::last_os_error();
- if error.raw_os_error() != Some(libc::ESRCH) {
+ group_signal_result(group, error, census)?;
+ }
+ Ok(())
+}
+
+/// EPERM may race the last live member becoming an unreaped zombie. Only a
+/// fresh, successful group census proving no non-zombie member makes it benign.
+fn group_signal_result(
+ group: i32,
+ error: io::Error,
+ census: impl FnOnce() -> Result>,
+) -> Result<()> {
+ if error.raw_os_error() == Some(libc::ESRCH) {
+ return Ok(());
+ }
+ if error.raw_os_error() == Some(libc::EPERM)
+ && !census()?.iter().any(|p| p.group == group && !p.zombie)
+ {
+ return Ok(());
+ }
+ Err(error)
+}
+
+/// Poll only pipe readiness between slow session checks. SIGCHLD and stop
+/// signals interrupt poll; flags also cover signals arriving before this call.
+pub fn wait_activity(fds: &[i32], timeout: std::time::Duration) -> Result<()> {
+ if STOP.load(Ordering::Relaxed) || CHILD_CHANGED.swap(false, Ordering::Relaxed) {
+ return Ok(());
+ }
+ let mut polls: Vec<_> = fds
+ .iter()
+ .map(|fd| libc::pollfd {
+ fd: *fd,
+ events: libc::POLLIN,
+ revents: 0,
+ })
+ .collect();
+ // SAFETY: initialized pollfd slice is live for its supplied length.
+ if unsafe {
+ libc::poll(
+ polls.as_mut_ptr(),
+ polls.len() as _,
+ timeout.as_millis() as i32,
+ )
+ } < 0
+ {
+ let error = io::Error::last_os_error();
+ if error.kind() != io::ErrorKind::Interrupted {
return Err(error);
}
}
Ok(())
}
+/// A dead leader's SID cannot be reused while any old session member remains.
+/// Validate the receipt and each incarnation again before individual delivery.
+/// Never use group delivery here: a group number can outlive its observed members.
+pub fn signal_recorded_session(state: &crate::storage::State, signal: i32) -> Result<()> {
+ if !recorded_live(state, false)? {
+ return Ok(());
+ }
+ for member in census()?
+ .into_iter()
+ .filter(|p| p.session == state.guard_pid && !p.zombie)
+ {
+ if member.uid != state.uid || member.start < state.guard_start_time {
+ return Err(refused());
+ }
+ if !recorded_live(state, false)? {
+ return Ok(());
+ }
+ signal_member(&member, signal)?;
+ }
+ Ok(())
+}
+
+/// A successful census must contain our own readable identity. Known restricted
+/// proc mounts are unsupported: hidden same-user members defeat absence proofs.
+pub fn census_preflight() -> Result<()> {
+ #[cfg(target_os = "linux")]
+ {
+ let mounts = fs::read_to_string("/proc/self/mountinfo")?;
+ if restricted_mount(&mounts) {
+ return Err(refused());
+ }
+ }
+ readable_self(&census()?)
+}
+
+#[cfg(target_os = "linux")]
+fn restricted_mount(mounts: &str) -> bool {
+ mounts.lines().any(|line| {
+ line.split_whitespace().any(|field| {
+ field
+ .split(',')
+ .any(|option| option.starts_with("hidepid=") && option != "hidepid=0")
+ })
+ })
+}
+
+fn readable_self(members: &[Identity]) -> Result<()> {
+ let pid = std::process::id() as i32;
+ if !members.iter().any(|p| p.pid == pid && !p.zombie) {
+ return Err(refused());
+ }
+ Ok(())
+}
+
+pub fn flock_retry(fd: i32, operation: i32) -> Result<()> {
+ retry_interrupted(|| {
+ // SAFETY: caller retains ownership of the live descriptor across retries.
+ if unsafe { libc::flock(fd, operation) } == 0 {
+ Ok(())
+ } else {
+ Err(io::Error::last_os_error())
+ }
+ })
+}
+
+fn retry_interrupted(mut operation: impl FnMut() -> Result<()>) -> Result<()> {
+ loop {
+ match operation() {
+ Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
+ result => return result,
+ }
+ }
+}
+
pub fn nonblocking(fd: &impl AsRawFd) -> Result<()> {
// SAFETY: fcntl borrows a live descriptor and supplies integer flags only.
let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) };
@@ -554,6 +760,96 @@ pub fn timestamp() -> Result {
#[cfg(test)]
mod tests {
+ #[test]
+ fn flock_interruption_retries_but_other_errors_do_not() {
+ let mut calls = 0;
+ super::retry_interrupted(|| {
+ calls += 1;
+ if calls < 3 {
+ Err(std::io::Error::from_raw_os_error(libc::EINTR))
+ } else {
+ Ok(())
+ }
+ })
+ .unwrap();
+ assert_eq!(calls, 3);
+ let error =
+ super::retry_interrupted(|| Err(std::io::Error::from_raw_os_error(libc::EPERM)))
+ .unwrap_err();
+ assert_eq!(error.raw_os_error(), Some(libc::EPERM));
+ }
+
+ #[test]
+ fn unusable_census_refuses_before_launch() {
+ assert!(super::readable_self(&[]).is_err());
+ assert!(super::readable_self(&super::census().unwrap()).is_ok());
+ #[cfg(target_os = "linux")]
+ {
+ assert!(super::restricted_mount(
+ "1 2 0:1 / /proc rw - proc proc rw,hidepid=2"
+ ));
+ assert!(super::restricted_mount(
+ "1 2 0:1 / /proc rw - proc proc rw,hidepid=invisible"
+ ));
+ assert!(!super::restricted_mount(
+ "1 2 0:1 / /proc rw - proc proc rw,hidepid=0"
+ ));
+ }
+ }
+
+ #[test]
+ fn eperm_requires_fresh_zombie_only_group_proof() {
+ use super::*;
+ let member = Identity {
+ pid: 123,
+ group: 123,
+ session: 123,
+ start: 1,
+ uid: 0,
+ zombie: true,
+ };
+ let denied = || io::Error::from_raw_os_error(libc::EPERM);
+ assert!(group_signal_result(123, denied(), || Ok(vec![member.clone()])).is_ok());
+ let mut live = member.clone();
+ live.zombie = false;
+ assert!(group_signal_result(123, denied(), || Ok(vec![member.clone(), live])).is_err());
+ assert!(group_signal_result(123, denied(), || Err(refused())).is_err());
+ assert!(
+ group_signal_result(123, io::Error::from_raw_os_error(libc::EIO), || Ok(vec![]))
+ .is_err()
+ );
+ }
+
+ #[test]
+ fn real_unreaped_group_is_not_live() {
+ use super::*;
+ // SAFETY: child uses only async-signal-safe calls and _exit, never Rust.
+ let pid = unsafe { libc::fork() };
+ assert!(pid >= 0);
+ if pid == 0 {
+ unsafe {
+ libc::setsid();
+ libc::_exit(0);
+ }
+ }
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ loop {
+ if inspect(pid).unwrap().is_some_and(|p| p.zombie) {
+ break;
+ }
+ assert!(std::time::Instant::now() < deadline);
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ }
+ assert!(!live_group(pid, None).unwrap());
+ assert!(
+ group_signal_result(pid, io::Error::from_raw_os_error(libc::EPERM), census).is_ok()
+ );
+ group_signal(pid, libc::SIGTERM).unwrap();
+ let mut status = 0;
+ // SAFETY: reap only our own still-unreaped child, with a valid status pointer.
+ assert_eq!(unsafe { libc::waitpid(pid, &mut status, 0) }, pid);
+ }
+
#[cfg(target_os = "linux")]
#[test]
fn non_utf8_process_name_does_not_invalidate_identity_fields() {
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index d52babf29..b8d1a3b8f 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -19,6 +19,7 @@ use std::{
};
const GRACE: Duration = Duration::from_secs(5);
+const IDLE_TICK: Duration = Duration::from_millis(500);
const TICK: Duration = Duration::from_millis(20);
const LINE_LIMIT: usize = 64 * 1024;
@@ -29,6 +30,30 @@ struct Receipt {
supervisor_pid: u32,
outcome: &'static str,
processes: Vec,
+ live_process_names: Vec,
+ unknown_process_names: Vec,
+}
+
+impl Receipt {
+ fn new(supervisor_pid: u32, outcome: &'static str, processes: Vec) -> Self {
+ let mut live_process_names = Vec::new();
+ let mut unknown_process_names = Vec::new();
+ for state in &processes {
+ match os::observe_state(state) {
+ os::Observation::Live => live_process_names.push(state.name.clone()),
+ os::Observation::Absent => (),
+ os::Observation::Unknown => unknown_process_names.push(state.name.clone()),
+ }
+ }
+ Self {
+ version: 1,
+ supervisor_pid,
+ outcome,
+ processes,
+ live_process_names,
+ unknown_process_names,
+ }
+ }
}
pub fn main() -> Result<()> {
@@ -67,22 +92,20 @@ pub fn main() -> Result<()> {
};
protocol::write(
&mut std::io::stdout(),
- &Receipt {
- version: 1,
- supervisor_pid: 0,
- outcome: if result.is_err() {
+ &Receipt::new(
+ 0,
+ if result.is_err() {
"unknown"
} else if args[0] == "stop" {
"stopped"
} else {
"observed"
},
- processes: root
- .states()?
+ root.states()?
.into_iter()
.filter(|state| state.owner_token == request.owner_token)
.collect(),
- },
+ ),
)?;
result
}
@@ -103,7 +126,8 @@ fn run(request: Launch) -> Result<()> {
}
}
let root = Root::open(&request.root)?;
- let _instance_lock = root.lock(".instance.lock")?;
+ let _instance_lock = instance_lock(&root)?;
+ os::census_preflight()?;
let launch_lock = root.lock_wait(".stop.lock")?;
// Validate every old claim before starting any new command. A guard survives
// project-owner death; its command lock also closes concurrent restart races.
@@ -127,7 +151,10 @@ fn run(request: Launch) -> Result<()> {
.env_clear()
.stdin(Stdio::piped())
.stdout(Stdio::piped())
- .stderr(Stdio::null());
+ // Keep stop admission leased even if this supervisor dies
+ // before the guard publishes its durable identity. fd 2 is
+ // reserved until guard receipt delivery/detach (or exit).
+ .stderr(Stdio::from(launch_lock.try_clone()?));
// SAFETY: setsid is async-signal-safe, and this closure allocates no
// application state. It runs in the freshly forked child only.
unsafe {
@@ -154,12 +181,7 @@ fn run(request: Launch) -> Result<()> {
}
protocol::write(
&mut std::io::stdout(),
- &Receipt {
- version: 1,
- supervisor_pid: std::process::id(),
- outcome: "ready",
- processes: states,
- },
+ &Receipt::new(std::process::id(), "ready", states),
)?;
drop(launch_lock);
detach_stdio()?;
@@ -182,7 +204,7 @@ fn run(request: Launch) -> Result<()> {
if !running {
return Ok(());
}
- thread::sleep(TICK);
+ os::wait_activity(&[], IDLE_TICK)?;
}
})();
if outcome.is_err() {
@@ -193,6 +215,31 @@ fn run(request: Launch) -> Result<()> {
outcome
}
+fn instance_lock(root: &Root) -> Result {
+ // Preserve the uncontended fast path; a bounded retry below distinguishes
+ // contention from persistent file/OS failures with the actual errno.
+ if let Ok(file) = root.lock(".instance.lock") {
+ return Ok(file);
+ }
+ let file = root.file(".instance.lock", libc::O_RDWR | libc::O_CREAT)?;
+ let deadline = Instant::now() + Duration::from_secs(1);
+ loop {
+ // SAFETY: file remains owned throughout this bounded retry.
+ if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
+ return Ok(file);
+ }
+ let error = std::io::Error::last_os_error();
+ if !matches!(
+ error.kind(),
+ std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
+ ) || Instant::now() >= deadline
+ {
+ return Err(error);
+ }
+ thread::sleep(TICK);
+ }
+}
+
fn rollback(children: &mut [Child]) {
for child in children.iter_mut() {
if child.try_wait().ok().flatten().is_some() {
@@ -223,7 +270,7 @@ fn guard(request: GuardLaunch) -> Result<()> {
protocol::validate_control(request.version, &request.root, &request.owner_token)?;
request.process.validate()?;
let root = Root::open(&request.root)?;
- let _command_lock = root.lock(&format!("{}.lock", request.process.name))?;
+ let _command_lock = root.lock_wait(&format!("{}.lock", request.process.name))?;
if let Some(previous) = root.load(&request.process.name)?
&& os::recorded_live(&previous, false)?
{
@@ -236,6 +283,7 @@ fn guard(request: GuardLaunch) -> Result<()> {
}
os::signals()?;
os::subreaper()?;
+ os::census_preflight()?;
let mut state = State {
version: 1,
name: request.process.name.clone(),
@@ -316,10 +364,19 @@ fn guard_command(
let mut killed = false;
let mut direct_exited = false;
let mut drained_after_exit = None;
+ let mut last_census = Instant::now() - IDLE_TICK;
+ let mut live = true;
+ let mut stdout_open = true;
+ let mut stderr_open = true;
loop {
// Alternate bounded reads so stdout bursts cannot starve stderr/stop.
- out.drain(&mut stdout, log, &state.name, "stdout")?;
- err.drain(&mut stderr, log, &state.name, "stderr")?;
+ if stdout_open {
+ stdout_open = out.drain(&mut stdout, log, &state.name, "stdout")?;
+ }
+ if stderr_open {
+ stderr_open = err.drain(&mut stderr, log, &state.name, "stderr")?;
+ }
+ let was_exited = direct_exited;
if !direct_exited && let Some(status) = child.try_wait()? {
direct_exited = true;
state.exit_code = status.code();
@@ -335,7 +392,14 @@ fn guard_command(
state.signal = status.signal();
}
}
- let live = os::live_group(state.process_group_id, Some(state.guard_pid))?;
+ // A live direct child already proves work remains. Census is needed
+ // only after that child exits (descendants may remain), or during stop.
+ if (direct_exited && (last_census.elapsed() >= IDLE_TICK || !was_exited))
+ || stopping.is_some()
+ {
+ live = os::live_group(state.process_group_id, Some(state.guard_pid))?;
+ last_census = Instant::now();
+ }
if os::STOP.swap(false, Ordering::Relaxed) && stopping.is_none() {
state.phase = "stopping".into();
root.save(state)?;
@@ -366,43 +430,108 @@ fn guard_command(
}
os::session_signal(state.guard_pid, libc::SIGKILL)?;
}
- thread::sleep(TICK);
+ let fds: Vec<_> = [
+ (stdout_open, stdout.as_raw_fd()),
+ (stderr_open, stderr.as_raw_fd()),
+ ]
+ .into_iter()
+ .filter_map(|(open, fd)| open.then_some(fd))
+ .collect();
+ os::wait_activity(
+ &fds,
+ if stopping.is_some() || drained_after_exit.is_some() {
+ TICK
+ } else {
+ IDLE_TICK
+ },
+ )?;
}
}
fn stop(states: &[State]) -> Result<()> {
+ struct Target {
+ state: State,
+ guard_live: bool,
+ orphan_term: Option,
+ }
let mut active = Vec::new();
- // Validate the full set before taking the first effect.
+ // Validate every receipt before the first effect. Absence may race a guard's
+ // natural exit; a dead leader still leaves a recoverable recorded session.
for state in states {
if !os::recorded_live(state, false)? {
continue;
}
- let identity = os::inspect(state.guard_pid)?.ok_or_else(refused)?;
- os::same(state, &identity)?;
- active.push(state.clone());
+ let guard_live = match os::inspect(state.guard_pid)? {
+ Some(identity) if !identity.zombie => {
+ os::same(state, &identity)?;
+ true
+ }
+ _ => false,
+ };
+ active.push(Target {
+ state: state.clone(),
+ guard_live,
+ orphan_term: None,
+ });
}
- for state in &active {
- if let Err(error) = os::signal_guard(state) {
- // Natural exit can race between admission and pidfd delivery.
- // Only a new kernel absence proof turns that race into success.
- if os::recorded_live(state, false)? {
- return Err(error);
+ for target in &mut active {
+ let state = &target.state;
+ let result = if target.guard_live {
+ os::signal_guard(state)
+ } else {
+ let result = os::signal_recorded_session(state, libc::SIGTERM);
+ target.orphan_term = Some(Instant::now());
+ result
+ };
+ if let Err(error) = result {
+ if !os::recorded_live(state, false)? {
+ continue;
+ }
+ // Guard death during admission switches to verified session recovery.
+ match os::inspect(state.guard_pid)? {
+ None => {
+ os::signal_recorded_session(state, libc::SIGTERM)?;
+ target.orphan_term = Some(Instant::now());
+ }
+ Some(identity) if identity.zombie => {
+ os::signal_recorded_session(state, libc::SIGTERM)?;
+ target.orphan_term = Some(Instant::now());
+ }
+ _ => return Err(error),
}
}
}
- let deadline = Instant::now() + GRACE + Duration::from_secs(2);
- while Instant::now() < deadline {
+ let start = Instant::now();
+ while start.elapsed() < GRACE + Duration::from_secs(2) {
let mut live = false;
- for state in &active {
- live |= os::recorded_live(state, false)?;
+ for target in &mut active {
+ let state = &target.state;
+ if !os::recorded_live(state, false)? {
+ continue;
+ }
+ live = true;
+ // Guard death after TERM admission must not skip authored TERM or
+ // borrow the dead guard's grace interval. Late loss may time out as
+ // unknown; it never shortens this recovered session's own grace.
+ let orphan = os::inspect(state.guard_pid)?.is_none_or(|p| p.zombie);
+ if orphan {
+ if target.orphan_term.is_none() {
+ os::signal_recorded_session(state, libc::SIGTERM)?;
+ target.orphan_term = Some(Instant::now());
+ }
+ if target
+ .orphan_term
+ .is_some_and(|since| since.elapsed() >= GRACE)
+ {
+ os::signal_recorded_session(state, libc::SIGKILL)?;
+ }
+ }
}
if !live {
return Ok(());
}
thread::sleep(TICK);
}
- // Unknown outcome: the guard retains its command lock while any live
- // member remains. Neither timeout nor zombie-only existence implies alive.
Err(refused())
}
@@ -433,11 +562,11 @@ impl Lines {
log: &mut File,
name: &str,
stream: &str,
- ) -> Result<()> {
+ ) -> Result {
let mut buffer = [0u8; 8192];
for _ in 0..32 {
match input.read(&mut buffer) {
- Ok(0) => break,
+ Ok(0) => return Ok(false),
Ok(count) => {
for byte in &buffer[..count] {
if *byte == b'\n' {
@@ -458,7 +587,7 @@ impl Lines {
Err(error) => return Err(error),
}
}
- Ok(())
+ Ok(true)
}
fn flush(&mut self, log: &mut File, name: &str, stream: &str, trim_cr: bool) -> Result<()> {
@@ -471,17 +600,11 @@ impl Lines {
let record = format!("{}\t{name}\t{stream}\t{message}\n", os::timestamp()?);
// One append per complete record; concurrent guards share the same
// flock so records larger than PIPE_BUF never interleave.
- // SAFETY: flock borrows this live file descriptor until LOCK_UN.
- if unsafe { libc::flock(log.as_raw_fd(), libc::LOCK_EX) } != 0 {
- return Err(std::io::Error::last_os_error());
- }
+ os::flock_retry(log.as_raw_fd(), libc::LOCK_EX)?;
let result = log.write_all(record.as_bytes());
- // SAFETY: unlock the same live descriptor after the write completes.
- let unlocked = unsafe { libc::flock(log.as_raw_fd(), libc::LOCK_UN) };
+ let unlocked = os::flock_retry(log.as_raw_fd(), libc::LOCK_UN);
result?;
- if unlocked != 0 {
- return Err(std::io::Error::last_os_error());
- }
+ unlocked?;
self.pending.clear();
Ok(())
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index dd3a3aae7..b6fd702e0 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -461,6 +461,7 @@ fn zombies_do_not_hold_liveness() {
wait(&mut caller, TIMEOUT).success(),
"zombie-only session remained alive"
);
+ assert!(f.log().contains("unreaped-zombie-confirmed"));
}
#[test]
@@ -556,6 +557,161 @@ fn child_descriptor_allowlist() {
assert!(!f.log().contains("descriptor-leak"));
}
+#[test]
+fn repeated_immediate_restart_waits_for_instance_lock_release() {
+ let f = Fixture::new();
+ let mut previous = None;
+ for _ in 0..8 {
+ let (caller, _) = f.start(vec![f.process("worker", "hold")]);
+ if let Some(mut old) = previous.take() {
+ assert!(wait(&mut old, TIMEOUT).success());
+ }
+ f.stop();
+ previous = Some(caller);
+ }
+ assert!(wait(previous.as_mut().unwrap(), TIMEOUT).success());
+}
+
+#[cfg(target_os = "linux")]
+#[test]
+fn raised_descriptor_limit_does_not_delay_startup() {
+ // Exercise the largest supported soft limit without changing this host's
+ // hard limit. The exact million-fd qualification remains separate below.
+ let mut limit: libc::rlimit = unsafe { std::mem::zeroed() };
+ assert_eq!(
+ unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) },
+ 0
+ );
+ descriptor_limit_startup(limit.rlim_max.min(1_048_576));
+}
+
+#[cfg(target_os = "linux")]
+#[test]
+#[ignore = "qualification requires a hard RLIMIT_NOFILE of at least 1048576"]
+fn million_descriptor_limit_qualification() {
+ descriptor_limit_startup(1_048_576);
+}
+
+#[cfg(target_os = "linux")]
+fn descriptor_limit_startup(descriptors: libc::rlim_t) {
+ use std::os::unix::process::CommandExt;
+ let f = Fixture::new();
+ let mut cmd = command(&f.root, "run");
+ // SAFETY: adjusts only this freshly forked fixture child's descriptor limit.
+ unsafe {
+ cmd.pre_exec(move || {
+ let limit = libc::rlimit {
+ rlim_cur: descriptors,
+ rlim_max: descriptors,
+ };
+ if libc::setrlimit(libc::RLIMIT_NOFILE, &limit) != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
+ }
+ let mut caller = cmd.spawn().unwrap();
+ let start = Instant::now();
+ send(
+ caller.stdin.as_mut().unwrap(),
+ &f.launch(vec![f.process("worker", "descriptors")]),
+ );
+ let receipt = receive(caller.stdout.as_mut().unwrap());
+ f.remember(&receipt);
+ assert_eq!(receipt["outcome"], "ready");
+ assert!(wait(&mut caller, TIMEOUT).success());
+ assert!(start.elapsed() < Duration::from_secs(3));
+ assert!(f.log().contains("descriptor-allowlist-ok"));
+}
+
+#[cfg(target_os = "linux")]
+#[test]
+#[ignore = "bounded idle CPU/context-switch measurement; run alone with --nocapture"]
+fn idle_guard_cpu_and_wakeup_measurement() {
+ fn sample(pid: i32) -> (u64, u64) {
+ let stat = fs::read_to_string(format!("/proc/{pid}/stat")).unwrap();
+ let fields: Vec<_> = stat
+ .rsplit_once(") ")
+ .unwrap()
+ .1
+ .split_whitespace()
+ .collect();
+ let ticks = fields[11].parse::().unwrap() + fields[12].parse::().unwrap();
+ let status = fs::read_to_string(format!("/proc/{pid}/status")).unwrap();
+ let switches = status
+ .lines()
+ .find_map(|line| {
+ line.strip_prefix("voluntary_ctxt_switches:")
+ .map(|v| v.trim().parse::().unwrap())
+ })
+ .unwrap();
+ (ticks, switches)
+ }
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let guard = pid(&f.state("worker"), "guardPid");
+ let hz = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as f64;
+ for trial in 1..=3 {
+ let before = sample(guard);
+ let before_parent = sample(caller.id() as i32);
+ let start = Instant::now();
+ thread::sleep(Duration::from_secs(5));
+ let wall = start.elapsed().as_secs_f64();
+ let after = sample(guard);
+ let after_parent = sample(caller.id() as i32);
+ let cpu = (after.0 - before.0) as f64 / hz;
+ let switches = after.1 - before.1;
+ eprintln!(
+ "trial={trial} wall_seconds={wall:.3} guard_cpu_seconds={cpu:.3} guard_voluntary_switches={switches} supervisor_cpu_seconds={:.3} supervisor_voluntary_switches={}",
+ (after_parent.0 - before_parent.0) as f64 / hz,
+ after_parent.1 - before_parent.1
+ );
+ assert!(cpu / wall < 0.01, "idle guard exceeded 1% of one CPU");
+ assert!(
+ (switches as f64) / wall < 5.0,
+ "idle guard exceeded 5 voluntary switches/s"
+ );
+ }
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn natural_exit_during_stop_admission_is_successful_absence() {
+ let f = Fixture::new();
+ for _ in 0..8 {
+ let (mut caller, _) = f.start(vec![f.process("worker", "unicode")]);
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert!(wait(&mut stop, TIMEOUT).success());
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(wait(&mut caller, TIMEOUT).success());
+ }
+}
+
+#[test]
+fn status_reports_kernel_liveness_separately_from_persisted_phase() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ let (mut status, receipt) = f.request("status", &f.control(TOKEN));
+ assert!(wait(&mut status, TIMEOUT).success());
+ assert_eq!(receipt["liveProcessNames"], json!(["worker"]));
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let mut state = f.state("worker");
+ state["phase"] = json!("running");
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&state).unwrap(),
+ )
+ .unwrap();
+ let (mut status, receipt) = f.request("status", &f.control(TOKEN));
+ assert!(wait(&mut status, TIMEOUT).success());
+ assert_eq!(receipt["processes"][0]["phase"], "running");
+ assert_eq!(receipt["liveProcessNames"], json!([]));
+ assert_eq!(receipt["unknownProcessNames"], json!([]));
+}
+
#[test]
fn concurrent_stop_requests_settle() {
let f = Fixture::new();
@@ -612,6 +768,148 @@ fn inherited_ignored_signal_is_reset() {
assert!(wait(&mut caller, TIMEOUT).success());
}
+#[cfg(target_os = "linux")]
+#[test]
+fn supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased() {
+ use std::os::unix::fs::OpenOptionsExt;
+ let f = Fixture::new();
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create_new(true)
+ .mode(0o600)
+ .open(f.root.join("worker.lock"))
+ .unwrap();
+ assert_eq!(unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) }, 0);
+ let mut caller = command(&f.root, "run").spawn().unwrap();
+ send(
+ caller.stdin.as_mut().unwrap(),
+ &f.launch(vec![
+ f.process("first", "hold"),
+ f.process("worker", "hold"),
+ ]),
+ );
+ f.await_log("fixture-ready");
+ let first = f.state("first");
+ f.remember(&json!({"processes":[first.clone()]}));
+ let guard = eventually(|| {
+ fs::read_to_string(format!(
+ "/proc/{}/task/{}/children",
+ caller.id(),
+ caller.id()
+ ))
+ .ok()
+ .and_then(|s| {
+ s.split_whitespace()
+ .filter_map(|v| v.parse::().ok())
+ .find(|p| *p != pid(&first, "guardPid"))
+ })
+ });
+ // Wait for the guard to exec and close all inherited descriptors except
+ // stdio; fd 2 must still carry the stop lease while command lock is held.
+ eventually(|| {
+ fs::read_link(format!("/proc/{guard}/exe"))
+ .ok()
+ .filter(|p| {
+ p.file_name()
+ .is_some_and(|n| n == "hack-lifecycle-supervisor")
+ })
+ });
+ let lease_inode = fs::metadata(f.root.join(".stop.lock")).unwrap().ino();
+ eventually(|| {
+ fs::metadata(format!("/proc/{guard}/fd/2"))
+ .ok()
+ .filter(|metadata| metadata.ino() == lease_inode)
+ });
+ let guard_identity = identity(guard).unwrap();
+ f.owned.borrow_mut().push(guard_identity.clone());
+ caller.kill().unwrap();
+ caller.wait().unwrap();
+ let mut stop = command(&f.root, "stop").spawn().unwrap();
+ send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ thread::sleep(Duration::from_millis(150));
+ assert!(
+ stop.try_wait().unwrap().is_none(),
+ "stop passed an unpublished guard"
+ );
+ assert!(!f.root.join("worker.json").exists());
+ // SAFETY: captured direct guard is still blocked on our fixture lock.
+ assert!(identity(guard).as_ref() == Some(&guard_identity));
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
+ assert!(wait(&mut stop, TIMEOUT).success());
+ drop(lock);
+ let (mut next, _) = f.start(vec![f.process("worker", "hold")]);
+ f.stop();
+ assert!(wait(&mut next, TIMEOUT).success());
+}
+
+#[test]
+fn supervisor_killed_while_ready_receipt_is_blocked_recovers() {
+ use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
+ let f = Fixture::new();
+ let (_reader, mut writer) = std::os::unix::net::UnixStream::pair().unwrap();
+ writer.set_nonblocking(true).unwrap();
+ // Saturate the very descriptor used for the caller's receipt. Authored
+ // launch can finish, but the supervisor cannot commit a ready frame.
+ let buffer = [0u8; 8192];
+ loop {
+ match writer.write(&buffer) {
+ Ok(_) => (),
+ Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break,
+ Err(e) => panic!("fixture receipt saturation: {e}"),
+ }
+ }
+ // SAFETY: ownership moves from UnixStream into OwnedFd exactly once.
+ let fd = unsafe { OwnedFd::from_raw_fd(writer.into_raw_fd()) };
+ let mut caller = command(&f.root, "run")
+ .stdout(Stdio::from(fd))
+ .spawn()
+ .unwrap();
+ send(
+ caller.stdin.as_mut().unwrap(),
+ &f.launch(vec![f.process("worker", "hold")]),
+ );
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ f.remember(&json!({"processes":[state]}));
+ assert!(caller.try_wait().unwrap().is_none());
+ caller.kill().unwrap();
+ caller.wait().unwrap();
+ f.stop();
+}
+
+#[test]
+fn guard_crash_before_complete_launch_frame_starts_nothing() {
+ use std::os::unix::process::CommandExt;
+ let f = Fixture::new();
+ let mut cmd = command(&f.root, "guard");
+ // SAFETY: changes only the fresh child into its own session.
+ unsafe {
+ cmd.pre_exec(|| {
+ if libc::setsid() < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
+ }
+ let mut guard = cmd.spawn().unwrap();
+ // The frame header authorizes no launch; kill at the deterministic input
+ // admission boundary rather than relying on scheduler timing.
+ guard
+ .stdin
+ .as_mut()
+ .unwrap()
+ .write_all(&100u32.to_be_bytes())
+ .unwrap();
+ guard.kill().unwrap();
+ guard.wait().unwrap();
+ assert!(!f.root.join("worker.json").exists());
+ assert!(!f.root.join("lifecycle.log").exists());
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
#[test]
fn supervisor_killed_during_output_burst_is_recoverable() {
let f = Fixture::new();
@@ -623,24 +921,49 @@ fn supervisor_killed_during_output_burst_is_recoverable() {
}
#[test]
-fn dead_guard_with_live_session_refuses_duplicate_and_stop() {
+fn dead_guard_with_live_session_recovers_term_then_kill() {
let f = Fixture::new();
- let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]);
f.await_log("fixture-ready");
let state = f.state("worker");
- // SAFETY: these PIDs came from this fixture's ready receipt only.
+ // SAFETY: this fixture owns the captured live guard incarnation.
unsafe {
libc::kill(pid(&state, "guardPid"), libc::SIGKILL);
}
assert!(wait(&mut caller, TIMEOUT).success());
refused(&f, "run", f.launch(vec![f.process("worker", "hold")]));
- let (mut stop, _) = f.request("stop", &f.control(TOKEN));
- assert!(!wait(&mut stop, TIMEOUT).success());
- // The guard cannot prove ownership after its own death. The test itself
- // retains direct launch provenance and must clean up its authored child.
- unsafe {
- libc::kill(pid(&state, "commandPid"), libc::SIGKILL);
- }
+ let (mut status, observed) = f.request("status", &f.control(TOKEN));
+ assert!(wait(&mut status, TIMEOUT).success());
+ assert_eq!(observed["liveProcessNames"], json!([]));
+ assert_eq!(observed["unknownProcessNames"], json!(["worker"]));
+ let start = Instant::now();
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert!(wait(&mut stop, TIMEOUT).success());
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(start.elapsed() >= Duration::from_secs(5));
+ assert_eq!(receipt["liveProcessNames"], json!([]));
+}
+
+#[test]
+fn guard_dying_during_stop_gets_a_fresh_authored_term_grace() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]);
+ f.await_log("fixture-ready");
+ let guard = pid(&f.state("worker"), "guardPid");
+ let birth = identity(guard).unwrap();
+ // Suspend this owned guard so its pending TERM cannot reach the command.
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
+ let mut stop = command(&f.root, "stop").spawn().unwrap();
+ send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ thread::sleep(Duration::from_millis(150));
+ assert!(identity(guard).as_ref() == Some(&birth));
+ let killed_at = Instant::now();
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
+ let receipt = receive(stop.stdout.as_mut().unwrap());
+ assert!(wait(&mut stop, TIMEOUT).success());
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(killed_at.elapsed() >= Duration::from_secs(5));
+ assert!(wait(&mut caller, TIMEOUT).success());
}
#[test]
@@ -995,7 +1318,33 @@ fn fixture_process() {
libc::_exit(0);
}
}
- thread::sleep(Duration::from_millis(100));
+ eventually(|| {
+ #[cfg(target_os = "linux")]
+ let zombie = fs::read_to_string(format!("/proc/{child}/stat"))
+ .ok()
+ .is_some_and(|s| {
+ s.rsplit_once(") ")
+ .is_some_and(|(_, tail)| tail.starts_with("Z "))
+ });
+ #[cfg(target_os = "macos")]
+ let zombie = {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ let size = unsafe {
+ libc::proc_pidinfo(
+ child,
+ libc::PROC_PIDTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ size as usize == std::mem::size_of::()
+ && unsafe { info.assume_init().pbi_status == libc::SZOMB }
+ };
+ zombie.then_some(())
+ });
+ println!("unreaped-zombie-confirmed {child}");
+ std::io::stdout().flush().unwrap();
std::process::exit(0);
}
#[cfg(target_os = "linux")]
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index 5416ba319..cdb52237d 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -1,7 +1,7 @@
# Lifecycle supervisor ownership model
`LifecycleSupervisor.tla` models one configured command name and at most two
-start attempts. The model precedes implementation. The first artifact is a
+start attempts. This is a finite control-flow abstraction, not a model of kernel observations or lock admission. The first artifact is a
standalone Rust binary exercised by a fake caller; the CLI remains unchanged.
| Action/property | Source transition | Regression boundary |
@@ -9,7 +9,7 @@ standalone Rust binary exercised by a fake caller; the CLI remains unchanged.
| `Reserve`, `Publish`, `Spawn`; `DurableBeforeLaunch` | Per-command flock, guard session creation, synced identity before argv spawn | `starts_and_records_identity`; refused publication/spawn cannot run a later command |
| `SupervisorDeath`; `GuardRetainsLock` | Resident guard retains flock and stdout/stderr readers after project supervisor loss | `supervisor_kill_then_recover_stop`; output continues after supervisor SIGKILL |
| `GuardDeath`, `Retry`; `NoDuplicate` | Retained session census refuses a retry even if separately killed guard released its flock | Guard-loss retry and identity refusal regression |
-| `Stop`, `Kill`; `NoForeignSignal` | Exact owner/boot/start identity before TERM and KILL; guard remains outside command groups | Wrong-owner/stale-identity refusal; TERM-ignore escalation |
+| `Stop`, `Kill`; `NoForeignSignal` | Model assumes identity remains valid across signaling; source performs native rechecks | Wrong-owner/stale-identity refusal; TERM-ignore escalation |
| `ProcessExit`, `Reap` | Session census ignores zombies; reap direct/adopted children; release lock after no live members | Zombie liveness; child changes process group within session |
| `StopUnknown` | Bounded confirmation can report uncertainty while live members remain | Honest unknown recovery outcome, never successful absence |
@@ -23,6 +23,15 @@ Checker: checksum-pinned TLA+ 1.7.4 artifact (TLC reports version 2.19), Java 17
two workers and a bounded heap. The maintained runner pins the positive count and
validates the negative invariant/action/witness. Apalache is not qualified here.
+`NoForeignSignal` is true by construction: both signal actions require
+`identity = "match"` and preserve that value in their assignment to `unsafeSignal`.
+It does not independently validate observation freshness, PID/session reuse, or the
+source's effect boundary. `GuardRetainsLock` similarly checks the abstract action
+assignments, not descriptor inheritance or actual flock ownership. `NoDuplicate`
+checks the selected live fence and its negative control in this finite abstraction;
+it does not prove lock/census correctness. The positive state count is retained
+because no model actions, variables or checker registry have changed in this slice.
+
Assumptions: flock serialization, atomic/synchronized publication, reliable native
identity and session census, and rechecking identity before effects. Signal delivery
does not imply process exit: `KillPending` can retain live processes, and
@@ -36,3 +45,17 @@ PID reuse between observation and signaling, hostile same-user mutation, and
commands escaping their session with `setsid`. Those remain native implementation
or explicitly unsupported boundaries. This finite model does not qualify a shipped
artifact, macOS behavior, CLI integration or the default runner.
+
+Native recovery added after this abstraction is deliberately not claimed as model
+coverage: dead-guard TERM/grace/KILL recovery, supervisor death before guard
+publication, inherited launch-admission lease, stop/exit inspection races, census
+errors and restricted views, high descriptor limits, EINTR retries, and idle
+poll scheduling. `GuardDeath` can reach `Orphaned`, but `Stop` requires `guard` and
+cannot recover it in this model. Native regressions include
+`dead_guard_with_live_session_recovers_term_then_kill`,
+`supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased`,
+`supervisor_killed_while_ready_receipt_is_blocked_recovers`, and
+`guard_crash_before_complete_launch_frame_starts_nothing`.
+They qualify only their exercised boundaries on the host running them.
+No observation/lock/census refinement or liveness proof is asserted here; the
+shared registry and model-coverage map remain owned by the integration task.
From 1a8d027706caf22aec3c6cf2dd5d4aabc444a206 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 23:12:53 -0500
Subject: [PATCH 07/45] docs(lifecycle): describe verified orphan recovery and
idle bounds
---
docs/diagrams/lifecycle-supervisor.mmd | 5 ++-
docs/diagrams/lifecycle-supervisor.svg | 4 +--
docs/lifecycle-supervisor.md | 40 +++++++++++++++++++----
packages/lifecycle-supervisor/src/main.rs | 6 ++--
4 files changed, 42 insertions(+), 13 deletions(-)
diff --git a/docs/diagrams/lifecycle-supervisor.mmd b/docs/diagrams/lifecycle-supervisor.mmd
index 06cb2bd80..5f7ffd0a9 100644
--- a/docs/diagrams/lifecycle-supervisor.mmd
+++ b/docs/diagrams/lifecycle-supervisor.mmd
@@ -12,6 +12,8 @@ stateDiagram-v2
Running --> Orphaned : GuardDeath [members live]
Recoverable --> Orphaned : GuardDeath [members live]
Recoverable --> Stopping : Stop [identity matches]
+ Orphaned --> Stopping : RecoverStop [boot and session members verified]
+ Orphaned --> StopUnknown : RecoverStop [identity unconfirmed]
Running --> Stopping : Stop [identity matches]
Running --> Exited : Reap [no live members]
Stopping --> Exited : Reap [no live members]
@@ -22,6 +24,7 @@ stateDiagram-v2
INV identity precedes command
end note
note right of Orphaned
- Unknown identity refuses stop
+ Unknown health retains ownership
+ Unverified identity refuses signals
Live session refuses duplicate
end note
diff --git a/docs/diagrams/lifecycle-supervisor.svg b/docs/diagrams/lifecycle-supervisor.svg
index 32561f1ad..674e5047f 100644
--- a/docs/diagrams/lifecycle-supervisor.svg
+++ b/docs/diagrams/lifecycle-supervisor.svg
@@ -1,4 +1,4 @@
-
\ No newline at end of file
+]]>Lifecycle supervisor · ownership and recoveryReserve [locks acquired]Spawn [identity durable]StartFailureSupervisorDeathRetry [duplicate refused]GuardDeath [members live]GuardDeath [members live]Stop [identity matches]RecoverStop [boot andsession members verified]RecoverStop [identityunconfirmed]Stop [identity matches]Reap [no live members]Reap [no live members]Kill [grace elapsed]Reap [no live members]StopUnknown [members live]AbsentRunningExitedRecoverableStoppingStopUnknownKillPendingStartingOrphanedINV identity precedes commandUnknown health retains ownershipUnverified identity refuses signalsLive session refuses duplicate
\ No newline at end of file
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index f9158d940..709620979 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -35,7 +35,8 @@ directly. A shell-authored command can explicitly use `sh -c`; the supervisor ad
no shell wrapper. The full environment is explicit, with no ambient inheritance.
`stop` and `status` accept `{version: 1, root, ownerToken}`. Read-only `status`
-returns persisted receipts; a receipt is not a health check. It creates no files,
+returns saved phases plus kernel-verified `liveProcessNames` and
+`unknownProcessNames`; a saved phase alone is not a health check. It creates no files,
processes or ownership. Stop
requires matching ownership and a live identity before sending a signal. Unknown
identity is a refusal, never proof that a process is absent. A different boot or a
@@ -97,7 +98,7 @@ not claims that a dead or paused guard rewrote its receipt.
| Stopping/KillPending | Confirmation deadline expires with live members | Retain identity and command lock; report uncertainty | StopUnknown | `unresponsive_guard_stop_remains_unknown` |
| Running | Project supervisor dies | Guard and durable identity remain | Running/recoverable | `supervisor_kill_then_recover_stop` |
| Recoverable | Another start sees held command lock or live old identity | Refuse duplicate; preserve original | Recoverable | `supervisor_kill_then_recover_stop` |
-| Running/recoverable | Guard itself is separately killed with live session members | Retain recorded session, refuse duplicate and unsafe stop | Orphaned | `dead_guard_with_live_session_refuses_duplicate_and_stop` |
+| Running/recoverable | Guard itself is separately killed with live session members | Health unknown; refuse duplicate, permit verified session recovery stop | Orphaned | `dead_guard_with_live_session_recovers_term_then_kill` |
| Any live recorded | Owner differs or identity is unknown | Refuse; signal nothing | Unchanged | `foreign_owner_cannot_stop` |
| Any recorded | Boot differs or leader PID is reused | Treat old session as absent; signal no replacement | Unchanged | `stale_reused_session_leader_does_not_block_fresh_launch` |
| Running | Command exits; only zombies remain | Reap direct child; zombies do not hold liveness | Exited | `zombies_do_not_hold_liveness` |
@@ -108,11 +109,13 @@ A resident guard closes the publication gap: command ownership is durable before
spawn, and the group leader survives the project supervisor's death. The guard
keeps the command lock until its session has no live members, so a restarted project
supervisor cannot launch a second copy. If a guard itself is separately killed
-while its command survives, identity becomes unknown; recovery refuses unsafe
-signals and duplicate launch. This is a distinct failure from killing the project
-supervisor and remains visible to callers rather than being reported as a
-successful stop. Launch and stop share a bounded lock so stop cannot race past a
-partially launched plan.
+while its command survives, health becomes unknown and duplicate launch is
+refused. Stop can recover the recorded session after verifying the boot,
+leader identity and each remaining member's UID, birth and session. Recovery
+delivers TERM individually, grants a fresh five-second grace, then KILL and
+confirms absence. Unverified identity never authorizes signals or a successful
+stop. Launch and stop share a bounded lock; a startup descriptor lease keeps
+admission locked through guard publication even if the project owner dies.
The table models process death and retry, not filesystem hardware failure, hostile
same-user mutation, commands that deliberately create new sessions, or power-loss
@@ -167,3 +170,26 @@ Default migration removes the Python/FIFO wrapper in
POSIX `sh -c` intentionally. Interactive tmux environment delivery stays separate.
Zellij removal, its breaking-change notice, and the workspace-only `MuxBackend`
contract belong to the third unit.
+
+## Resource and qualification boundaries
+
+A live direct child proves its guard is needed without a full-system census.
+Guards wait for output or SIGCHLD; once only descendants remain, a 500 ms
+fallback checks the session. Stop uses the short retirement tick. Startup closes
+actual open descriptors rather than iterating the open-file limit. A known
+restricted process view is refused before authored code; silently hidden
+same-user processes remain unsupported.
+
+Linux debug-build measurements with one quiet guard and owner recorded less than
+10 ms CPU accounting resolution per five-second sample and two voluntary context
+switches per second, across three samples. These are neither hardware wakeup
+counts nor macOS measurements. The host's hard descriptor limit was 524,288;
+the exact 1,048,576 test requires a host that permits that limit.
+Darwin compilation does not qualify execution, group-signal EPERM recovery or
+idle cost on a real Mac.
+
+The 129-state TLA+ artifact is an abstract phase/ownership exploration. It does
+not model kernel census observations, real flock behavior, orphan recovery,
+startup descriptor leases or signal errors. Some safety predicates hold by
+construction. Native regressions, rather than that model, are the evidence for
+those boundaries; see the model coverage document for the reduced claims.
diff --git a/packages/lifecycle-supervisor/src/main.rs b/packages/lifecycle-supervisor/src/main.rs
index 41a8850f9..8799f6ed9 100644
--- a/packages/lifecycle-supervisor/src/main.rs
+++ b/packages/lifecycle-supervisor/src/main.rs
@@ -13,14 +13,14 @@
//! | starting/running | project owner dies | same | resident guard retains lock and identity |
//! | any live | duplicate command / lock busy | refused | no second execution |
//! | any | token or identity mismatch | refused | no signal |
-//! | any | guard dead and authored session live | orphaned | duplicate and stop refused |
+//! | any | guard dead and authored session live | orphaned | duplicate refused; verified session stop permitted |
//!
//! A resident guard is the session leader in its own process group. Authored
//! commands receive a different group in that session. It publishes its kernel
//! identity before starting authored code, and remains present while any live
//! session member remains. A project-owner crash therefore leaves a verifiable
-//! leader. Killing that guard separately is an unknown-ownership outcome and
-//! fails closed when a live group remains.
+//! leader. Killing that guard separately makes health unknown; stop can recover
+//! the recorded session only after boot, identity and member-incarnation checks.
mod os;
mod protocol;
From e8511bd925dc4918885b2a54bed034b67b7e4dcc Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 23:28:27 -0500
Subject: [PATCH 08/45] fix(lifecycle): cancel delayed launches after
generation retirement
---
docs/lifecycle-supervisor.md | 7 +++
packages/lifecycle-supervisor/src/storage.rs | 21 +++++++
.../lifecycle-supervisor/src/supervisor.rs | 14 ++++-
.../lifecycle-supervisor/tests/fake_caller.rs | 59 +++++++++++++++----
.../tla/lifecycle-supervisor/COVERAGE.md | 8 +++
5 files changed, 96 insertions(+), 13 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 709620979..56ea68d7f 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -193,3 +193,10 @@ not model kernel census observations, real flock behavior, orphan recovery,
startup descriptor leases or signal errors. Some safety predicates hold by
construction. Native regressions, rather than that model, are the evidence for
those boundaries; see the model coverage document for the reduced claims.
+
+Stop durably retires its owner generation before confirming teardown, even if no
+command receipt exists. A delayed run checks that fence under the admission lock
+before spawning. New generations must use fresh owner tokens; markers persist in
+the private root and contain no command environment. Retired-but-live work remains
+unconfirmed. Linux guard spawning uses `/proc/self/exe`, preserving an executable
+reference even after atomic helper replacement.
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index 6c28444fa..1fb23da23 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -37,6 +37,27 @@ pub struct Root {
}
impl Root {
+ fn retirement_key(token: &str) -> String {
+ // Stable non-authentication index. A collision can only refuse launch.
+ let hash = token.bytes().fold(0xcbf29ce484222325_u64, |hash, byte| {
+ (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3)
+ });
+ format!(".retired-{hash:016x}")
+ }
+
+ pub fn retire(&self, token: &str) -> Result<()> {
+ self.file(&Self::retirement_key(token), libc::O_WRONLY | libc::O_CREAT)?
+ .sync_all()?;
+ self.directory.sync_all()
+ }
+
+ pub fn assert_not_retired(&self, token: &str) -> Result<()> {
+ match self.file(&Self::retirement_key(token), libc::O_RDONLY) {
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ _ => Err(refused()),
+ }
+ }
+
pub fn open(path: &Path) -> Result {
let directory = OpenOptions::new()
.read(true)
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index b8d1a3b8f..837f3ba68 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -86,7 +86,14 @@ pub fn main() -> Result<()> {
}
}
let result = if args[0] == "stop" {
+ root.retire(&request.owner_token)?;
stop(&states)
+ } else if root.assert_not_retired(&request.owner_token).is_err()
+ && states
+ .iter()
+ .any(|state| !matches!(os::observe_state(state), os::Observation::Absent))
+ {
+ Err(refused())
} else {
Ok(())
};
@@ -129,6 +136,7 @@ fn run(request: Launch) -> Result<()> {
let _instance_lock = instance_lock(&root)?;
os::census_preflight()?;
let launch_lock = root.lock_wait(".stop.lock")?;
+ root.assert_not_retired(&request.owner_token)?;
// Validate every old claim before starting any new command. A guard survives
// project-owner death; its command lock also closes concurrent restart races.
for state in root.states()? {
@@ -145,7 +153,11 @@ fn run(request: Launch) -> Result<()> {
if os::STOP.load(Ordering::Relaxed) {
return Err(refused());
}
- let mut command = Command::new(std::env::current_exe()?);
+ #[cfg(target_os = "linux")]
+ let executable = std::path::PathBuf::from("/proc/self/exe");
+ #[cfg(not(target_os = "linux"))]
+ let executable = std::env::current_exe()?;
+ let mut command = Command::new(executable);
command
.arg("guard")
.env_clear()
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index b6fd702e0..0a1353a52 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -78,6 +78,17 @@ impl Fixture {
self.request("run", &self.launch(processes))
}
+ fn start_generation(&self, processes: Vec, token: &str) -> (Child, Value) {
+ let mut plan = self.launch(processes);
+ plan["ownerToken"] = json!(token);
+ self.request("run", &plan)
+ }
+
+ fn stop_generation(&self, token: &str) {
+ let (mut stop, _) = self.request("stop", &self.control(token));
+ assert!(wait(&mut stop, TIMEOUT).success());
+ }
+
fn stop(&self) {
let mut child = command(&self.root, "stop").spawn().unwrap();
send(child.stdin.as_mut().unwrap(), &self.control(TOKEN));
@@ -561,12 +572,13 @@ fn child_descriptor_allowlist() {
fn repeated_immediate_restart_waits_for_instance_lock_release() {
let f = Fixture::new();
let mut previous = None;
- for _ in 0..8 {
- let (caller, _) = f.start(vec![f.process("worker", "hold")]);
+ for generation in 0..8 {
+ let token = format!("synthetic-generation-{generation}");
+ let (caller, _) = f.start_generation(vec![f.process("worker", "hold")], &token);
if let Some(mut old) = previous.take() {
assert!(wait(&mut old, TIMEOUT).success());
}
- f.stop();
+ f.stop_generation(&token);
previous = Some(caller);
}
assert!(wait(previous.as_mut().unwrap(), TIMEOUT).success());
@@ -680,9 +692,10 @@ fn idle_guard_cpu_and_wakeup_measurement() {
#[test]
fn natural_exit_during_stop_admission_is_successful_absence() {
let f = Fixture::new();
- for _ in 0..8 {
- let (mut caller, _) = f.start(vec![f.process("worker", "unicode")]);
- let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ for generation in 0..8 {
+ let token = format!("synthetic-generation-{generation}");
+ let (mut caller, _) = f.start_generation(vec![f.process("worker", "unicode")], &token);
+ let (mut stop, receipt) = f.request("stop", &f.control(&token));
assert!(wait(&mut stop, TIMEOUT).success());
assert_eq!(receipt["outcome"], "stopped");
assert!(wait(&mut caller, TIMEOUT).success());
@@ -838,8 +851,11 @@ fn supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased() {
assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
assert!(wait(&mut stop, TIMEOUT).success());
drop(lock);
- let (mut next, _) = f.start(vec![f.process("worker", "hold")]);
- f.stop();
+ let (mut next, _) = f.start_generation(
+ vec![f.process("worker", "hold")],
+ "synthetic-next-generation",
+ );
+ f.stop_generation("synthetic-next-generation");
assert!(wait(&mut next, TIMEOUT).success());
}
@@ -1044,8 +1060,11 @@ fn stale_reused_session_leader_does_not_block_fresh_launch() {
serde_json::to_vec(&state).unwrap(),
)
.unwrap();
- let (mut next, _) = f.start(vec![f.process("worker", "hold")]);
- f.stop();
+ let (mut next, _) = f.start_generation(
+ vec![f.process("worker", "hold")],
+ "synthetic-next-generation",
+ );
+ f.stop_generation("synthetic-next-generation");
assert!(wait(&mut next, TIMEOUT).success());
assert!(
foreign.0.try_wait().unwrap().is_none(),
@@ -1137,8 +1156,11 @@ fn case_only_rename_accepts_terminal_receipt_with_previous_spelling() {
// Also emulate the case-insensitive lookup on Linux: the payload retains
// its original spelling while the lookup uses the new command name.
fs::rename(f.root.join("Worker.json"), f.root.join("worker.json")).unwrap();
- let (mut next, _) = f.start(vec![f.process("worker", "hold")]);
- f.stop();
+ let (mut next, _) = f.start_generation(
+ vec![f.process("worker", "hold")],
+ "synthetic-next-generation",
+ );
+ f.stop_generation("synthetic-next-generation");
assert!(wait(&mut next, TIMEOUT).success());
}
@@ -1446,3 +1468,16 @@ fn fixture_process() {
}
use std::os::unix::fs::DirBuilderExt;
+
+#[test]
+fn completed_stop_fences_a_run_whose_frame_arrives_after_retirement() {
+ let f = Fixture::new();
+ let mut delayed = command(&f.root, "run").spawn().unwrap();
+ f.stop();
+ send(
+ delayed.stdin.as_mut().unwrap(),
+ &f.launch(vec![f.process("late", "hold")]),
+ );
+ assert!(!wait(&mut delayed, TIMEOUT).success());
+ assert!(!f.root.join("late.json").exists());
+}
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index cdb52237d..217adea6e 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -59,3 +59,11 @@ cannot recover it in this model. Native regressions include
They qualify only their exercised boundaries on the host running them.
No observation/lock/census refinement or liveness proof is asserted here; the
shared registry and model-coverage map remain owned by the integration task.
+
+Durable owner-generation retirement is an additional implementation fence, not a
+transition checked by this finite model. Stop records cancellation even before a
+late launch frame arrives; run checks it while holding admission. New generations
+use fresh tokens. The regression
+`completed_stop_fences_a_run_whose_frame_arrives_after_retirement` proves the
+late frame creates no command receipt. Linux executable replacement is qualified
+by the incremental integration test in the opt-in slice, not by this model.
From 047175a6c55e2f330bbe0a9dd87edfc19c905333 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 23:34:09 -0500
Subject: [PATCH 09/45] test(lifecycle): observe unreaped exit without Darwin
session identity
---
packages/lifecycle-supervisor/src/os.rs | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index df436fef5..5bba81ea2 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -832,14 +832,23 @@ mod tests {
libc::_exit(0);
}
}
- let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
- loop {
- if inspect(pid).unwrap().is_some_and(|p| p.zombie) {
- break;
- }
- assert!(std::time::Instant::now() < deadline);
- std::thread::sleep(std::time::Duration::from_millis(10));
- }
+ // Darwin getsid may return ESRCH for zombies, so inspect intentionally
+ // reports absence. Observe exit without reaping through the wait API
+ // instead of requiring a live-process identity for an exited child.
+ let mut event = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: this is our direct child, event is a writable kernel buffer,
+ // and WNOWAIT deliberately retains the child for the assertions below.
+ assert_eq!(
+ unsafe {
+ libc::waitid(
+ libc::P_PID,
+ pid as libc::id_t,
+ event.as_mut_ptr(),
+ libc::WEXITED | libc::WNOWAIT,
+ )
+ },
+ 0
+ );
assert!(!live_group(pid, None).unwrap());
assert!(
group_signal_result(pid, io::Error::from_raw_os_error(libc::EPERM), census).is_ok()
From e2130cfd870dc7e43ee0c669feb3a634033bc568 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 23:39:14 -0500
Subject: [PATCH 10/45] fix(lifecycle): verify Linux process UID independently
of proc ownership
---
docs/lifecycle-supervisor.md | 4 ++
packages/lifecycle-supervisor/src/os.rs | 37 +++++++++++++-----
.../lifecycle-supervisor/tests/fake_caller.rs | 38 +++++++++++++++++++
3 files changed, 69 insertions(+), 10 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 56ea68d7f..05d3a7b83 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -200,3 +200,7 @@ before spawning. New generations must use fresh owner tokens; markers persist in
the private root and contain no command environment. Retired-but-live work remains
unconfirmed. Linux guard spawning uses `/proc/self/exe`, preserving an executable
reference even after atomic helper replacement.
+
+Linux ownership checks read the process's effective UID from `/proc//status`.
+They do not infer UID from proc-file ownership, which changes when a command
+disables dumpability. Such a command remains recoverable after guard loss.
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 5bba81ea2..e58393e21 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -65,7 +65,6 @@ pub struct Identity {
#[cfg(target_os = "linux")]
pub fn inspect(pid: i32) -> Result> {
- use std::os::unix::fs::MetadataExt;
let path = format!("/proc/{pid}/stat");
let bytes = match fs::read(&path) {
Ok(value) => value,
@@ -86,21 +85,40 @@ pub fn inspect(pid: i32) -> Result> {
.get(3)
.and_then(|v| v.parse().ok())
.ok_or_else(refused)?;
- let metadata = match fs::metadata(&path) {
- Ok(value) => value,
- Err(error) if absent_error(&error) => return Ok(None),
- Err(error) => return Err(error),
+ let uid = match linux_uid(pid)? {
+ Some(uid) => uid,
+ None => return Ok(None),
};
Ok(Some(Identity {
pid,
group,
session,
start,
- uid: metadata.uid(),
+ uid,
zombie: fields.first() == Some(&"Z") || fields.first() == Some(&"X"),
}))
}
+#[cfg(target_os = "linux")]
+fn linux_uid(pid: i32) -> Result> {
+ let status = match fs::read(format!("/proc/{pid}/status")) {
+ Ok(status) => status,
+ Err(error) if absent_error(&error) => return Ok(None),
+ Err(error) => return Err(error),
+ };
+ let line = status
+ .split(|byte| *byte == b'\n')
+ .find_map(|line| line.strip_prefix(b"Uid:"))
+ .ok_or_else(refused)?;
+ let uid = std::str::from_utf8(line)
+ .map_err(|_| refused())?
+ .split_ascii_whitespace()
+ .nth(1)
+ .and_then(|uid| uid.parse().ok())
+ .ok_or_else(refused)?;
+ Ok(Some(uid))
+}
+
#[cfg(target_os = "linux")]
fn proc_stat_tail(bytes: &[u8]) -> Result<&str> {
// comm is arbitrary bytes and may itself contain parentheses. Only the
@@ -124,11 +142,10 @@ pub fn census() -> Result> {
Err(error)
if matches!(error.raw_os_error(), Some(libc::EPERM) | Some(libc::EACCES)) =>
{
- use std::os::unix::fs::MetadataExt;
- match entry.metadata() {
+ match linux_uid(pid)? {
// SAFETY: geteuid is an observation with no preconditions.
- Ok(metadata) if metadata.uid() != unsafe { libc::geteuid() } => (),
- Err(error) if absent_error(&error) => (),
+ Some(uid) if uid != unsafe { libc::geteuid() } => (),
+ None => (),
_ => return Err(error),
}
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 0a1353a52..aad2946f7 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1434,6 +1434,12 @@ fn fixture_process() {
out.flush().unwrap();
std::process::exit(0);
}
+ #[cfg(target_os = "linux")]
+ "non-dumpable" => {
+ // SAFETY: affects only this owned fixture's dumpability, not its UID.
+ assert_eq!(unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0) }, 0);
+ println!("non-dumpable-ready");
+ }
"signal-default" => {
let mut disposition: libc::sigaction = unsafe { std::mem::zeroed() };
assert_eq!(
@@ -1481,3 +1487,35 @@ fn completed_stop_fences_a_run_whose_frame_arrives_after_retirement() {
assert!(!wait(&mut delayed, TIMEOUT).success());
assert!(!f.root.join("late.json").exists());
}
+
+#[cfg(target_os = "linux")]
+#[test]
+fn non_dumpable_command_keeps_its_process_uid_for_orphan_recovery() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "non-dumpable")]);
+ f.await_log("non-dumpable-ready");
+ let state = f.state("worker");
+ let command = pid(&state, "commandPid");
+ // On an unprivileged runner the inode owner changes to root, while the
+ // process UID stays ours. The recovery path must use the latter.
+ if unsafe { libc::geteuid() } != 0 {
+ assert_eq!(
+ fs::metadata(format!("/proc/{command}/stat")).unwrap().uid(),
+ 0
+ );
+ }
+ // SAFETY: this fixture owns the still-running guard recorded at launch.
+ assert_eq!(
+ unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
+ 0
+ );
+ f.stop();
+ let _ = wait(&mut caller, TIMEOUT);
+ let live = fs::read_to_string(format!("/proc/{command}/stat"))
+ .ok()
+ .is_some_and(|stat| {
+ stat.rsplit_once(") ")
+ .is_some_and(|(_, fields)| !fields.starts_with("Z ") && !fields.starts_with("X "))
+ });
+ assert!(!live);
+}
From e203585a75c82362f6b4db13bf7707f7d1b4a37e Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Fri, 13 Nov 2026 23:45:16 -0500
Subject: [PATCH 11/45] test(lifecycle): confirm fixture zombies with
non-reaping wait
---
.../lifecycle-supervisor/tests/fake_caller.rs | 39 +++++++------------
1 file changed, 14 insertions(+), 25 deletions(-)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index aad2946f7..566eb2c38 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1340,31 +1340,20 @@ fn fixture_process() {
libc::_exit(0);
}
}
- eventually(|| {
- #[cfg(target_os = "linux")]
- let zombie = fs::read_to_string(format!("/proc/{child}/stat"))
- .ok()
- .is_some_and(|s| {
- s.rsplit_once(") ")
- .is_some_and(|(_, tail)| tail.starts_with("Z "))
- });
- #[cfg(target_os = "macos")]
- let zombie = {
- let mut info = std::mem::MaybeUninit::::zeroed();
- let size = unsafe {
- libc::proc_pidinfo(
- child,
- libc::PROC_PIDTBSDINFO,
- 0,
- info.as_mut_ptr().cast(),
- std::mem::size_of::() as i32,
- )
- };
- size as usize == std::mem::size_of::()
- && unsafe { info.assume_init().pbi_status == libc::SZOMB }
- };
- zombie.then_some(())
- });
+ let mut event = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: observe only this fixture's direct child into a writable
+ // buffer. WNOWAIT confirms exit while deliberately retaining it.
+ assert_eq!(
+ unsafe {
+ libc::waitid(
+ libc::P_PID,
+ child as libc::id_t,
+ event.as_mut_ptr(),
+ libc::WEXITED | libc::WNOWAIT,
+ )
+ },
+ 0
+ );
println!("unreaped-zombie-confirmed {child}");
std::io::stdout().flush().unwrap();
std::process::exit(0);
From d60d4e2de23e819a92e3889692b48a4365b2e560 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 03:25:26 -0500
Subject: [PATCH 12/45] fix(supervisor): fence orphan recovery against session
ID reuse
---
docs/diagrams/lifecycle-supervisor.mmd | 4 +-
docs/diagrams/lifecycle-supervisor.svg | 4 +-
docs/lifecycle-supervisor.md | 37 ++-
docs/reference/v5-model-coverage.md | 4 +-
packages/lifecycle-supervisor/README.md | 10 +
packages/lifecycle-supervisor/src/os.rs | 37 ++-
packages/lifecycle-supervisor/src/storage.rs | 2 +
.../lifecycle-supervisor/src/supervisor.rs | 2 +
.../lifecycle-supervisor/tests/fake_caller.rs | 247 +++++++++++++++---
scripts/lib/tla-runtime-models.ts | 15 +-
.../tla/lifecycle-supervisor/COVERAGE.md | 26 +-
.../LifecycleSupervisor.tla | 24 +-
.../tla/lifecycle-supervisor/negative.cfg | 1 +
.../tla/lifecycle-supervisor/positive.cfg | 1 +
.../tla/lifecycle-supervisor/sid-reuse.cfg | 5 +
15 files changed, 357 insertions(+), 62 deletions(-)
create mode 100644 tests/models/tla/lifecycle-supervisor/sid-reuse.cfg
diff --git a/docs/diagrams/lifecycle-supervisor.mmd b/docs/diagrams/lifecycle-supervisor.mmd
index 5f7ffd0a9..1021a15f0 100644
--- a/docs/diagrams/lifecycle-supervisor.mmd
+++ b/docs/diagrams/lifecycle-supervisor.mmd
@@ -12,7 +12,7 @@ stateDiagram-v2
Running --> Orphaned : GuardDeath [members live]
Recoverable --> Orphaned : GuardDeath [members live]
Recoverable --> Stopping : Stop [identity matches]
- Orphaned --> Stopping : RecoverStop [boot and session members verified]
+ Orphaned --> Stopping : RecoverStop [boot, old birth witness and members verified]
Orphaned --> StopUnknown : RecoverStop [identity unconfirmed]
Running --> Stopping : Stop [identity matches]
Running --> Exited : Reap [no live members]
@@ -25,6 +25,6 @@ stateDiagram-v2
end note
note right of Orphaned
Unknown health retains ownership
- Unverified identity refuses signals
+ No old birth witness refuses signals
Live session refuses duplicate
end note
diff --git a/docs/diagrams/lifecycle-supervisor.svg b/docs/diagrams/lifecycle-supervisor.svg
index 674e5047f..18bff95da 100644
--- a/docs/diagrams/lifecycle-supervisor.svg
+++ b/docs/diagrams/lifecycle-supervisor.svg
@@ -1,4 +1,4 @@
-
\ No newline at end of file
+]]>Lifecycle supervisor · ownership and recoveryReserve [locks acquired]Spawn [identity durable]StartFailureSupervisorDeathRetry [duplicate refused]GuardDeath [members live]GuardDeath [members live]Stop [identity matches]RecoverStop [boot, old birthwitness and members verified]RecoverStop [identityunconfirmed]Stop [identity matches]Reap [no live members]Reap [no live members]Kill [grace elapsed]Reap [no live members]StopUnknown [members live]AbsentRunningExitedRecoverableStoppingStopUnknownKillPendingStartingOrphanedINV identity precedes commandUnknown health retains ownershipNo old birth witness refuses signalsLive session refuses duplicate
\ No newline at end of file
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 05d3a7b83..90b5fde04 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -41,8 +41,12 @@ processes or ownership. Stop
requires matching ownership and a live identity before sending a signal. Unknown
identity is a refusal, never proof that a process is absent. A different boot or a
leader with a different start identity proves that the old session ended; it
-grants no authority over the replacement. Old terminal receipts are retained and
-ignored only after native absence is proved.
+grants no authority over the replacement. Guard-written terminal receipts are retained as proof that the old session ended.
+A missing leader plus a numeric SID is not ownership: orphan recovery requires
+a same-user surviving member born no later than the recorded command birth.
+Only newer members, or legacy receipts without that birth evidence, report
+unknown without signals. Descendants born after the command whose birth was
+recorded may therefore require manual recovery once that command exits.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
`observed`, `stopped`, or `unknown`). A stop whose confirmation times out emits
@@ -55,7 +59,7 @@ The root contains per-command state and lock files, an instance lock and
`lifecycle.log`. Directories are mode 0700; files are mode 0600. Symlinks,
hard-linked files, foreign owners and broad permissions are refused. State contains
name, ownership token, guard PID, process group, OS start identity, boot identity,
-UID, command PID when known, phase, and the direct command's exit code or signal
+UID, command PID and start identity when known, phase, and the direct command's exit code or signal
when observed. It contains no command or environment.
State is atomically published and synchronized before authored code can run.
The guard leads a new session in its own process group. The authored command
@@ -116,6 +120,12 @@ delivers TERM individually, grants a fresh five-second grace, then KILL and
confirms absence. Unverified identity never authorizes signals or a successful
stop. Launch and stop share a bounded lock; a startup descriptor lease keeps
admission locked through guard publication even if the project owner dies.
+The guard temporarily inherits this stop-lock lease on stderr (fd 2), then closes
+it by detaching stdio after publication. This descriptor is an internal ownership
+lease, not a diagnostic stream; authored commands receive a separate stderr pipe.
+On Darwin, processes of a different UID may be hidden by kernel inspection
+permissions. Such descendants are unsupported: a readable census cannot prove
+absence of a hidden different-UID member, and an ordinary owner cannot signal it.
The table models process death and retry, not filesystem hardware failure, hostile
same-user mutation, commands that deliberately create new sessions, or power-loss
@@ -188,8 +198,8 @@ the exact 1,048,576 test requires a host that permits that limit.
Darwin compilation does not qualify execution, group-signal EPERM recovery or
idle cost on a real Mac.
-The 129-state TLA+ artifact is an abstract phase/ownership exploration. It does
-not model kernel census observations, real flock behavior, orphan recovery,
+The 168-state TLA+ artifact is an abstract phase/ownership exploration. It models SID reuse and a required old-birth witness for orphan signaling. It does
+not model actual kernel census observations, real flock behavior, full orphan recovery,
startup descriptor leases or signal errors. Some safety predicates hold by
construction. Native regressions, rather than that model, are the evidence for
those boundaries; see the model coverage document for the reduced claims.
@@ -204,3 +214,20 @@ reference even after atomic helper replacement.
Linux ownership checks read the process's effective UID from `/proc//status`.
They do not infer UID from proc-file ownership, which changes when a command
disables dumpability. Such a command remains recoverable after guard loss.
+
+Restart policy and restart backoff belong to the CLI integration in #811, not
+this binary's command ownership loop. This crate does not restart commands.
+
+Logs are append-only and have no automatic rotation. Stop all generations and
+confirm native absence before archiving the private root/log and starting with
+a fresh root. Crash-left private .tmp files can be removed only while no writer
+holds the root. Retirement markers must not be pruned individually: they fence
+arbitrarily delayed frames. Retire the entire quiescent root only after all old
+callers and guards have exited. These files have no automatic size/count bound;
+long-running installations must arrange a maintenance window and offline rotation.
+
+Native crash tests currently cover prepublication command-lock blocking, incomplete
+guard launch frames, and blocked ready delivery. Deterministic tests at the two
+smaller instruction windows (starting-state save to spawn, and spawn to running-state
+save) remain unqualified; polling a receipt cannot reliably stop a process in either
+window. They require a dedicated synchronization seam, not a timing-only test.
diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md
index 4649fe217..5eb379bc7 100644
--- a/docs/reference/v5-model-coverage.md
+++ b/docs/reference/v5-model-coverage.md
@@ -36,7 +36,7 @@ must be attached to the eventual exact integration revision.
| Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification |
| --- | --- | --- | --- |
-| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown identities, arbitrary process exit and supervisor/guard loss; 129 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
+| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 168 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
| Interactive exec write-ahead admission, crash, EOF, replacement boot and retirement | interactive-session (459 states); no-replay, EOF-retirement, failed-inspect and wrong-boot negative controls | `interactive_session.rs` atomic intent/exec records, exact stopped-boot proof, known-undispatched withdrawal, restore-before-busy-retirement and run-scoped cleanup-report tests, and lock-free observation/revalidation tests; Engine effect fences; authored owner start and SCM_RIGHTS handoff | One session, two boot identities, two possible starts. Atomic durable publication and exact identity/absence proofs are assumptions; real fsync/power loss, real owner/Engine handoff and VM loss need Mac acceptance; receipt lock timing and terminal restoration are regression-covered, not modeled. Validated exact exec exit permits retirement; it makes no descendant cleanup claim. |
| Private interactive envelope, echo state and bytes still in flight on failure | interactive-envelope (15 states); restore-on-error negative control | HKEE2 launcher and host gate, Linux PTY seven-case matrix, bounded marker/ACK and coalesced-output tests | Two abstract private bytes, one handshake. PTY fixtures prove local termios ordering; Docker detach scanning, real guest wiring, same-user access and VM effects remain separate. Source/compiled Bun PTY and native-process fixtures cover lifetime/terminal boundaries, not model liveness. |
| Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. |
@@ -83,7 +83,7 @@ liveness/reachability controls have their own documented assumptions.
| Model | Positive distinct states | Configurations |
| --- | ---: | ---: |
-| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 129 | 2 |
+| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 168 | 2 |
| [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 |
| [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 |
| [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 |
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 03b909e49..80e23a774 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -79,3 +79,13 @@ alone with `--ignored --exact idle_guard_cpu_and_wakeup_measurement --nocapture`
The ignored `million_descriptor_limit_qualification` fixture requires a host hard
RLIMIT_NOFILE of at least 1048576; the ordinary raised-limit fixture uses the host's
available limit. A host refusing that raise does not qualify the million-fd case.
+
+Orphan recovery requires a surviving member whose kernel birth is no later than
+the recorded command birth. A reused numeric SID alone never authorizes a signal.
+Terminal receipts prove the old guard completed cleanup; uncertain legacy or
+newer-member-only sessions remain unknown without effects. See
+[resource and qualification boundaries](../../docs/lifecycle-supervisor.md#resource-and-qualification-boundaries)
+for offline rotation, Darwin different-UID limits and remaining crash-test gaps.
+The ignored idle measurement has a native Darwin variant using proc_pid_rusage
+over 60 seconds; its kernel idle/interrupt wakeups differ from Linux context switches.
+Descriptor-limit and functional prepublication lease tests run on both platforms.
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index e58393e21..a333cb369 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -327,20 +327,45 @@ pub fn absent_error(error: &io::Error) -> bool {
|| matches!(error.raw_os_error(), Some(libc::ESRCH) | Some(libc::ENOENT))
}
-/// A boot change or a reused leader PID proves the old session is gone. A
-/// missing leader does not: descendants retain its SID and require a census.
+/// Terminal receipts prove the guard finished its census. With no leader, a
+/// matching SID alone is insufficient: that number may belong to a later session.
+/// An observed member born by the recorded command birth anchors the old session.
+/// Missing legacy birth evidence or only newer members is unknown, never authority.
pub fn recorded_live(state: &crate::storage::State, excluding_guard: bool) -> Result {
- if state.boot_id != boot()? {
+ if matches!(state.phase.as_str(), "exited" | "killed" | "failed-start")
+ || state.boot_id != boot()?
+ {
return Ok(false);
}
- if let Some(identity) = inspect(state.guard_pid)?
+ let leader = inspect(state.guard_pid)?;
+ if let Some(identity) = &leader
&& (identity.start != state.guard_start_time
|| identity.uid != state.uid
|| identity.session != state.guard_pid)
{
return Ok(false);
}
- live_group(state.guard_pid, excluding_guard.then_some(state.guard_pid))
+ let members: Vec<_> = census()?
+ .into_iter()
+ .filter(|p| p.session == state.guard_pid && !p.zombie)
+ .collect();
+ if members.is_empty() {
+ return Ok(false);
+ }
+ if leader.is_none_or(|p| p.zombie)
+ && !members.iter().any(|p| {
+ p.uid == state.uid
+ && p.start >= state.guard_start_time
+ && state
+ .command_start_time
+ .is_some_and(|birth| p.start <= birth)
+ })
+ {
+ return Err(refused());
+ }
+ Ok(members
+ .iter()
+ .any(|p| !excluding_guard || p.pid != state.guard_pid))
}
pub enum Observation {
@@ -353,7 +378,7 @@ pub enum Observation {
/// with remaining session members is unknown, never healthy or absent.
pub fn observe_state(state: &crate::storage::State) -> Observation {
let observed = (|| -> Result {
- if state.boot_id != boot()? {
+ if !recorded_live(state, true)? {
return Ok(Observation::Absent);
}
let leader = inspect(state.guard_pid)?;
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index 1fb23da23..8a17d2514 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -22,6 +22,8 @@ pub struct State {
pub boot_id: String,
pub uid: u32,
pub command_pid: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub command_start_time: Option,
pub phase: String,
#[serde(default)]
pub exit_code: Option,
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 837f3ba68..37859b632 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -306,6 +306,7 @@ fn guard(request: GuardLaunch) -> Result<()> {
boot_id: os::boot()?,
uid: identity.uid,
command_pid: None,
+ command_start_time: None,
phase: "starting".into(),
exit_code: None,
signal: None,
@@ -362,6 +363,7 @@ fn guard_command(
}
};
state.command_pid = Some(child.id() as i32);
+ state.command_start_time = os::inspect(child.id() as i32)?.map(|identity| identity.start);
state.phase = "running".into();
root.save(state)?;
let mut stdout = child.stdout.take().ok_or_else(refused)?;
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 566eb2c38..7e171cac3 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -226,6 +226,35 @@ fn census() -> Vec {
.collect()
}
+fn parent_pid(pid: i32) -> Option {
+ #[cfg(target_os = "linux")]
+ {
+ let text = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
+ text.rsplit_once(") ")?
+ .1
+ .split_whitespace()
+ .nth(1)?
+ .parse()
+ .ok()
+ }
+ #[cfg(target_os = "macos")]
+ {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: live writable buffer with the exact native struct size.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ (size as usize == std::mem::size_of::())
+ .then(|| unsafe { info.assume_init() }.pbi_ppid as i32)
+ }
+}
+
fn command(root: &Path, action: &str) -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_hack-lifecycle-supervisor"));
command
@@ -413,6 +442,10 @@ fn term_ignoring_command_is_killed_after_grace() {
start.elapsed() >= Duration::from_secs(5),
"TERM grace was shortened"
);
+ assert!(
+ start.elapsed() < Duration::from_secs(7),
+ "stop exceeded its confirmation budget"
+ );
assert!(wait(&mut caller, TIMEOUT).success());
}
@@ -584,7 +617,6 @@ fn repeated_immediate_restart_waits_for_instance_lock_release() {
assert!(wait(previous.as_mut().unwrap(), TIMEOUT).success());
}
-#[cfg(target_os = "linux")]
#[test]
fn raised_descriptor_limit_does_not_delay_startup() {
// Exercise the largest supported soft limit without changing this host's
@@ -597,14 +629,12 @@ fn raised_descriptor_limit_does_not_delay_startup() {
descriptor_limit_startup(limit.rlim_max.min(1_048_576));
}
-#[cfg(target_os = "linux")]
#[test]
#[ignore = "qualification requires a hard RLIMIT_NOFILE of at least 1048576"]
fn million_descriptor_limit_qualification() {
descriptor_limit_startup(1_048_576);
}
-#[cfg(target_os = "linux")]
fn descriptor_limit_startup(descriptors: libc::rlim_t) {
use std::os::unix::process::CommandExt;
let f = Fixture::new();
@@ -751,6 +781,47 @@ fn changed_process_group_remains_in_owned_session() {
assert!(wait(&mut caller, TIMEOUT).success());
}
+#[cfg(target_os = "macos")]
+#[test]
+#[ignore = "60-second native idle CPU and kernel wakeup measurement; run alone"]
+fn idle_guard_cpu_and_wakeup_measurement() {
+ fn sample(pid: i32) -> libc::rusage_info_v2 {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: proc_pid_rusage's opaque output points at the complete V2
+ // struct for the requested flavor, live and writable for this call.
+ assert_eq!(
+ unsafe { libc::proc_pid_rusage(pid, libc::RUSAGE_INFO_V2, info.as_mut_ptr().cast()) },
+ 0
+ );
+ unsafe { info.assume_init() }
+ }
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let guard = pid(&f.state("worker"), "guardPid");
+ let before = sample(guard);
+ let parent_before = sample(caller.id() as i32);
+ let start = Instant::now();
+ thread::sleep(Duration::from_secs(60));
+ let after = sample(guard);
+ let parent_after = sample(caller.id() as i32);
+ for (label, before, after) in [
+ ("guard", before, after),
+ ("owner", parent_before, parent_after),
+ ] {
+ println!(
+ "{label}: wall_s={:.3} cpu_ns={} idle_wakeups={} interrupt_wakeups={}",
+ start.elapsed().as_secs_f64(),
+ (after.ri_user_time - before.ri_user_time)
+ + (after.ri_system_time - before.ri_system_time),
+ after.ri_pkg_idle_wkups - before.ri_pkg_idle_wkups,
+ after.ri_interrupt_wkups - before.ri_interrupt_wkups
+ );
+ }
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
#[test]
fn inherited_ignored_signal_is_reset() {
use std::os::unix::process::CommandExt;
@@ -781,7 +852,6 @@ fn inherited_ignored_signal_is_reset() {
assert!(wait(&mut caller, TIMEOUT).success());
}
-#[cfg(target_os = "linux")]
#[test]
fn supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased() {
use std::os::unix::fs::OpenOptionsExt;
@@ -806,34 +876,34 @@ fn supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased() {
let first = f.state("first");
f.remember(&json!({"processes":[first.clone()]}));
let guard = eventually(|| {
- fs::read_to_string(format!(
- "/proc/{}/task/{}/children",
- caller.id(),
- caller.id()
- ))
- .ok()
- .and_then(|s| {
- s.split_whitespace()
- .filter_map(|v| v.parse::().ok())
- .find(|p| *p != pid(&first, "guardPid"))
- })
- });
- // Wait for the guard to exec and close all inherited descriptors except
- // stdio; fd 2 must still carry the stop lease while command lock is held.
- eventually(|| {
- fs::read_link(format!("/proc/{guard}/exe"))
- .ok()
- .filter(|p| {
- p.file_name()
- .is_some_and(|n| n == "hack-lifecycle-supervisor")
+ census()
+ .into_iter()
+ .find(|p| {
+ p.pid != pid(&first, "guardPid")
+ && p.session == p.pid
+ && parent_pid(p.pid) == Some(caller.id() as i32)
})
+ .map(|p| p.pid)
});
- let lease_inode = fs::metadata(f.root.join(".stop.lock")).unwrap().ino();
- eventually(|| {
- fs::metadata(format!("/proc/{guard}/fd/2"))
- .ok()
- .filter(|metadata| metadata.ino() == lease_inode)
- });
+ // Linux additionally observes the exact fd-2 inode. Both platforms prove
+ // the lease behavior by a blocked stop after owner death below.
+ #[cfg(target_os = "linux")]
+ {
+ eventually(|| {
+ fs::read_link(format!("/proc/{guard}/exe"))
+ .ok()
+ .filter(|p| {
+ p.file_name()
+ .is_some_and(|n| n == "hack-lifecycle-supervisor")
+ })
+ });
+ let lease_inode = fs::metadata(f.root.join(".stop.lock")).unwrap().ino();
+ eventually(|| {
+ fs::metadata(format!("/proc/{guard}/fd/2"))
+ .ok()
+ .filter(|metadata| metadata.ino() == lease_inode)
+ });
+ }
let guard_identity = identity(guard).unwrap();
f.owned.borrow_mut().push(guard_identity.clone());
caller.kill().unwrap();
@@ -1072,6 +1142,106 @@ fn stale_reused_session_leader_does_not_block_fresh_launch() {
);
}
+#[test]
+fn dead_reused_session_cannot_authorize_signals_from_old_receipt() {
+ use std::os::unix::process::CommandExt;
+ let f = Fixture::new();
+ let (mut old, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ f.stop();
+ assert!(wait(&mut old, TIMEOUT).success());
+ let terminal = f.state("worker");
+ // Use a real dead session leader and real surviving child. Relocate the
+ // old receipt's SID to that session rather than waiting for host PID wrap.
+ thread::sleep(Duration::from_millis(30));
+ let mut launch = Command::new(std::env::current_exe().unwrap());
+ launch
+ .args(["--exact", "fixture_process", "--nocapture"])
+ .env_clear()
+ .env("SUPERVISOR_FIXTURE", "leave-session-child")
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null());
+ unsafe {
+ launch.pre_exec(|| {
+ if libc::setsid() < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ });
+ }
+ let mut leader = launch.spawn().unwrap();
+ let sid = leader.id() as i32;
+ let mut output = String::new();
+ leader
+ .stdout
+ .take()
+ .unwrap()
+ .read_to_string(&mut output)
+ .unwrap();
+ assert!(wait(&mut leader, TIMEOUT).success());
+ let child_pid: i32 = output
+ .lines()
+ .find_map(|line| {
+ line.strip_prefix("session-child ")
+ .and_then(|pid| pid.parse().ok())
+ })
+ .unwrap();
+ let child = identity(child_pid).unwrap();
+ f.owned.borrow_mut().push(child.clone());
+ assert_eq!(child.session, sid);
+ assert!(identity(sid).is_none(), "replacement leader must be gone");
+ let mut relocated = terminal.clone();
+ relocated["guardPid"] = json!(sid);
+ relocated["processGroupId"] = json!(sid);
+ for phase in ["running", "starting", "exited", "killed", "failed-start"] {
+ relocated["phase"] = json!(phase);
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&relocated).unwrap(),
+ )
+ .unwrap();
+ let (mut status, observed) = f.request("status", &f.control(TOKEN));
+ let uncertain = matches!(phase, "running" | "starting");
+ assert_eq!(wait(&mut status, TIMEOUT).success(), !uncertain);
+ assert_eq!(
+ observed["unknownProcessNames"].as_array().unwrap().len(),
+ usize::from(uncertain)
+ );
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert_eq!(
+ receipt["outcome"],
+ if uncertain { "unknown" } else { "stopped" }
+ );
+ assert_eq!(wait(&mut stop, TIMEOUT).success(), !uncertain);
+ assert!(
+ identity(child_pid).as_ref() == Some(&child),
+ "unrelated session was signalled"
+ );
+ }
+ // A terminal old generation must not fence an unrelated generation.
+ let (mut next, _) =
+ f.start_generation(vec![f.process("worker", "hold")], "synthetic-new-owner");
+ f.stop_generation("synthetic-new-owner");
+ assert!(wait(&mut next, TIMEOUT).success());
+ assert!(identity(child_pid).as_ref() == Some(&child));
+}
+
+#[test]
+fn preexisting_broad_log_is_refused_before_launch() {
+ use std::os::unix::fs::PermissionsExt;
+ let f = Fixture::new();
+ fs::write(f.root.join("lifecycle.log"), b"sentinel").unwrap();
+ fs::set_permissions(
+ f.root.join("lifecycle.log"),
+ fs::Permissions::from_mode(0o644),
+ )
+ .unwrap();
+ refused(&f, "run", f.launch(vec![f.process("worker", "hold")]));
+ assert!(!f.root.join("worker.json").exists());
+ assert_eq!(fs::read(f.root.join("lifecycle.log")).unwrap(), b"sentinel");
+}
+
#[test]
fn incomplete_frame_has_one_whole_handoff_deadline() {
let f = Fixture::new();
@@ -1452,6 +1622,23 @@ fn fixture_process() {
libc::signal(libc::SIGTERM, libc::SIG_IGN);
}
}
+ "leave-session-child" => {
+ // SAFETY: fork child uses only async-signal-safe operations and
+ // never returns into libtest's multithreaded runtime.
+ let child = unsafe { libc::fork() };
+ assert!(child >= 0);
+ if child == 0 {
+ unsafe {
+ libc::close(1);
+ loop {
+ libc::pause();
+ }
+ }
+ }
+ println!("session-child {child}");
+ std::io::stdout().flush().unwrap();
+ std::process::exit(0);
+ }
"hold" => (),
_ => std::process::exit(99),
}
diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts
index 07b905dd9..56fada166 100644
--- a/scripts/lib/tla-runtime-models.ts
+++ b/scripts/lib/tla-runtime-models.ts
@@ -144,10 +144,23 @@ const contracts: readonly ModelContract[] = [
{
name: "lifecycle-supervisor",
module: "LifecycleSupervisor",
- states: 129,
+ states: 168,
invariant: "NoDuplicate",
action: "Retry",
fields: ["duplicate = TRUE", "live = TRUE", "durable = TRUE"],
+ additionalControls: [
+ {
+ name: "sid-reuse",
+ negative: true,
+ invariant: "NoForeignSignal",
+ action: "RecoverOrphan",
+ fields: [
+ 'identity = "sid-reused"',
+ "unsafeSignal = TRUE",
+ "guard = FALSE",
+ ],
+ },
+ ],
},
{
name: "interactive-session",
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index 217adea6e..f4e0d382a 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -13,7 +13,7 @@ standalone Rust binary exercised by a fake caller; the CLI remains unchanged.
| `ProcessExit`, `Reap` | Session census ignores zombies; reap direct/adopted children; release lock after no live members | Zombie liveness; child changes process group within session |
| `StopUnknown` | Bounded confirmation can report uncertainty while live members remain | Honest unknown recovery outcome, never successful absence |
-Positive TLC exploration: 129 distinct states, queue exhausted, depth 11. Negative
+Positive TLC exploration: 168 distinct states, queue exhausted, depth 11. Negative
control sets `SkipLiveFence = TRUE`; `Retry` then violates `NoDuplicate` with live,
durable and duplicate all true in the same state. Both guard-loss and project-loss
traces can witness this violation; the registry requires the named action and
@@ -23,14 +23,16 @@ Checker: checksum-pinned TLA+ 1.7.4 artifact (TLC reports version 2.19), Java 17
two workers and a bounded heap. The maintained runner pins the positive count and
validates the negative invariant/action/witness. Apalache is not qualified here.
-`NoForeignSignal` is true by construction: both signal actions require
-`identity = "match"` and preserve that value in their assignment to `unsafeSignal`.
-It does not independently validate observation freshness, PID/session reuse, or the
-source's effect boundary. `GuardRetainsLock` similarly checks the abstract action
+The guard-owned Stop and Kill checks still assume matching identity. The added
+orphan signal admission is exercised by a separate SID-reuse negative control.
+The orphan path includes an environment SID-reuse action and requires an old-birth
+witness, represented by matching identity. The sid-reuse negative control removes
+that witness guard and must violate NoForeignSignal at RecoverOrphan with identity
+sid-reused. Native clock units, enumeration and identity-to-effect races are omitted. `GuardRetainsLock` similarly checks the abstract action
assignments, not descriptor inheritance or actual flock ownership. `NoDuplicate`
checks the selected live fence and its negative control in this finite abstraction;
-it does not prove lock/census correctness. The positive state count is retained
-because no model actions, variables or checker registry have changed in this slice.
+it does not prove lock/census correctness. The positive count now includes ReuseSessionID and RecoverOrphan. Terminal phases
+have no recovery signal action.
Assumptions: flock serialization, atomic/synchronized publication, reliable native
identity and session census, and rechecking identity before effects. Signal delivery
@@ -50,8 +52,8 @@ Native recovery added after this abstraction is deliberately not claimed as mode
coverage: dead-guard TERM/grace/KILL recovery, supervisor death before guard
publication, inherited launch-admission lease, stop/exit inspection races, census
errors and restricted views, high descriptor limits, EINTR retries, and idle
-poll scheduling. `GuardDeath` can reach `Orphaned`, but `Stop` requires `guard` and
-cannot recover it in this model. Native regressions include
+poll scheduling. RecoverOrphan models signal admission only, not grace, KILL timing or confirmation.
+ReuseSessionID represents a later survivor after its replacement leader exits. Native regressions include
`dead_guard_with_live_session_recovers_term_then_kill`,
`supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased`,
`supervisor_killed_while_ready_receipt_is_blocked_recovers`, and
@@ -67,3 +69,9 @@ use fresh tokens. The regression
`completed_stop_fences_a_run_whose_frame_arrives_after_retirement` proves the
late frame creates no command receipt. Linux executable replacement is qualified
by the incremental integration test in the opt-in slice, not by this model.
+
+The real-process dead_reused_session_cannot_authorize_signals_from_old_receipt
+regression creates a dead session leader and surviving child, then relocates
+a stale receipt SID to reproduce reuse without forcing the host PID allocator
+to wrap. It checks terminal absence, non-terminal unknown without signals, and
+fresh-owner launch after a terminal receipt.
diff --git a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
index 0fc117f65..3dbcbee0f 100644
--- a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
+++ b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
@@ -1,6 +1,6 @@
------------------------ MODULE LifecycleSupervisor ------------------------
EXTENDS Naturals
-CONSTANT SkipLiveFence
+CONSTANT SkipLiveFence, SkipOrphanFence
VARIABLES phase, durable, guard, locked, live, supervisor, attempts,
identity, unsafeSignal, duplicate
vars == <>
Kill == /\ phase = "Stopping" /\ guard /\ identity = "match"
/\ phase' = "KillPending"
- /\ unsafeSignal' = unsafeSignal \/ identity # "match"
+ /\ unsafeSignal' = (unsafeSignal \/ identity # "match")
/\ UNCHANGED <>
StopUnknown == /\ phase \in {"Stopping", "KillPending"} /\ live
@@ -69,13 +69,27 @@ Reap == /\ ~live /\ phase \notin {"Absent", "Starting", "Exited"}
/\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE
/\ supervisor' = FALSE
/\ UNCHANGED <>
-Next == Reserve \/ Publish \/ Spawn \/ StartFailure \/ SupervisorDeath
+\* Environment: after the old session empties, a new leader can take its
+\* numeric SID, fork, and exit. The survivor has no old birth witness.
+ReuseSessionID == /\ durable /\ ~guard /\ ~live
+ /\ identity = "match"
+ /\ phase \in {"Orphaned", "Exited"}
+ /\ identity' = "sid-reused" /\ live' = TRUE
+ /\ UNCHANGED <>
+RecoverOrphan == /\ phase = "Orphaned" /\ ~guard /\ live /\ durable
+ /\ (identity = "match" \/ (SkipOrphanFence /\ identity = "sid-reused"))
+ /\ unsafeSignal' = (unsafeSignal \/ identity # "match")
+ /\ phase' = "Stopping"
+ /\ UNCHANGED <>
+Next == ReuseSessionID \/ RecoverOrphan \/ Reserve \/ Publish \/ Spawn \/ StartFailure \/ SupervisorDeath
\/ GuardDeath \/ Retry \/ ReplaceIdentity \/ Stop \/ Kill
\/ StopUnknown \/ ProcessExit \/ Reap
Spec == Init /\ [][Next]_vars
TypeOK == /\ phase \in {"Absent", "Starting", "Running", "Recoverable",
"Orphaned", "Stopping", "KillPending", "StopUnknown", "Exited"}
- /\ attempts \in 0..2 /\ identity \in {"match", "foreign", "unknown"}
+ /\ attempts \in 0..2 /\ identity \in {"match", "foreign", "unknown", "sid-reused"}
/\ \A b \in {durable, guard, locked, live, supervisor,
unsafeSignal, duplicate} : b \in BOOLEAN
DurableBeforeLaunch == live => durable
diff --git a/tests/models/tla/lifecycle-supervisor/negative.cfg b/tests/models/tla/lifecycle-supervisor/negative.cfg
index a53d6e384..1fb04f6c8 100644
--- a/tests/models/tla/lifecycle-supervisor/negative.cfg
+++ b/tests/models/tla/lifecycle-supervisor/negative.cfg
@@ -1,4 +1,5 @@
SPECIFICATION Spec
+CONSTANT SkipOrphanFence = FALSE
CONSTANT SkipLiveFence = TRUE
INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-supervisor/positive.cfg b/tests/models/tla/lifecycle-supervisor/positive.cfg
index 58bdd8c68..b687855fe 100644
--- a/tests/models/tla/lifecycle-supervisor/positive.cfg
+++ b/tests/models/tla/lifecycle-supervisor/positive.cfg
@@ -1,4 +1,5 @@
SPECIFICATION Spec
+CONSTANT SkipOrphanFence = FALSE
CONSTANT SkipLiveFence = FALSE
INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg b/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg
new file mode 100644
index 000000000..06c12eaa4
--- /dev/null
+++ b/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg
@@ -0,0 +1,5 @@
+SPECIFICATION Spec
+CONSTANT SkipLiveFence = FALSE
+CONSTANT SkipOrphanFence = TRUE
+INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
+CHECK_DEADLOCK FALSE
From f13b2c85d8851a6a9d3bec5fd86418059dbec73c Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 03:32:38 -0500
Subject: [PATCH 13/45] test(cli): disable daemon autostart in env plan
acceptance
---
scripts/check-native-env-plan-cli.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/scripts/check-native-env-plan-cli.ts b/scripts/check-native-env-plan-cli.ts
index 04e189bc0..5f6e91c4c 100644
--- a/scripts/check-native-env-plan-cli.ts
+++ b/scripts/check-native-env-plan-cli.ts
@@ -298,6 +298,7 @@ async function invoke(
PATH: "/usr/bin:/bin",
HOME: home,
HACK_LOGGER: "console",
+ HACK_DAEMON_AUTOSTART_DISABLE: "1",
AWS_SECRET_ACCESS_KEY: sentinel,
...extraEnv,
},
From 0301ffce3e77953a4f5d2980d2ce62019dab12ea Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 04:14:47 -0500
Subject: [PATCH 14/45] test(cli): identify unexpected env plan fixture state
---
scripts/check-native-env-plan-cli.ts | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/scripts/check-native-env-plan-cli.ts b/scripts/check-native-env-plan-cli.ts
index 5f6e91c4c..0ed1d9b9a 100644
--- a/scripts/check-native-env-plan-cli.ts
+++ b/scripts/check-native-env-plan-cli.ts
@@ -276,10 +276,14 @@ try {
detachedResult.ok === false &&
record(detachedResult.error).code ===
"E_STARTUP_INCOMPLETE", "admitted detached startup returns one JSON failure envelope without readiness");
- require((await readdir(home)).length === 0 &&
- !(await readdir(join(checkout, ".hack"))).some(
- (name) => name === ".internal" || name === ".branch"
- ), "no registry or generated runtime state");
+ const homeEntries = await readdir(home);
+ const generatedEntries = (await readdir(join(checkout, ".hack"))).filter(
+ (name) => name === ".internal" || name === ".branch"
+ );
+ // Only relative names inside this synthetic fixture cross the diagnostic boundary.
+ require(homeEntries.length === 0 &&
+ generatedEntries.length ===
+ 0, `no registry or generated runtime state (fixture HOME: ${JSON.stringify(homeEntries)}; generated: ${JSON.stringify(generatedEntries)})`);
process.stdout.write(
"Relocated native env plan acceptance: offline validation, metadata-only output, missing overlays/refs, immutable directives, separate hashes, real linked-worktree layers, CI/slim/opt-out, unsafe-input and runtime refusal passed\n"
);
From b228781ad02f3b9b246a2a16afd1ed1ad18125b7 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 07:55:04 -0500
Subject: [PATCH 15/45] fix(supervisor): retain recovery authority after
witness exit
---
docs/diagrams/lifecycle-supervisor.mmd | 5 +-
docs/diagrams/lifecycle-supervisor.svg | 4 +-
docs/lifecycle-supervisor.md | 49 +++++-
docs/reference/v5-model-coverage.md | 4 +-
packages/lifecycle-supervisor/README.md | 6 +-
packages/lifecycle-supervisor/src/os.rs | 159 ++++++++++++------
.../lifecycle-supervisor/src/supervisor.rs | 82 ++++-----
.../lifecycle-supervisor/tests/fake_caller.rs | 118 +++++++++++++
scripts/lib/tla-runtime-models.ts | 9 +-
.../tla/lifecycle-supervisor/COVERAGE.md | 18 +-
.../LifecycleSupervisor.tla | 62 ++++---
.../tla/lifecycle-supervisor/negative.cfg | 3 +-
.../tla/lifecycle-supervisor/positive.cfg | 3 +-
.../tla/lifecycle-supervisor/sid-reuse.cfg | 3 +-
.../tla/lifecycle-supervisor/witness-loss.cfg | 6 +
15 files changed, 391 insertions(+), 140 deletions(-)
create mode 100644 tests/models/tla/lifecycle-supervisor/witness-loss.cfg
diff --git a/docs/diagrams/lifecycle-supervisor.mmd b/docs/diagrams/lifecycle-supervisor.mmd
index 1021a15f0..c9fa975bf 100644
--- a/docs/diagrams/lifecycle-supervisor.mmd
+++ b/docs/diagrams/lifecycle-supervisor.mmd
@@ -12,12 +12,13 @@ stateDiagram-v2
Running --> Orphaned : GuardDeath [members live]
Recoverable --> Orphaned : GuardDeath [members live]
Recoverable --> Stopping : Stop [identity matches]
- Orphaned --> Stopping : RecoverStop [boot, old birth witness and members verified]
+ Orphaned --> Stopping : RecoverStop [exact witness; capture members]
Orphaned --> StopUnknown : RecoverStop [identity unconfirmed]
Running --> Stopping : Stop [identity matches]
Running --> Exited : Reap [no live members]
Stopping --> Exited : Reap [no live members]
Stopping --> KillPending : Kill [grace elapsed]
+ Stopping --> Stopping : WitnessExit [captured identities retained]
KillPending --> Exited : Reap [no live members]
KillPending --> StopUnknown : StopUnknown [members live]
note right of Starting
@@ -25,6 +26,6 @@ stateDiagram-v2
end note
note right of Orphaned
Unknown health retains ownership
- No old birth witness refuses signals
+ No exact witness refuses capture
Live session refuses duplicate
end note
diff --git a/docs/diagrams/lifecycle-supervisor.svg b/docs/diagrams/lifecycle-supervisor.svg
index 18bff95da..91c811ee9 100644
--- a/docs/diagrams/lifecycle-supervisor.svg
+++ b/docs/diagrams/lifecycle-supervisor.svg
@@ -1,4 +1,4 @@
-
\ No newline at end of file
+]]>Lifecycle supervisor · ownership and recoveryReserve [locks acquired]Spawn [identity durable]StartFailureSupervisorDeathRetry [duplicate refused]GuardDeath [members live]GuardDeath [members live]Stop [identity matches]RecoverStop [exact witnessRecoverStop [identityunconfirmed]Stop [identity matches]Reap [no live members]Reap [no live members]Kill [grace elapsed]WitnessExit [capturedidentities retained]Reap [no live members]StopUnknown [members live]AbsentRunningExitedRecoverableStopping;capturemembers]StopUnknownKillPendingStartingOrphanedINV identity precedes commandUnknown health retains ownershipNo exact witness refuses captureLive session refuses duplicate
\ No newline at end of file
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 90b5fde04..b1e066636 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -43,10 +43,9 @@ identity is a refusal, never proof that a process is absent. A different boot or
leader with a different start identity proves that the old session ended; it
grants no authority over the replacement. Guard-written terminal receipts are retained as proof that the old session ended.
A missing leader plus a numeric SID is not ownership: orphan recovery requires
-a same-user surviving member born no later than the recorded command birth.
-Only newer members, or legacy receipts without that birth evidence, report
-unknown without signals. Descendants born after the command whose birth was
-recorded may therefore require manual recovery once that command exits.
+the exact recorded command PID/birth or a matching guard, including a zombie.
+Only unrecorded members, or legacy receipts without a surviving matching guard, report
+unknown without signals. Descendants alone may require manual recovery when both recorded processes are gone.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
`observed`, `stopped`, or `unknown`). A stop whose confirmation times out emits
@@ -116,8 +115,10 @@ supervisor cannot launch a second copy. If a guard itself is separately killed
while its command survives, health becomes unknown and duplicate launch is
refused. Stop can recover the recorded session after verifying the boot,
leader identity and each remaining member's UID, birth and session. Recovery
-delivers TERM individually, grants a fresh five-second grace, then KILL and
-confirms absence. Unverified identity never authorizes signals or a successful
+delivers TERM individually, grants a fresh five-second grace, then KILL to the captured incarnations and
+confirms absence. The capture remains authoritative if TERM ends the witness;
+every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux).
+Unobserved descendants never inherit this authority and retain an unknown outcome. Unverified identity never authorizes signals or a successful
stop. Launch and stop share a bounded lock; a startup descriptor lease keeps
admission locked through guard publication even if the project owner dies.
The guard temporarily inherits this stop-lock lease on stderr (fd 2), then closes
@@ -198,7 +199,7 @@ the exact 1,048,576 test requires a host that permits that limit.
Darwin compilation does not qualify execution, group-signal EPERM recovery or
idle cost on a real Mac.
-The 168-state TLA+ artifact is an abstract phase/ownership exploration. It models SID reuse and a required old-birth witness for orphan signaling. It does
+The 322-state TLA+ artifact is an abstract phase/ownership exploration. It models SID reuse and a required old-birth witness for orphan signaling. It does
not model actual kernel census observations, real flock behavior, full orphan recovery,
startup descriptor leases or signal errors. Some safety predicates hold by
construction. Native regressions, rather than that model, are the evidence for
@@ -231,3 +232,37 @@ guard launch frames, and blocked ready delivery. Deterministic tests at the two
smaller instruction windows (starting-state save to spawn, and spawn to running-state
save) remain unqualified; polling a receipt cannot reliably stop a process in either
window. They require a dedicated synchronization seam, not a timing-only test.
+
+## Manual recovery after an unknown stop
+
+Stop retires the owner token before validating all receipts. One unknown receipt
+can therefore leave otherwise healthy guards running and block every new start
+in that private root. Repeating up with a new token does not bypass the uncertainty.
+This also applies when a guard dies after spawn but before publishing running:
+the receipt has no command birth evidence, so an unanchored session is unknown.
+
+1. Keep the private root and all receipts. Run status with the recorded token;
+ use the reported guard/command PIDs and session as leads for inspection, never
+ as authority to kill. Inspect current processes with the OS process tools
+ (ps on macOS; ps and /proc on Linux), including PID, parent PID, SID, UID,
+ start time and command. Identify the actual application processes yourself.
+2. For each process you confirm belongs to this failed launch, record its current
+ start identity. Send TERM to those individual PIDs, wait at least five seconds,
+ and recheck each identity before sending KILL to any remaining process. Never
+ signal a numeric process group/session from the stale receipt alone. Leave
+ unfamiliar or inaccessible processes untouched and seek operator help.
+3. Retry stop with the original token (or hack down through the same persisted
+ project owner). Once the uncertain session is absent, normal stop can retire
+ the remaining healthy guards. If another receipt is unknown, inspect it in the
+ same way. Confirm stopped/status absence before a fresh up with a fresh token.
+4. Do not delete receipts, locks, retirement markers or the root to unblock up.
+ That discards the ownership evidence without stopping anything. Offline root
+ rotation is allowed only after all original callers/guards and authored work
+ have been independently confirmed gone.
+
+Darwin start timestamps come from the wall clock and can decrease between guard
+and command creation. Recovery therefore never orders those timestamps: it matches
+PID and the recorded birth exactly. A backward step between those starts does not
+invalidate a still-matching command witness. The native identity contract still
+assumes a PID and exact recorded birth value identify one incarnation; Darwin has
+no pidfd, so its short inspection-to-signal race remains as documented above.
diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md
index 59e008b65..737dca745 100644
--- a/docs/reference/v5-model-coverage.md
+++ b/docs/reference/v5-model-coverage.md
@@ -36,7 +36,7 @@ must be attached to the eventual exact integration revision.
| Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification |
| --- | --- | --- | --- |
-| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 168 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
+| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 322 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
| Interactive exec write-ahead admission, crash, EOF, replacement boot and retirement | interactive-session (459 states); no-replay, EOF-retirement, failed-inspect and wrong-boot negative controls | `interactive_session.rs` atomic intent/exec records, exact stopped-boot proof, known-undispatched withdrawal, restore-before-busy-retirement and run-scoped cleanup-report tests, and lock-free observation/revalidation tests; Engine effect fences; authored owner start and SCM_RIGHTS handoff | One session, two boot identities, two possible starts. Atomic durable publication and exact identity/absence proofs are assumptions; real fsync/power loss, real owner/Engine handoff and VM loss need Mac acceptance; receipt lock timing and terminal restoration are regression-covered, not modeled. Validated exact exec exit permits retirement; it makes no descendant cleanup claim. |
| Private interactive envelope, echo state and bytes still in flight on failure | interactive-envelope (15 states); restore-on-error negative control | HKEE2 launcher and host gate, Linux PTY seven-case matrix, bounded marker/ACK and coalesced-output tests | Two abstract private bytes, one handshake. PTY fixtures prove local termios ordering; Docker detach scanning, real guest wiring, same-user access and VM effects remain separate. Source/compiled Bun PTY and native-process fixtures cover lifetime/terminal boundaries, not model liveness. |
| Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. |
@@ -83,7 +83,7 @@ liveness/reachability controls have their own documented assumptions.
| Model | Positive distinct states | Configurations |
| --- | ---: | ---: |
-| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 168 | 2 |
+| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 322 | 4 |
| [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 |
| [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 |
| [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 |
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 80e23a774..138cbab5f 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -80,8 +80,10 @@ The ignored `million_descriptor_limit_qualification` fixture requires a host har
RLIMIT_NOFILE of at least 1048576; the ordinary raised-limit fixture uses the host's
available limit. A host refusing that raise does not qualify the million-fd case.
-Orphan recovery requires a surviving member whose kernel birth is no later than
-the recorded command birth. A reused numeric SID alone never authorizes a signal.
+Orphan recovery requires an exact command PID/birth or matching guard, including
+a zombie. It captures verified members before TERM and retains those incarnations
+through KILL even after the witness exits. Darwin birth timestamps are matched,
+never ordered across processes. A reused numeric SID alone never authorizes a signal.
Terminal receipts prove the old guard completed cleanup; uncertain legacy or
newer-member-only sessions remain unknown without effects. See
[resource and qualification boundaries](../../docs/lifecycle-supervisor.md#resource-and-qualification-boundaries)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index a333cb369..fec64c157 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -190,7 +190,12 @@ pub fn inspect(pid: i32) -> Result> {
// SAFETY: the kernel initialized exactly the expected struct size above.
let info = unsafe { info.assume_init() };
// SAFETY: getsid only observes the named PID. ESRCH is an exit race.
- let session = unsafe { libc::getsid(pid) };
+ let mut session = unsafe { libc::getsid(pid) };
+ if session < 0 && info.pbi_status == libc::SZOMB && info.pbi_pgid as i32 == pid {
+ // Darwin may hide getsid for zombies. A matching recorded leader's
+ // immutable PID/birth/UID still proves it has not been reused.
+ session = pid;
+ }
if session < 0 {
let error = io::Error::last_os_error();
if error.raw_os_error() == Some(libc::ESRCH) {
@@ -327,43 +332,62 @@ pub fn absent_error(error: &io::Error) -> bool {
|| matches!(error.raw_os_error(), Some(libc::ESRCH) | Some(libc::ENOENT))
}
-/// Terminal receipts prove the guard finished its census. With no leader, a
-/// matching SID alone is insufficient: that number may belong to a later session.
-/// An observed member born by the recorded command birth anchors the old session.
-/// Missing legacy birth evidence or only newer members is unknown, never authority.
-pub fn recorded_live(state: &crate::storage::State, excluding_guard: bool) -> Result {
- if matches!(state.phase.as_str(), "exited" | "killed" | "failed-start")
- || state.boot_id != boot()?
- {
- return Ok(false);
- }
- let leader = inspect(state.guard_pid)?;
- if let Some(identity) = &leader
- && (identity.start != state.guard_start_time
- || identity.uid != state.uid
- || identity.session != state.guard_pid)
- {
- return Ok(false);
- }
- let members: Vec<_> = census()?
- .into_iter()
- .filter(|p| p.session == state.guard_pid && !p.zombie)
- .collect();
- if members.is_empty() {
- return Ok(false);
- }
- if leader.is_none_or(|p| p.zombie)
- && !members.iter().any(|p| {
- p.uid == state.uid
- && p.start >= state.guard_start_time
- && state
- .command_start_time
- .is_some_and(|birth| p.start <= birth)
- })
- {
- return Err(refused());
+fn incarnation_matches(expected: &Identity, current: &Identity) -> bool {
+ expected.pid == current.pid
+ && expected.start == current.start
+ && expected.uid == current.uid
+ && expected.session == current.session
+}
+
+/// Return one census with session identity proved across that census. A matching
+/// leader (including zombie) or exact recorded command birth is the witness.
+/// No ordering comparison uses Darwin's adjustable wall-clock start timestamps.
+fn recorded_members(state: &crate::storage::State) -> Result> {
+ for _ in 0..2 {
+ if matches!(state.phase.as_str(), "exited" | "killed" | "failed-start")
+ || state.boot_id != boot()?
+ {
+ return Ok(Vec::new());
+ }
+ let leader = inspect(state.guard_pid)?;
+ if let Some(p) = &leader
+ && (p.start != state.guard_start_time
+ || p.uid != state.uid
+ || p.session != state.guard_pid)
+ {
+ return Ok(Vec::new());
+ }
+ let members: Vec<_> = census()?
+ .into_iter()
+ .filter(|p| p.session == state.guard_pid && !p.zombie)
+ .collect();
+ if members.is_empty() {
+ return Ok(members);
+ }
+ let witness = leader
+ .as_ref()
+ .or_else(|| {
+ members.iter().find(|p| {
+ Some(p.pid) == state.command_pid
+ && Some(p.start) == state.command_start_time
+ && p.uid == state.uid
+ })
+ })
+ .ok_or_else(refused)?;
+ // Recheck after enumeration: the witness held the SID throughout the census.
+ if !inspect(witness.pid)?
+ .as_ref()
+ .is_some_and(|p| incarnation_matches(witness, p))
+ {
+ continue;
+ }
+ return Ok(members);
}
- Ok(members
+ Err(refused())
+}
+
+pub fn recorded_live(state: &crate::storage::State, excluding_guard: bool) -> Result {
+ Ok(recorded_members(state)?
.iter()
.any(|p| !excluding_guard || p.pid != state.guard_pid))
}
@@ -441,7 +465,8 @@ fn signal_member(expected: &Identity, signal: i32) -> Result<()> {
let Some(current) = inspect(expected.pid)? else {
return Ok(());
};
- if current.start != expected.start
+ if current.zombie
+ || current.start != expected.start
|| current.session != expected.session
|| current.uid != expected.uid
{
@@ -470,7 +495,8 @@ fn signal_member(expected: &Identity, signal: i32) -> Result<()> {
let Some(current) = inspect(expected.pid)? else {
return Ok(());
};
- if current.start != expected.start
+ if current.zombie
+ || current.start != expected.start
|| current.session != expected.session
|| current.uid != expected.uid
{
@@ -690,26 +716,49 @@ pub fn wait_activity(fds: &[i32], timeout: std::time::Duration) -> Result<()> {
Ok(())
}
-/// A dead leader's SID cannot be reused while any old session member remains.
-/// Validate the receipt and each incarnation again before individual delivery.
-/// Never use group delivery here: a group number can outlive its observed members.
-pub fn signal_recorded_session(state: &crate::storage::State, signal: i32) -> Result<()> {
- if !recorded_live(state, false)? {
- return Ok(());
- }
- for member in census()?
- .into_iter()
- .filter(|p| p.session == state.guard_pid && !p.zombie)
- {
- if member.uid != state.uid || member.start < state.guard_start_time {
+/// In-memory authority captured while a witness holds the old SID. It survives
+/// witness exit but never grants authority over a later, unobserved incarnation.
+/// A new stop invocation must establish its own witness again.
+pub struct Recovery {
+ session: i32,
+ members: Vec,
+}
+
+impl Recovery {
+ pub fn capture(state: &crate::storage::State) -> Result {
+ let members: Vec<_> = recorded_members(state)?
+ .into_iter()
+ .filter(|p| p.pid != state.guard_pid)
+ .collect();
+ if members.iter().any(|p| p.uid != state.uid) {
return Err(refused());
}
- if !recorded_live(state, false)? {
- return Ok(());
+ Ok(Self {
+ session: state.guard_pid,
+ members,
+ })
+ }
+
+ pub fn signal(&self, signal: i32) -> Result<()> {
+ for member in &self.members {
+ signal_member(member, signal)?;
}
- signal_member(&member, signal)?;
+ Ok(())
+ }
+
+ pub fn live(&self) -> Result {
+ for member in &self.members {
+ if inspect(member.pid)?.is_some_and(|p| !p.zombie && incarnation_matches(member, &p)) {
+ return Ok(true);
+ }
+ }
+ // Unobserved descendants or an unrelated reused session cannot be
+ // signalled by this snapshot, and must not be claimed as stopped.
+ if live_group(self.session, Some(self.session))? {
+ return Err(refused());
+ }
+ Ok(false)
}
- Ok(())
}
/// A successful census must contain our own readable identity. Known restricted
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 37859b632..589f6eb8c 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -80,8 +80,8 @@ pub fn main() -> Result<()> {
states.push(state);
} else if os::recorded_live(&state, false)? {
// Old terminal receipts survive renames and new generations.
- // Only a kernel census can prove they no longer hold work;
- // a persisted terminal phase alone is never sufficient.
+ // Guard-written terminal phases prove completed retirement;
+ // non-terminal receipts require native identity evidence.
return Err(refused());
}
}
@@ -465,12 +465,20 @@ fn guard_command(
fn stop(states: &[State]) -> Result<()> {
struct Target {
state: State,
- guard_live: bool,
+ recovery: Option,
orphan_term: Option,
}
+ impl Target {
+ fn recover(&mut self) -> Result<()> {
+ let recovery = os::Recovery::capture(&self.state)?;
+ recovery.signal(libc::SIGTERM)?;
+ self.orphan_term = Some(Instant::now());
+ self.recovery = Some(recovery);
+ Ok(())
+ }
+ }
let mut active = Vec::new();
- // Validate every receipt before the first effect. Absence may race a guard's
- // natural exit; a dead leader still leaves a recoverable recorded session.
+ // Validate all receipts and capture orphan members before the first effect.
for state in states {
if !os::recorded_live(state, false)? {
continue;
@@ -484,34 +492,23 @@ fn stop(states: &[State]) -> Result<()> {
};
active.push(Target {
state: state.clone(),
- guard_live,
+ recovery: if guard_live {
+ None
+ } else {
+ Some(os::Recovery::capture(state)?)
+ },
orphan_term: None,
});
}
for target in &mut active {
- let state = &target.state;
- let result = if target.guard_live {
- os::signal_guard(state)
- } else {
- let result = os::signal_recorded_session(state, libc::SIGTERM);
+ if let Some(recovery) = &target.recovery {
+ recovery.signal(libc::SIGTERM)?;
target.orphan_term = Some(Instant::now());
- result
- };
- if let Err(error) = result {
- if !os::recorded_live(state, false)? {
- continue;
- }
- // Guard death during admission switches to verified session recovery.
- match os::inspect(state.guard_pid)? {
- None => {
- os::signal_recorded_session(state, libc::SIGTERM)?;
- target.orphan_term = Some(Instant::now());
- }
- Some(identity) if identity.zombie => {
- os::signal_recorded_session(state, libc::SIGTERM)?;
- target.orphan_term = Some(Instant::now());
- }
- _ => return Err(error),
+ } else if let Err(error) = os::signal_guard(&target.state) {
+ if os::inspect(target.state.guard_pid)?.is_none_or(|p| p.zombie) {
+ target.recover()?;
+ } else if os::recorded_live(&target.state, false)? {
+ return Err(error);
}
}
}
@@ -519,25 +516,28 @@ fn stop(states: &[State]) -> Result<()> {
while start.elapsed() < GRACE + Duration::from_secs(2) {
let mut live = false;
for target in &mut active {
- let state = &target.state;
- if !os::recorded_live(state, false)? {
- continue;
+ if target.recovery.is_none() {
+ if !os::recorded_live(&target.state, false)? {
+ continue;
+ }
+ if os::inspect(target.state.guard_pid)?.is_none_or(|p| p.zombie) {
+ // Guard loss gets its own snapshot and full TERM grace.
+ target.recover()?;
+ } else {
+ live = true;
+ continue;
+ }
}
- live = true;
- // Guard death after TERM admission must not skip authored TERM or
- // borrow the dead guard's grace interval. Late loss may time out as
- // unknown; it never shortens this recovered session's own grace.
- let orphan = os::inspect(state.guard_pid)?.is_none_or(|p| p.zombie);
- if orphan {
- if target.orphan_term.is_none() {
- os::signal_recorded_session(state, libc::SIGTERM)?;
- target.orphan_term = Some(Instant::now());
+ if let Some(recovery) = &target.recovery {
+ if !recovery.live()? {
+ continue;
}
+ live = true;
if target
.orphan_term
.is_some_and(|since| since.elapsed() >= GRACE)
{
- os::signal_recorded_session(state, libc::SIGKILL)?;
+ recovery.signal(libc::SIGKILL)?;
}
}
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 7e171cac3..cccd2e75c 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1030,6 +1030,97 @@ fn dead_guard_with_live_session_recovers_term_then_kill() {
assert_eq!(receipt["liveProcessNames"], json!([]));
}
+#[test]
+fn orphan_recovery_keeps_observed_child_after_term_ends_witness() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "wrapper-term-child")]);
+ f.await_log("fixture-ready");
+ eventually(|| f.root.join("wrapper-child-ready").exists().then_some(()));
+ let state = f.state("worker");
+ let child = eventually(|| {
+ f.log().lines().find_map(|line| {
+ line.split("wrapper-child ")
+ .nth(1)
+ .and_then(|pid| pid.parse::().ok())
+ .and_then(identity)
+ })
+ });
+ f.owned.borrow_mut().push(child.clone());
+ assert!(child.birth > state["commandStartTime"].as_u64().unwrap());
+ // SAFETY: captured guard belongs to this fixture; command/wrapper survives.
+ assert_eq!(
+ unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
+ 0
+ );
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let start = Instant::now();
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert!(wait(&mut stop, TIMEOUT).success());
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(
+ start.elapsed() >= Duration::from_secs(5),
+ "child did not get a full TERM grace"
+ );
+ assert!(start.elapsed() < Duration::from_secs(7));
+ // A zombie is harmless and may await the host's subreaper; no live member remains.
+ assert!(
+ census()
+ .into_iter()
+ .all(|p| p.session != pid(&state, "guardPid") || process_is_zombie(p.pid))
+ );
+}
+
+fn process_is_zombie(pid: i32) -> bool {
+ #[cfg(target_os = "linux")]
+ {
+ fs::read_to_string(format!("/proc/{pid}/stat"))
+ .ok()
+ .and_then(|s| s.rsplit_once(") ").map(|(_, rest)| rest.starts_with('Z')))
+ .unwrap_or(true)
+ }
+ #[cfg(target_os = "macos")]
+ {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: native output buffer is writable and exactly sized.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ size == 0
+ || (size as usize == std::mem::size_of::()
+ && unsafe { info.assume_init() }.pbi_status == libc::SZOMB)
+ }
+}
+
+#[test]
+fn unreaped_matching_guard_proves_legacy_session_ownership() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let mut state = f.state("worker");
+ // Keep the parent from reaping its guard, then exercise legacy birth absence.
+ assert_eq!(unsafe { libc::kill(caller.id() as i32, libc::SIGSTOP) }, 0);
+ assert_eq!(
+ unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
+ 0
+ );
+ eventually(|| process_is_zombie(pid(&state, "guardPid")).then_some(()));
+ state["commandStartTime"] = Value::Null;
+ fs::write(
+ f.root.join("worker.json"),
+ serde_json::to_vec(&state).unwrap(),
+ )
+ .unwrap();
+ f.stop();
+ assert_eq!(unsafe { libc::kill(caller.id() as i32, libc::SIGCONT) }, 0);
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
#[test]
fn guard_dying_during_stop_gets_a_fresh_authored_term_grace() {
let f = Fixture::new();
@@ -1639,6 +1730,33 @@ fn fixture_process() {
std::io::stdout().flush().unwrap();
std::process::exit(0);
}
+ "wrapper-term-child" => {
+ // Make the child newer than the command even with Linux clock-tick
+ // resolution, so it cannot accidentally satisfy the old birth bound.
+ thread::sleep(Duration::from_millis(100));
+ // SAFETY: fork child invokes only libc operations and never returns
+ // into Rust/libtest's inherited multithreaded allocator state.
+ let child = unsafe { libc::fork() };
+ assert!(child >= 0);
+ if child == 0 {
+ unsafe {
+ libc::signal(libc::SIGTERM, libc::SIG_IGN);
+ let fd = libc::open(
+ c"wrapper-child-ready".as_ptr(),
+ libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL,
+ 0o600,
+ );
+ if fd < 0 {
+ libc::_exit(98);
+ }
+ libc::close(fd);
+ loop {
+ libc::pause();
+ }
+ }
+ }
+ println!("wrapper-child {child}");
+ }
"hold" => (),
_ => std::process::exit(99),
}
diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts
index 56fada166..97c17b6ff 100644
--- a/scripts/lib/tla-runtime-models.ts
+++ b/scripts/lib/tla-runtime-models.ts
@@ -144,7 +144,7 @@ const contracts: readonly ModelContract[] = [
{
name: "lifecycle-supervisor",
module: "LifecycleSupervisor",
- states: 168,
+ states: 322,
invariant: "NoDuplicate",
action: "Retry",
fields: ["duplicate = TRUE", "live = TRUE", "durable = TRUE"],
@@ -160,6 +160,13 @@ const contracts: readonly ModelContract[] = [
"guard = FALSE",
],
},
+ {
+ name: "witness-loss",
+ negative: true,
+ invariant: "RecoveryAuthorityRetained",
+ action: "WitnessExit",
+ fields: ["snapshot = FALSE", "witness = FALSE", "live = TRUE"],
+ },
],
},
{
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index f4e0d382a..b5af7fb7e 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -13,7 +13,7 @@ standalone Rust binary exercised by a fake caller; the CLI remains unchanged.
| `ProcessExit`, `Reap` | Session census ignores zombies; reap direct/adopted children; release lock after no live members | Zombie liveness; child changes process group within session |
| `StopUnknown` | Bounded confirmation can report uncertainty while live members remain | Honest unknown recovery outcome, never successful absence |
-Positive TLC exploration: 168 distinct states, queue exhausted, depth 11. Negative
+Positive TLC exploration: 322 distinct states, queue exhausted, depth 14. Negative
control sets `SkipLiveFence = TRUE`; `Retry` then violates `NoDuplicate` with live,
durable and duplicate all true in the same state. Both guard-loss and project-loss
traces can witness this violation; the registry requires the named action and
@@ -25,7 +25,7 @@ validates the negative invariant/action/witness. Apalache is not qualified here.
The guard-owned Stop and Kill checks still assume matching identity. The added
orphan signal admission is exercised by a separate SID-reuse negative control.
-The orphan path includes an environment SID-reuse action and requires an old-birth
+The orphan path includes an environment SID-reuse action and requires an exact recorded incarnation
witness, represented by matching identity. The sid-reuse negative control removes
that witness guard and must violate NoForeignSignal at RecoverOrphan with identity
sid-reused. Native clock units, enumeration and identity-to-effect races are omitted. `GuardRetainsLock` similarly checks the abstract action
@@ -52,7 +52,8 @@ Native recovery added after this abstraction is deliberately not claimed as mode
coverage: dead-guard TERM/grace/KILL recovery, supervisor death before guard
publication, inherited launch-admission lease, stop/exit inspection races, census
errors and restricted views, high descriptor limits, EINTR retries, and idle
-poll scheduling. RecoverOrphan models signal admission only, not grace, KILL timing or confirmation.
+poll scheduling. RecoverOrphan and OrphanKill model capture, witness loss and KILL admission;
+wall-clock grace timing and native confirmation remain implementation tests.
ReuseSessionID represents a later survivor after its replacement leader exits. Native regressions include
`dead_guard_with_live_session_recovers_term_then_kill`,
`supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased`,
@@ -75,3 +76,14 @@ regression creates a dead session leader and surviving child, then relocates
a stale receipt SID to reproduce reuse without forcing the host PID allocator
to wrap. It checks terminal absence, non-terminal unknown without signals, and
fresh-owner launch after a terminal receipt.
+
+Orphan recovery now models WitnessExit after RecoverOrphan captured the old
+incarnations. OrphanKill remains enabled after witness loss; the implementation
+retains those exact identities through its grace/KILL loop. RecoveryAuthorityRetained
+and OrphanKillEnabled check that availability boundary. The witness-loss negative
+control deliberately clears the captured snapshot on WitnessExit and must violate
+RecoveryAuthorityRetained with live work and no snapshot. SID reuse may follow
+an emptied session even during stopping; the old snapshot never authorizes a
+new incarnation. The native wrapper/TERM-ignoring-child regression checks this
+sequence with real processes. New descendants not in the snapshot remain an
+uncovered recovery/unknown boundary; no eventual-exit claim is made.
diff --git a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
index 3dbcbee0f..1d16e3f22 100644
--- a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
+++ b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
@@ -1,89 +1,103 @@
------------------------ MODULE LifecycleSupervisor ------------------------
EXTENDS Naturals
-CONSTANT SkipLiveFence, SkipOrphanFence
+CONSTANT SkipLiveFence, SkipOrphanFence, LoseRecoverySnapshot
VARIABLES phase, durable, guard, locked, live, supervisor, attempts,
- identity, unsafeSignal, duplicate
+ identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled
vars == <>
+ identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
Init == /\ phase = "Absent"
/\ durable = FALSE /\ guard = FALSE /\ locked = FALSE
/\ live = FALSE /\ supervisor = FALSE /\ attempts = 0
/\ identity = "match" /\ unsafeSignal = FALSE /\ duplicate = FALSE
+ /\ witness = FALSE /\ snapshot = FALSE /\ orphanKilled = FALSE
Reserve == /\ phase \in {"Absent", "Exited"} /\ ~live /\ attempts < 2
/\ phase' = "Starting" /\ guard' = TRUE /\ locked' = TRUE
/\ supervisor' = TRUE /\ durable' = FALSE
/\ attempts' = attempts + 1
+ /\ /\ witness' = FALSE /\ snapshot' = FALSE /\ orphanKilled' = FALSE
/\ UNCHANGED <>
Publish == /\ phase = "Starting" /\ ~durable
/\ durable' = TRUE
/\ UNCHANGED <>
+ identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
Spawn == /\ phase = "Starting" /\ durable /\ guard /\ locked
/\ phase' = "Running" /\ live' = TRUE
- /\ UNCHANGED <>
+ /\ /\ witness' = TRUE
+ /\ UNCHANGED <>
StartFailure == /\ phase = "Starting"
/\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE
/\ supervisor' = FALSE
/\ UNCHANGED <>
+ unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
SupervisorDeath == /\ supervisor /\ phase = "Running"
/\ supervisor' = FALSE /\ phase' = "Recoverable"
/\ UNCHANGED <>
+ identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
GuardDeath == /\ guard /\ live
/\ guard' = FALSE /\ locked' = FALSE
/\ phase' = "Orphaned" /\ supervisor' = FALSE
/\ UNCHANGED <>
+ unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
Retry == /\ ~supervisor /\ live /\ attempts < 2
/\ SkipLiveFence
/\ duplicate' = TRUE /\ attempts' = attempts + 1
/\ UNCHANGED <>
+ identity, unsafeSignal, witness, snapshot, orphanKilled>>
ReplaceIdentity == /\ identity = "match" /\ durable
/\ identity' \in {"foreign", "unknown"}
/\ UNCHANGED <>
+ supervisor, attempts, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
Stop == /\ phase \in {"Running", "Recoverable"}
/\ guard /\ durable /\ identity = "match"
/\ phase' = "Stopping"
/\ unsafeSignal' = (unsafeSignal \/ identity # "match")
/\ UNCHANGED <>
+ identity, duplicate, witness, snapshot, orphanKilled>>
Kill == /\ phase = "Stopping" /\ guard /\ identity = "match"
/\ phase' = "KillPending"
/\ unsafeSignal' = (unsafeSignal \/ identity # "match")
/\ UNCHANGED <>
+ identity, duplicate, witness, snapshot, orphanKilled>>
StopUnknown == /\ phase \in {"Stopping", "KillPending"} /\ live
/\ phase' = "StopUnknown"
/\ UNCHANGED <>
+ identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
ProcessExit == /\ live /\ live' = FALSE
/\ UNCHANGED <>
+ identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
Reap == /\ ~live /\ phase \notin {"Absent", "Starting", "Exited"}
/\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE
/\ supervisor' = FALSE
- /\ UNCHANGED <>
+ /\ UNCHANGED <>
\* Environment: after the old session empties, a new leader can take its
\* numeric SID, fork, and exit. The survivor has no old birth witness.
ReuseSessionID == /\ durable /\ ~guard /\ ~live
/\ identity = "match"
- /\ phase \in {"Orphaned", "Exited"}
+ /\ phase \in {"Orphaned", "Exited", "Stopping", "KillPending"}
/\ identity' = "sid-reused" /\ live' = TRUE
/\ UNCHANGED <>
+ attempts, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
RecoverOrphan == /\ phase = "Orphaned" /\ ~guard /\ live /\ durable
/\ (identity = "match" \/ (SkipOrphanFence /\ identity = "sid-reused"))
/\ unsafeSignal' = (unsafeSignal \/ identity # "match")
/\ phase' = "Stopping"
- /\ UNCHANGED <>
-Next == ReuseSessionID \/ RecoverOrphan \/ Reserve \/ Publish \/ Spawn \/ StartFailure \/ SupervisorDeath
+ /\ /\ (witness \/ SkipOrphanFence) /\ snapshot' = TRUE
+ /\ UNCHANGED <>
+WitnessExit == /\ phase = "Stopping" /\ ~guard /\ live /\ witness
+ /\ witness' = FALSE
+ /\ snapshot' = IF LoseRecoverySnapshot THEN FALSE ELSE snapshot
+ /\ UNCHANGED <>
+OrphanKill == /\ phase = "Stopping" /\ ~guard /\ live /\ snapshot
+ /\ identity = "match"
+ /\ phase' = "KillPending" /\ orphanKilled' = TRUE
+ /\ UNCHANGED <>
+Next == WitnessExit \/ OrphanKill \/ ReuseSessionID \/ RecoverOrphan \/ Reserve \/ Publish \/ Spawn \/ StartFailure \/ SupervisorDeath
\/ GuardDeath \/ Retry \/ ReplaceIdentity \/ Stop \/ Kill
\/ StopUnknown \/ ProcessExit \/ Reap
Spec == Init /\ [][Next]_vars
@@ -91,9 +105,13 @@ TypeOK == /\ phase \in {"Absent", "Starting", "Running", "Recoverable",
"Orphaned", "Stopping", "KillPending", "StopUnknown", "Exited"}
/\ attempts \in 0..2 /\ identity \in {"match", "foreign", "unknown", "sid-reused"}
/\ \A b \in {durable, guard, locked, live, supervisor,
- unsafeSignal, duplicate} : b \in BOOLEAN
+ unsafeSignal, duplicate, witness, snapshot, orphanKilled} : b \in BOOLEAN
DurableBeforeLaunch == live => durable
GuardRetainsLock == guard => locked
NoForeignSignal == ~unsafeSignal
NoDuplicate == ~duplicate
+RecoveryAuthorityRetained == (phase \in {"Stopping", "KillPending"} /\ ~guard
+ /\ live /\ identity = "match") => snapshot
+OrphanKillEnabled == (phase = "Stopping" /\ ~guard /\ live /\ ~witness
+ /\ identity = "match" /\ snapshot) => ENABLED OrphanKill
=============================================================================
diff --git a/tests/models/tla/lifecycle-supervisor/negative.cfg b/tests/models/tla/lifecycle-supervisor/negative.cfg
index 1fb04f6c8..7659fcd6c 100644
--- a/tests/models/tla/lifecycle-supervisor/negative.cfg
+++ b/tests/models/tla/lifecycle-supervisor/negative.cfg
@@ -1,5 +1,6 @@
SPECIFICATION Spec
+CONSTANT LoseRecoverySnapshot = FALSE
CONSTANT SkipOrphanFence = FALSE
CONSTANT SkipLiveFence = TRUE
-INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
+INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate RecoveryAuthorityRetained OrphanKillEnabled
CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-supervisor/positive.cfg b/tests/models/tla/lifecycle-supervisor/positive.cfg
index b687855fe..2383ec14c 100644
--- a/tests/models/tla/lifecycle-supervisor/positive.cfg
+++ b/tests/models/tla/lifecycle-supervisor/positive.cfg
@@ -1,5 +1,6 @@
SPECIFICATION Spec
+CONSTANT LoseRecoverySnapshot = FALSE
CONSTANT SkipOrphanFence = FALSE
CONSTANT SkipLiveFence = FALSE
-INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
+INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate RecoveryAuthorityRetained OrphanKillEnabled
CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg b/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg
index 06c12eaa4..02e0a9b41 100644
--- a/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg
+++ b/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg
@@ -1,5 +1,6 @@
SPECIFICATION Spec
CONSTANT SkipLiveFence = FALSE
+CONSTANT LoseRecoverySnapshot = FALSE
CONSTANT SkipOrphanFence = TRUE
-INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate
+INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate RecoveryAuthorityRetained OrphanKillEnabled
CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-supervisor/witness-loss.cfg b/tests/models/tla/lifecycle-supervisor/witness-loss.cfg
new file mode 100644
index 000000000..fb7e98a26
--- /dev/null
+++ b/tests/models/tla/lifecycle-supervisor/witness-loss.cfg
@@ -0,0 +1,6 @@
+SPECIFICATION Spec
+CONSTANT SkipLiveFence = FALSE
+CONSTANT SkipOrphanFence = FALSE
+CONSTANT LoseRecoverySnapshot = TRUE
+INVARIANTS TypeOK NoForeignSignal RecoveryAuthorityRetained OrphanKillEnabled
+CHECK_DEADLOCK FALSE
From 30dde3891c36010cdeb56404b0258e0795434096 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 07:59:10 -0500
Subject: [PATCH 16/45] test(supervisor): reap paused caller on failed zombie
assertion
---
.../lifecycle-supervisor/tests/fake_caller.rs | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index cccd2e75c..d4be3fadc 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1100,11 +1100,15 @@ fn process_is_zombie(pid: i32) -> bool {
#[test]
fn unreaped_matching_guard_proves_legacy_session_ownership() {
let f = Fixture::new();
- let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ let (caller, _) = f.start(vec![f.process("worker", "hold")]);
+ let mut caller = ForeignChild(caller);
f.await_log("fixture-ready");
let mut state = f.state("worker");
// Keep the parent from reaping its guard, then exercise legacy birth absence.
- assert_eq!(unsafe { libc::kill(caller.id() as i32, libc::SIGSTOP) }, 0);
+ assert_eq!(
+ unsafe { libc::kill(caller.0.id() as i32, libc::SIGSTOP) },
+ 0
+ );
assert_eq!(
unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
0
@@ -1117,8 +1121,11 @@ fn unreaped_matching_guard_proves_legacy_session_ownership() {
)
.unwrap();
f.stop();
- assert_eq!(unsafe { libc::kill(caller.id() as i32, libc::SIGCONT) }, 0);
- assert!(wait(&mut caller, TIMEOUT).success());
+ assert_eq!(
+ unsafe { libc::kill(caller.0.id() as i32, libc::SIGCONT) },
+ 0
+ );
+ assert!(wait(&mut caller.0, TIMEOUT).success());
}
#[test]
From 090013367b0da536f21a7a1b96b910815886d83b Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 08:12:20 -0500
Subject: [PATCH 17/45] test(supervisor): report receipt on failed owned stop
---
packages/lifecycle-supervisor/tests/fake_caller.rs | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index d4be3fadc..845e3a83e 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -93,7 +93,14 @@ impl Fixture {
let mut child = command(&self.root, "stop").spawn().unwrap();
send(child.stdin.as_mut().unwrap(), &self.control(TOKEN));
drop(child.stdin.take());
- assert!(wait(&mut child, TIMEOUT).success(), "owned stop failed");
+ let status = wait(&mut child, TIMEOUT);
+ if !status.success() {
+ // Receipts contain only identity/phase metadata, never argv/env.
+ // Read after the same exit deadline so diagnostic collection cannot
+ // change stop admission or hide a resumed-guard timing failure.
+ let receipt = receive(child.stdout.as_mut().unwrap());
+ panic!("owned stop failed: {receipt}");
+ }
}
fn state(&self, name: &str) -> Value {
From 71fa093c402fabceb7f93f9ba544d5d3bdf034e7 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 08:20:53 -0500
Subject: [PATCH 18/45] fix(supervisor): retain zombie guard birth proof on
Darwin
---
docs/lifecycle-supervisor.md | 5 ++
packages/lifecycle-supervisor/src/os.rs | 74 ++++++++++++++++++++-----
2 files changed, 65 insertions(+), 14 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index b1e066636..4c3e91a9e 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -260,6 +260,11 @@ the receipt has no command birth evidence, so an unanchored session is unknown.
rotation is allowed only after all original callers/guards and authored work
have been independently confirmed gone.
+Darwin may discard a zombie guard's session/group and credential metadata while
+retaining its PID and birth timestamp. A matching boot, PID and birth still prove
+the original guard has not been reused; this zombie never counts as live work
+and is never signalled. Live recovery members still require the recorded UID.
+
Darwin start timestamps come from the wall clock and can decrease between guard
and command creation. Recovery therefore never orders those timestamps: it matches
PID and the recorded birth exactly. A backward step between those starts does not
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index fec64c157..69ee12ba4 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -191,10 +191,13 @@ pub fn inspect(pid: i32) -> Result> {
let info = unsafe { info.assume_init() };
// SAFETY: getsid only observes the named PID. ESRCH is an exit race.
let mut session = unsafe { libc::getsid(pid) };
- if session < 0 && info.pbi_status == libc::SZOMB && info.pbi_pgid as i32 == pid {
- // Darwin may hide getsid for zombies. A matching recorded leader's
- // immutable PID/birth/UID still proves it has not been reused.
- session = pid;
+ if session < 0
+ && info.pbi_status == libc::SZOMB
+ && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH)
+ {
+ // A zombie can lose session/group/credential metadata during exit.
+ // Preserve its immutable PID/birth, but never invent live membership.
+ session = 0;
}
if session < 0 {
let error = io::Error::last_os_error();
@@ -339,6 +342,14 @@ fn incarnation_matches(expected: &Identity, current: &Identity) -> bool {
&& expected.session == current.session
}
+/// PID plus immutable birth pins the original zombie guard even when Darwin
+/// has discarded its session/credentials. Live leaders still require UID/SID.
+fn recorded_guard_matches(state: &crate::storage::State, p: &Identity) -> bool {
+ p.pid == state.guard_pid
+ && p.start == state.guard_start_time
+ && (p.zombie || (p.uid == state.uid && p.session == state.guard_pid))
+}
+
/// Return one census with session identity proved across that census. A matching
/// leader (including zombie) or exact recorded command birth is the witness.
/// No ordering comparison uses Darwin's adjustable wall-clock start timestamps.
@@ -351,9 +362,7 @@ fn recorded_members(state: &crate::storage::State) -> Result> {
}
let leader = inspect(state.guard_pid)?;
if let Some(p) = &leader
- && (p.start != state.guard_start_time
- || p.uid != state.uid
- || p.session != state.guard_pid)
+ && !recorded_guard_matches(state, p)
{
return Ok(Vec::new());
}
@@ -375,10 +384,13 @@ fn recorded_members(state: &crate::storage::State) -> Result> {
})
.ok_or_else(refused)?;
// Recheck after enumeration: the witness held the SID throughout the census.
- if !inspect(witness.pid)?
- .as_ref()
- .is_some_and(|p| incarnation_matches(witness, p))
- {
+ if !inspect(witness.pid)?.as_ref().is_some_and(|p| {
+ if witness.pid == state.guard_pid {
+ recorded_guard_matches(state, p)
+ } else {
+ incarnation_matches(witness, p)
+ }
+ }) {
continue;
}
return Ok(members);
@@ -407,9 +419,7 @@ pub fn observe_state(state: &crate::storage::State) -> Observation {
}
let leader = inspect(state.guard_pid)?;
if let Some(identity) = &leader
- && (identity.start != state.guard_start_time
- || identity.uid != state.uid
- || identity.session != state.guard_pid)
+ && !recorded_guard_matches(state, identity)
{
return Ok(Observation::Absent);
}
@@ -851,6 +861,42 @@ pub fn timestamp() -> Result {
#[cfg(test)]
mod tests {
+ #[test]
+ fn zombie_guard_birth_survives_discarded_exit_metadata() {
+ let state = crate::storage::State {
+ version: 1,
+ name: "worker".into(),
+ owner_token: "synthetic".into(),
+ guard_pid: 44,
+ process_group_id: 44,
+ guard_start_time: 100,
+ boot_id: String::new(),
+ uid: 501,
+ command_pid: None,
+ command_start_time: None,
+ phase: "running".into(),
+ exit_code: None,
+ signal: None,
+ };
+ let mut identity = super::Identity {
+ pid: 44,
+ group: 0,
+ session: 0,
+ start: 100,
+ uid: 0,
+ zombie: true,
+ };
+ assert!(super::recorded_guard_matches(&state, &identity));
+ identity.start += 1;
+ assert!(!super::recorded_guard_matches(&state, &identity));
+ identity.start = 100;
+ identity.zombie = false;
+ assert!(!super::recorded_guard_matches(&state, &identity));
+ identity.uid = state.uid;
+ identity.session = state.guard_pid;
+ assert!(super::recorded_guard_matches(&state, &identity));
+ }
+
#[test]
fn flock_interruption_retries_but_other_errors_do_not() {
let mut calls = 0;
From 7dca61724014fff13035fdd2f4f4be0529a19b94 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 09:33:27 -0500
Subject: [PATCH 19/45] fix(supervisor): inspect unreaped Darwin guard records
---
docs/lifecycle-supervisor.md | 9 ++++---
packages/lifecycle-supervisor/src/os.rs | 16 ++++++-----
.../lifecycle-supervisor/tests/fake_caller.rs | 27 ++++++++++++++-----
3 files changed, 36 insertions(+), 16 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 4c3e91a9e..8d816dff2 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -260,10 +260,11 @@ the receipt has no command birth evidence, so an unanchored session is unknown.
rotation is allowed only after all original callers/guards and authored work
have been independently confirmed gone.
-Darwin may discard a zombie guard's session/group and credential metadata while
-retaining its PID and birth timestamp. A matching boot, PID and birth still prove
-the original guard has not been reused; this zombie never counts as live work
-and is never signalled. Live recovery members still require the recorded UID.
+Darwin libproc requires a nonzero PROC_PIDTBSDINFO argument to include unreaped
+zombies. The inspector requests it explicitly. A matching boot, PID, birth and
+UID prove the original zombie guard has not been reused even when getsid rejects
+it; this zombie never counts as live work and is never signalled. Live guards
+and recovery members still require the recorded session and UID.
Darwin start timestamps come from the wall clock and can decrease between guard
and command creation. Recovery therefore never orders those timestamps: it matches
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 69ee12ba4..d9c708cc1 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -172,7 +172,7 @@ pub fn inspect(pid: i32) -> Result> {
libc::proc_pidinfo(
pid,
libc::PROC_PIDTBSDINFO,
- 0,
+ 1, // XNU requires nonzero arg to search unreaped zombie records.
info.as_mut_ptr().cast(),
std::mem::size_of::() as i32,
)
@@ -195,7 +195,7 @@ pub fn inspect(pid: i32) -> Result> {
&& info.pbi_status == libc::SZOMB
&& io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH)
{
- // A zombie can lose session/group/credential metadata during exit.
+ // Darwin getsid can reject zombies even when libproc returns birth/UID.
// Preserve its immutable PID/birth, but never invent live membership.
session = 0;
}
@@ -342,12 +342,13 @@ fn incarnation_matches(expected: &Identity, current: &Identity) -> bool {
&& expected.session == current.session
}
-/// PID plus immutable birth pins the original zombie guard even when Darwin
-/// has discarded its session/credentials. Live leaders still require UID/SID.
+/// PID/birth/UID pins the original zombie guard when Darwin getsid rejects it.
+/// Live leaders additionally require the recorded session.
fn recorded_guard_matches(state: &crate::storage::State, p: &Identity) -> bool {
p.pid == state.guard_pid
&& p.start == state.guard_start_time
- && (p.zombie || (p.uid == state.uid && p.session == state.guard_pid))
+ && p.uid == state.uid
+ && (p.zombie || p.session == state.guard_pid)
}
/// Return one census with session identity proved across that census. A matching
@@ -883,13 +884,16 @@ mod tests {
group: 0,
session: 0,
start: 100,
- uid: 0,
+ uid: 501,
zombie: true,
};
assert!(super::recorded_guard_matches(&state, &identity));
identity.start += 1;
assert!(!super::recorded_guard_matches(&state, &identity));
identity.start = 100;
+ identity.uid = 0;
+ assert!(!super::recorded_guard_matches(&state, &identity));
+ identity.uid = state.uid;
identity.zombie = false;
assert!(!super::recorded_guard_matches(&state, &identity));
identity.uid = state.uid;
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 845e3a83e..92e42b936 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1073,7 +1073,9 @@ fn orphan_recovery_keeps_observed_child_after_term_ends_witness() {
assert!(
census()
.into_iter()
- .all(|p| p.session != pid(&state, "guardPid") || process_is_zombie(p.pid))
+ .all(|p| p.session != pid(&state, "guardPid")
+ || identity(p.pid).is_none()
+ || process_is_zombie(p.pid))
);
}
@@ -1083,7 +1085,7 @@ fn process_is_zombie(pid: i32) -> bool {
fs::read_to_string(format!("/proc/{pid}/stat"))
.ok()
.and_then(|s| s.rsplit_once(") ").map(|(_, rest)| rest.starts_with('Z')))
- .unwrap_or(true)
+ .unwrap_or(false)
}
#[cfg(target_os = "macos")]
{
@@ -1093,14 +1095,13 @@ fn process_is_zombie(pid: i32) -> bool {
libc::proc_pidinfo(
pid,
libc::PROC_PIDTBSDINFO,
- 0,
+ 1,
info.as_mut_ptr().cast(),
std::mem::size_of::() as i32,
)
};
- size == 0
- || (size as usize == std::mem::size_of::()
- && unsafe { info.assume_init() }.pbi_status == libc::SZOMB)
+ size as usize == std::mem::size_of::()
+ && unsafe { info.assume_init() }.pbi_status == libc::SZOMB
}
}
@@ -1116,6 +1117,20 @@ fn unreaped_matching_guard_proves_legacy_session_ownership() {
unsafe { libc::kill(caller.0.id() as i32, libc::SIGSTOP) },
0
);
+ // SIGSTOP delivery is asynchronous. Observe our own child's stopped
+ // wait status before killing its guard, so it cannot win the reap race.
+ eventually(|| {
+ let mut status = 0;
+ // SAFETY: this is our unreaped child; status is writable for waitpid.
+ let observed = unsafe {
+ libc::waitpid(
+ caller.0.id() as i32,
+ &mut status,
+ libc::WNOHANG | libc::WUNTRACED,
+ )
+ };
+ (observed == caller.0.id() as i32 && libc::WIFSTOPPED(status)).then_some(())
+ });
assert_eq!(
unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
0
From f952a17733e26c7194bf3408ed78165a1974ca74 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 09:36:54 -0500
Subject: [PATCH 20/45] fix(supervisor): retain stop admission recovery
authority
---
docs/lifecycle-supervisor.md | 4 +++
docs/reference/v5-model-coverage.md | 4 +--
.../lifecycle-supervisor/src/supervisor.rs | 30 +++++++++-------
.../lifecycle-supervisor/tests/fake_caller.rs | 34 +++++++++++++++++++
scripts/lib/tla-runtime-models.ts | 2 +-
.../tla/lifecycle-supervisor/COVERAGE.md | 7 +++-
.../LifecycleSupervisor.tla | 9 ++---
7 files changed, 70 insertions(+), 20 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 8d816dff2..580627c03 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -46,6 +46,10 @@ A missing leader plus a numeric SID is not ownership: orphan recovery requires
the exact recorded command PID/birth or a matching guard, including a zombie.
Only unrecorded members, or legacy receipts without a surviving matching guard, report
unknown without signals. Descendants alone may require manual recovery when both recorded processes are gone.
+Stop captures verified members before signaling even a live guard. If that guard
+is lost during stop, recovery prefers a fresh capture and falls back to those
+admitted identities, rechecking each before TERM/KILL. Unobserved descendants
+still report unknown.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
`observed`, `stopped`, or `unknown`). A stop whose confirmation times out emits
diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md
index 737dca745..ab1ee7d2b 100644
--- a/docs/reference/v5-model-coverage.md
+++ b/docs/reference/v5-model-coverage.md
@@ -36,7 +36,7 @@ must be attached to the eventual exact integration revision.
| Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification |
| --- | --- | --- | --- |
-| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 322 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
+| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 426 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
| Interactive exec write-ahead admission, crash, EOF, replacement boot and retirement | interactive-session (459 states); no-replay, EOF-retirement, failed-inspect and wrong-boot negative controls | `interactive_session.rs` atomic intent/exec records, exact stopped-boot proof, known-undispatched withdrawal, restore-before-busy-retirement and run-scoped cleanup-report tests, and lock-free observation/revalidation tests; Engine effect fences; authored owner start and SCM_RIGHTS handoff | One session, two boot identities, two possible starts. Atomic durable publication and exact identity/absence proofs are assumptions; real fsync/power loss, real owner/Engine handoff and VM loss need Mac acceptance; receipt lock timing and terminal restoration are regression-covered, not modeled. Validated exact exec exit permits retirement; it makes no descendant cleanup claim. |
| Private interactive envelope, echo state and bytes still in flight on failure | interactive-envelope (15 states); restore-on-error negative control | HKEE2 launcher and host gate, Linux PTY seven-case matrix, bounded marker/ACK and coalesced-output tests | Two abstract private bytes, one handshake. PTY fixtures prove local termios ordering; Docker detach scanning, real guest wiring, same-user access and VM effects remain separate. Source/compiled Bun PTY and native-process fixtures cover lifetime/terminal boundaries, not model liveness. |
| Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. |
@@ -83,7 +83,7 @@ liveness/reachability controls have their own documented assumptions.
| Model | Positive distinct states | Configurations |
| --- | ---: | ---: |
-| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 322 | 4 |
+| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 426 | 4 |
| [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 |
| [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 |
| [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 |
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 589f6eb8c..5ba8426f1 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -466,11 +466,15 @@ fn stop(states: &[State]) -> Result<()> {
struct Target {
state: State,
recovery: Option,
+ admission: Option,
orphan_term: Option,
}
impl Target {
fn recover(&mut self) -> Result<()> {
- let recovery = os::Recovery::capture(&self.state)?;
+ // Prefer a fresh census, but retain already-proved identities if
+ // TERM ended the command witness before the guard was lost.
+ let recovery = os::Recovery::capture(&self.state)
+ .or_else(|error| self.admission.take().ok_or(error))?;
recovery.signal(libc::SIGTERM)?;
self.orphan_term = Some(Instant::now());
self.recovery = Some(recovery);
@@ -478,7 +482,7 @@ fn stop(states: &[State]) -> Result<()> {
}
}
let mut active = Vec::new();
- // Validate all receipts and capture orphan members before the first effect.
+ // Capture every target before effects, including live guards that may die mid-stop.
for state in states {
if !os::recorded_live(state, false)? {
continue;
@@ -490,13 +494,16 @@ fn stop(states: &[State]) -> Result<()> {
}
_ => false,
};
+ let captured = os::Recovery::capture(state)?;
+ let (recovery, admission) = if guard_live {
+ (None, Some(captured))
+ } else {
+ (Some(captured), None)
+ };
active.push(Target {
state: state.clone(),
- recovery: if guard_live {
- None
- } else {
- Some(os::Recovery::capture(state)?)
- },
+ recovery,
+ admission,
orphan_term: None,
});
}
@@ -517,14 +524,13 @@ fn stop(states: &[State]) -> Result<()> {
let mut live = false;
for target in &mut active {
if target.recovery.is_none() {
- if !os::recorded_live(&target.state, false)? {
- continue;
- }
if os::inspect(target.state.guard_pid)?.is_none_or(|p| p.zombie) {
- // Guard loss gets its own snapshot and full TERM grace.
+ // Guard loss gets a fresh census or the admission snapshot.
target.recover()?;
} else {
- live = true;
+ if os::recorded_live(&target.state, false)? {
+ live = true;
+ }
continue;
}
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 92e42b936..e34c5c668 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1037,6 +1037,40 @@ fn dead_guard_with_live_session_recovers_term_then_kill() {
assert_eq!(receipt["liveProcessNames"], json!([]));
}
+#[test]
+fn guard_lost_mid_stop_keeps_admission_snapshot() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "wrapper-term-child")]);
+ eventually(|| f.root.join("wrapper-child-ready").exists().then_some(()));
+ let state = f.state("worker");
+ let child = eventually(|| {
+ f.log().lines().find_map(|line| {
+ line.split("wrapper-child ")
+ .nth(1)
+ .and_then(|pid| pid.parse::().ok())
+ .and_then(identity)
+ })
+ });
+ f.owned.borrow_mut().push(child);
+ let mut stop = command(&f.root, "stop").spawn().unwrap();
+ send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ // The guard has acted on this stop once its wrapper is dead. Its five-second
+ // grace leaves time to remove the guard before it can KILL the child.
+ eventually(|| {
+ identity(pid(&state, "commandPid"))
+ .is_none_or(|_| process_is_zombie(pid(&state, "commandPid")))
+ .then_some(())
+ });
+ assert_eq!(
+ unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
+ 0
+ );
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let receipt = receive(stop.stdout.as_mut().unwrap());
+ assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "stopped");
+}
+
#[test]
fn orphan_recovery_keeps_observed_child_after_term_ends_witness() {
let f = Fixture::new();
diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts
index 97c17b6ff..f48ee4119 100644
--- a/scripts/lib/tla-runtime-models.ts
+++ b/scripts/lib/tla-runtime-models.ts
@@ -144,7 +144,7 @@ const contracts: readonly ModelContract[] = [
{
name: "lifecycle-supervisor",
module: "LifecycleSupervisor",
- states: 322,
+ states: 426,
invariant: "NoDuplicate",
action: "Retry",
fields: ["duplicate = TRUE", "live = TRUE", "durable = TRUE"],
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index b5af7fb7e..9b0e63589 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -13,7 +13,7 @@ standalone Rust binary exercised by a fake caller; the CLI remains unchanged.
| `ProcessExit`, `Reap` | Session census ignores zombies; reap direct/adopted children; release lock after no live members | Zombie liveness; child changes process group within session |
| `StopUnknown` | Bounded confirmation can report uncertainty while live members remain | Honest unknown recovery outcome, never successful absence |
-Positive TLC exploration: 322 distinct states, queue exhausted, depth 14. Negative
+Positive TLC exploration: 426 distinct states, queue exhausted, depth 14. Negative
control sets `SkipLiveFence = TRUE`; `Retry` then violates `NoDuplicate` with live,
durable and duplicate all true in the same state. Both guard-loss and project-loss
traces can witness this violation; the registry requires the named action and
@@ -87,3 +87,8 @@ an emptied session even during stopping; the old snapshot never authorizes a
new incarnation. The native wrapper/TERM-ignoring-child regression checks this
sequence with real processes. New descendants not in the snapshot remain an
uncovered recovery/unknown boundary; no eventual-exit claim is made.
+
+Stop now captures members while a live guard proves ownership as well. GuardDeath
+retains the stopping phase, and WitnessExit may happen before or after guard loss.
+The native guard_lost_mid_stop_keeps_admission_snapshot regression kills the guard
+after TERM ended its wrapper and checks KILL and confirmed absence for its child.
diff --git a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
index 1d16e3f22..e68d1b885 100644
--- a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
+++ b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla
@@ -38,7 +38,8 @@ SupervisorDeath == /\ supervisor /\ phase = "Running"
identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
GuardDeath == /\ guard /\ live
/\ guard' = FALSE /\ locked' = FALSE
- /\ phase' = "Orphaned" /\ supervisor' = FALSE
+ /\ phase' = IF phase = "Stopping" THEN "Stopping" ELSE "Orphaned"
+ /\ supervisor' = FALSE
/\ UNCHANGED <>
Retry == /\ ~supervisor /\ live /\ attempts < 2
@@ -52,10 +53,10 @@ ReplaceIdentity == /\ identity = "match" /\ durable
supervisor, attempts, unsafeSignal, duplicate, witness, snapshot, orphanKilled>>
Stop == /\ phase \in {"Running", "Recoverable"}
/\ guard /\ durable /\ identity = "match"
- /\ phase' = "Stopping"
+ /\ phase' = "Stopping" /\ snapshot' = TRUE
/\ unsafeSignal' = (unsafeSignal \/ identity # "match")
/\ UNCHANGED <>
+ identity, duplicate, witness, orphanKilled>>
Kill == /\ phase = "Stopping" /\ guard /\ identity = "match"
/\ phase' = "KillPending"
/\ unsafeSignal' = (unsafeSignal \/ identity # "match")
@@ -87,7 +88,7 @@ RecoverOrphan == /\ phase = "Orphaned" /\ ~guard /\ live /\ durable
/\ /\ (witness \/ SkipOrphanFence) /\ snapshot' = TRUE
/\ UNCHANGED <>
-WitnessExit == /\ phase = "Stopping" /\ ~guard /\ live /\ witness
+WitnessExit == /\ phase = "Stopping" /\ live /\ witness
/\ witness' = FALSE
/\ snapshot' = IF LoseRecoverySnapshot THEN FALSE ELSE snapshot
/\ UNCHANGED <
Date: Sat, 14 Nov 2026 09:44:08 -0500
Subject: [PATCH 21/45] fix(supervisor): preserve recovered stop grace and
guard admission
---
docs/lifecycle-supervisor.md | 7 ++-
.../lifecycle-supervisor/src/supervisor.rs | 43 +++++++++++++++----
.../lifecycle-supervisor/tests/fake_caller.rs | 13 ++++++
.../tla/lifecycle-supervisor/COVERAGE.md | 8 ++++
4 files changed, 61 insertions(+), 10 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 580627c03..33d8ba823 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -46,9 +46,12 @@ A missing leader plus a numeric SID is not ownership: orphan recovery requires
the exact recorded command PID/birth or a matching guard, including a zombie.
Only unrecorded members, or legacy receipts without a surviving matching guard, report
unknown without signals. Descendants alone may require manual recovery when both recorded processes are gone.
-Stop captures verified members before signaling even a live guard. If that guard
+Stop attempts to capture verified members before signaling even a live guard;
+a failed capture does not prevent that healthy guard from handling shutdown. If that guard
is lost during stop, recovery prefers a fresh capture and falls back to those
-admitted identities, rechecking each before TERM/KILL. Unobserved descendants
+admitted identities, rechecking each before TERM/KILL. Each recovered target gets
+its own five-second TERM grace and two-second confirmation bound. Recovery begins
+at most once per target; late guard loss can extend stop to roughly fourteen seconds. Unobserved descendants
still report unknown.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 5ba8426f1..c57f5fa9c 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -462,6 +462,19 @@ fn guard_command(
}
}
+// A live guard owns shutdown even when a complete recovery census is unavailable
+// (for example, a different-UID descendant). Orphans require strict capture.
+fn stop_capture(
+ guard_live: bool,
+ capture: impl FnOnce() -> Result,
+) -> Result<(Option, Option)> {
+ if guard_live {
+ Ok((None, capture().ok()))
+ } else {
+ Ok((Some(capture()?), None))
+ }
+}
+
fn stop(states: &[State]) -> Result<()> {
struct Target {
state: State,
@@ -494,12 +507,7 @@ fn stop(states: &[State]) -> Result<()> {
}
_ => false,
};
- let captured = os::Recovery::capture(state)?;
- let (recovery, admission) = if guard_live {
- (None, Some(captured))
- } else {
- (Some(captured), None)
- };
+ let (recovery, admission) = stop_capture(guard_live, || os::Recovery::capture(state))?;
active.push(Target {
state: state.clone(),
recovery,
@@ -520,7 +528,7 @@ fn stop(states: &[State]) -> Result<()> {
}
}
let start = Instant::now();
- while start.elapsed() < GRACE + Duration::from_secs(2) {
+ loop {
let mut live = false;
for target in &mut active {
if target.recovery.is_none() {
@@ -529,6 +537,9 @@ fn stop(states: &[State]) -> Result<()> {
target.recover()?;
} else {
if os::recorded_live(&target.state, false)? {
+ if start.elapsed() >= GRACE + Duration::from_secs(2) {
+ return Err(refused());
+ }
live = true;
}
continue;
@@ -538,6 +549,12 @@ fn stop(states: &[State]) -> Result<()> {
if !recovery.live()? {
continue;
}
+ if target
+ .orphan_term
+ .is_some_and(|since| since.elapsed() >= GRACE + Duration::from_secs(2))
+ {
+ return Err(refused());
+ }
live = true;
if target
.orphan_term
@@ -552,7 +569,6 @@ fn stop(states: &[State]) -> Result<()> {
}
thread::sleep(TICK);
}
- Err(refused())
}
fn detach_stdio() -> Result<()> {
@@ -652,6 +668,17 @@ fn incomplete_utf8_tail(bytes: &[u8]) -> usize {
mod tests {
use super::incomplete_utf8_tail;
+ #[test]
+ fn live_guard_admission_tolerates_unavailable_recovery_census() {
+ let denied = || Err::<(), _>(std::io::Error::from_raw_os_error(libc::EPERM));
+ let (recovery, admission) = super::stop_capture(true, denied).unwrap();
+ assert!(recovery.is_none() && admission.is_none());
+ assert!(super::stop_capture(false, denied).is_err());
+ let (recovery, admission) = super::stop_capture(true, || Ok(42)).unwrap();
+ assert!(recovery.is_none());
+ assert_eq!(admission, Some(42));
+ }
+
#[test]
fn chunk_boundary_retains_only_incomplete_codepoints() {
for text in ["a", "é", "€", "🦀"] {
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index e34c5c668..8e42f1057 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1039,6 +1039,15 @@ fn dead_guard_with_live_session_recovers_term_then_kill() {
#[test]
fn guard_lost_mid_stop_keeps_admission_snapshot() {
+ guard_lost_during_stop(Duration::ZERO);
+}
+
+#[test]
+fn guard_lost_late_in_stop_gets_full_recovery_deadline() {
+ guard_lost_during_stop(Duration::from_secs(3));
+}
+
+fn guard_lost_during_stop(delay: Duration) {
let f = Fixture::new();
let (mut caller, _) = f.start(vec![f.process("worker", "wrapper-term-child")]);
eventually(|| f.root.join("wrapper-child-ready").exists().then_some(()));
@@ -1061,6 +1070,8 @@ fn guard_lost_mid_stop_keeps_admission_snapshot() {
.is_none_or(|_| process_is_zombie(pid(&state, "commandPid")))
.then_some(())
});
+ thread::sleep(delay);
+ let recovery_started = Instant::now();
assert_eq!(
unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
0
@@ -1069,6 +1080,8 @@ fn guard_lost_mid_stop_keeps_admission_snapshot() {
let receipt = receive(stop.stdout.as_mut().unwrap());
assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}");
assert_eq!(receipt["outcome"], "stopped");
+ assert!(recovery_started.elapsed() >= Duration::from_secs(5));
+ assert!(recovery_started.elapsed() < Duration::from_secs(7));
}
#[test]
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index 9b0e63589..8f699fd69 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -92,3 +92,11 @@ Stop now captures members while a live guard proves ownership as well. GuardDeat
retains the stopping phase, and WitnessExit may happen before or after guard loss.
The native guard_lost_mid_stop_keeps_admission_snapshot regression kills the guard
after TERM ended its wrapper and checks KILL and confirmed absence for its child.
+
+Recovery deadlines are not modeled: each recovered target receives a fresh five-second
+TERM grace plus two seconds for confirmation, independently of the original guard
+wait. guard_lost_late_in_stop_gets_full_recovery_deadline kills the guard three
+seconds after wrapper death and checks the full recovery grace and successful
+KILL confirmation; the early-loss variant checks the same boundary without delay.
+Live-guard admission snapshots are best-effort; a denied census does not prevent
+the guard-owned shutdown path. Orphan capture stays strict (unit-tested admission seam).
From 2288c400c32228082a417b7769aaa61c00aa9813 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 09:51:56 -0500
Subject: [PATCH 22/45] fix(supervisor): recover guard loss during stop census
---
docs/lifecycle-supervisor.md | 3 +-
.../lifecycle-supervisor/src/supervisor.rs | 128 ++++++++++++------
.../lifecycle-supervisor/tests/fake_caller.rs | 46 +++++--
.../tla/lifecycle-supervisor/COVERAGE.md | 8 +-
4 files changed, 128 insertions(+), 57 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 33d8ba823..2c27c1b33 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -52,7 +52,8 @@ is lost during stop, recovery prefers a fresh capture and falls back to those
admitted identities, rechecking each before TERM/KILL. Each recovered target gets
its own five-second TERM grace and two-second confirmation bound. Recovery begins
at most once per target; late guard loss can extend stop to roughly fourteen seconds. Unobserved descendants
-still report unknown.
+still report unknown. A failed or expired target does not abandon other admitted
+targets: stop drives them to their own outcomes before returning unknown.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
`observed`, `stopped`, or `unknown`). A stop whose confirmation times out emits
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index c57f5fa9c..39bad47f1 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -475,12 +475,30 @@ fn stop_capture(
}
}
+fn guard_lost(state: &State) -> Result {
+ Ok(os::inspect(state.guard_pid)?.is_none_or(|p| p.zombie))
+}
+
+// None transfers to orphan recovery. A census error cannot bypass that transfer
+// when the guard disappears during the census or its witness recheck.
+fn guarded_live(state: &State) -> Result> {
+ if guard_lost(state)? {
+ return Ok(None);
+ }
+ match os::recorded_live(state, false) {
+ Ok(live) => Ok(Some(live)),
+ Err(_) if guard_lost(state)? => Ok(None),
+ Err(error) => Err(error),
+ }
+}
+
fn stop(states: &[State]) -> Result<()> {
struct Target {
state: State,
recovery: Option,
admission: Option,
orphan_term: Option,
+ finished: bool,
}
impl Target {
fn recover(&mut self) -> Result<()> {
@@ -493,79 +511,101 @@ fn stop(states: &[State]) -> Result<()> {
self.recovery = Some(recovery);
Ok(())
}
+
+ fn advance(&mut self, start: Instant) -> Result {
+ if self.recovery.is_none() {
+ match guarded_live(&self.state)? {
+ None => self.recover()?,
+ Some(false) => return Ok(false),
+ Some(true) => {
+ if start.elapsed() >= GRACE + Duration::from_secs(2) {
+ return Err(refused());
+ }
+ return Ok(true);
+ }
+ }
+ }
+ let recovery = self.recovery.as_ref().ok_or_else(refused)?;
+ if !recovery.live()? {
+ return Ok(false);
+ }
+ let elapsed = self.orphan_term.ok_or_else(refused)?.elapsed();
+ if elapsed >= GRACE + Duration::from_secs(2) {
+ return Err(refused());
+ }
+ if elapsed >= GRACE {
+ recovery.signal(libc::SIGKILL)?;
+ }
+ Ok(true)
+ }
}
let mut active = Vec::new();
- // Capture every target before effects, including live guards that may die mid-stop.
+ // Validate admission before effects. Capture while a live guard can still
+ // prove membership, so loss during the later liveness check retains authority.
for state in states {
- if !os::recorded_live(state, false)? {
- continue;
- }
let guard_live = match os::inspect(state.guard_pid)? {
Some(identity) if !identity.zombie => {
- os::same(state, &identity)?;
+ if let Err(error) = os::same(state, &identity) {
+ if !os::recorded_live(state, false)? {
+ continue;
+ }
+ return Err(error);
+ }
true
}
_ => false,
};
let (recovery, admission) = stop_capture(guard_live, || os::Recovery::capture(state))?;
+ match os::recorded_live(state, false) {
+ Ok(false) => continue,
+ Ok(true) => (),
+ Err(_) if guard_lost(state)? && (recovery.is_some() || admission.is_some()) => (),
+ Err(error) => return Err(error),
+ }
active.push(Target {
state: state.clone(),
recovery,
admission,
orphan_term: None,
+ finished: false,
});
}
+ let mut failure = None;
for target in &mut active {
- if let Some(recovery) = &target.recovery {
- recovery.signal(libc::SIGTERM)?;
+ let signalled = if let Some(recovery) = &target.recovery {
target.orphan_term = Some(Instant::now());
- } else if let Err(error) = os::signal_guard(&target.state) {
- if os::inspect(target.state.guard_pid)?.is_none_or(|p| p.zombie) {
- target.recover()?;
- } else if os::recorded_live(&target.state, false)? {
- return Err(error);
+ recovery.signal(libc::SIGTERM)
+ } else {
+ match os::signal_guard(&target.state) {
+ Ok(()) => Ok(()),
+ Err(error) => match guarded_live(&target.state) {
+ Ok(None) => target.recover(),
+ Ok(Some(false)) => Ok(()),
+ Ok(Some(true)) => Err(error),
+ Err(error) => Err(error),
+ },
}
+ };
+ if let Err(error) = signalled {
+ target.finished = true;
+ failure.get_or_insert(error);
}
}
let start = Instant::now();
loop {
let mut live = false;
- for target in &mut active {
- if target.recovery.is_none() {
- if os::inspect(target.state.guard_pid)?.is_none_or(|p| p.zombie) {
- // Guard loss gets a fresh census or the admission snapshot.
- target.recover()?;
- } else {
- if os::recorded_live(&target.state, false)? {
- if start.elapsed() >= GRACE + Duration::from_secs(2) {
- return Err(refused());
- }
- live = true;
- }
- continue;
- }
- }
- if let Some(recovery) = &target.recovery {
- if !recovery.live()? {
- continue;
- }
- if target
- .orphan_term
- .is_some_and(|since| since.elapsed() >= GRACE + Duration::from_secs(2))
- {
- return Err(refused());
- }
- live = true;
- if target
- .orphan_term
- .is_some_and(|since| since.elapsed() >= GRACE)
- {
- recovery.signal(libc::SIGKILL)?;
+ for target in active.iter_mut().filter(|target| !target.finished) {
+ match target.advance(start) {
+ Ok(true) => live = true,
+ Ok(false) => target.finished = true,
+ Err(error) => {
+ target.finished = true;
+ failure.get_or_insert(error);
}
}
}
if !live {
- return Ok(());
+ return failure.map_or(Ok(()), Err);
}
thread::sleep(TICK);
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 8e42f1057..eaf374e22 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1039,17 +1039,30 @@ fn dead_guard_with_live_session_recovers_term_then_kill() {
#[test]
fn guard_lost_mid_stop_keeps_admission_snapshot() {
- guard_lost_during_stop(Duration::ZERO);
+ guard_lost_during_stop(Duration::ZERO, false);
}
#[test]
fn guard_lost_late_in_stop_gets_full_recovery_deadline() {
- guard_lost_during_stop(Duration::from_secs(3));
+ guard_lost_during_stop(Duration::from_secs(3), false);
}
-fn guard_lost_during_stop(delay: Duration) {
+#[test]
+fn expired_target_does_not_abandon_other_recovery() {
+ guard_lost_during_stop(Duration::from_secs(3), true);
+}
+
+fn guard_lost_during_stop(delay: Duration, blocked: bool) {
let f = Fixture::new();
- let (mut caller, _) = f.start(vec![f.process("worker", "wrapper-term-child")]);
+ let mut processes = vec![f.process("worker", "wrapper-term-child")];
+ if blocked {
+ processes.push(f.process("blocked", "hold"));
+ }
+ let (mut caller, _) = f.start(processes);
+ let blocked_guard = blocked.then(|| pid(&f.state("blocked"), "guardPid"));
+ if let Some(guard) = blocked_guard {
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
+ }
eventually(|| f.root.join("wrapper-child-ready").exists().then_some(()));
let state = f.state("worker");
let child = eventually(|| {
@@ -1060,7 +1073,7 @@ fn guard_lost_during_stop(delay: Duration) {
.and_then(identity)
})
});
- f.owned.borrow_mut().push(child);
+ f.owned.borrow_mut().push(child.clone());
let mut stop = command(&f.root, "stop").spawn().unwrap();
send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
// The guard has acted on this stop once its wrapper is dead. Its five-second
@@ -1076,12 +1089,25 @@ fn guard_lost_during_stop(delay: Duration) {
unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
0
);
- assert!(wait(&mut caller, TIMEOUT).success());
+ if !blocked {
+ assert!(wait(&mut caller, TIMEOUT).success());
+ }
let receipt = receive(stop.stdout.as_mut().unwrap());
- assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}");
- assert_eq!(receipt["outcome"], "stopped");
- assert!(recovery_started.elapsed() >= Duration::from_secs(5));
- assert!(recovery_started.elapsed() < Duration::from_secs(7));
+ let stopped = wait(&mut stop, TIMEOUT).success();
+ assert_eq!(stopped, !blocked, "{receipt}");
+ assert_eq!(
+ receipt["outcome"],
+ if blocked { "unknown" } else { "stopped" }
+ );
+ // Scheduling may let the guard itself finish escalation. Either path must
+ // confirm absence; do not mistake test observation latency for grace timing.
+ assert!(recovery_started.elapsed() < TIMEOUT);
+ assert!(identity(child.pid).is_none() || process_is_zombie(child.pid));
+ if let Some(guard) = blocked_guard {
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGCONT) }, 0);
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+ }
}
#[test]
diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
index 8f699fd69..400a0eebf 100644
--- a/tests/models/tla/lifecycle-supervisor/COVERAGE.md
+++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md
@@ -96,7 +96,11 @@ after TERM ended its wrapper and checks KILL and confirmed absence for its child
Recovery deadlines are not modeled: each recovered target receives a fresh five-second
TERM grace plus two seconds for confirmation, independently of the original guard
wait. guard_lost_late_in_stop_gets_full_recovery_deadline kills the guard three
-seconds after wrapper death and checks the full recovery grace and successful
-KILL confirmation; the early-loss variant checks the same boundary without delay.
+seconds after wrapper death and checks successful KILL confirmation and child absence; the early-loss variant checks the same boundary without delay.
Live-guard admission snapshots are best-effort; a denied census does not prevent
the guard-owned shutdown path. Orphan capture stays strict (unit-tested admission seam).
+
+The census-to-guard-loss handoff and multi-target timeout scheduling are native
+implementation boundaries, not modeled. guarded_live rechecks guard loss after
+a failed census. expired_target_does_not_abandon_other_recovery checks that one
+paused guard timing out does not prevent another target from completing KILL.
From 59bd8d52923e9e32f47d0578a3ffcced8c69a464 Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 12:12:54 -0500
Subject: [PATCH 23/45] test(supervisor): make guard-loss recovery regressions
deterministic
Freeze the guard before removing it and prove it has not begun its own
escalation and that the TERM-ignoring child is still live, so only the
stop's admission snapshot can KILL the child. Require a fresh five-second
recovery grace after guard loss. Align the documented model summary with
the maintained 426-state coverage.
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 6 ++--
.../lifecycle-supervisor/tests/fake_caller.rs | 31 +++++++++++++++----
2 files changed, 29 insertions(+), 8 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 2c27c1b33..3221004d8 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -207,8 +207,10 @@ the exact 1,048,576 test requires a host that permits that limit.
Darwin compilation does not qualify execution, group-signal EPERM recovery or
idle cost on a real Mac.
-The 322-state TLA+ artifact is an abstract phase/ownership exploration. It models SID reuse and a required old-birth witness for orphan signaling. It does
-not model actual kernel census observations, real flock behavior, full orphan recovery,
+The 426-state TLA+ artifact is an abstract phase/ownership exploration. It models
+SID reuse, an exact recorded-incarnation witness for orphan signaling, witness loss
+after capture and the retained-snapshot KILL step. It does
+not model actual kernel census observations, real flock behavior, recovery timing,
startup descriptor leases or signal errors. Some safety predicates hold by
construction. Native regressions, rather than that model, are the evidence for
those boundaries; see the model coverage document for the reduced claims.
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index eaf374e22..420ded1da 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1084,11 +1084,24 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
.then_some(())
});
thread::sleep(delay);
- let recovery_started = Instant::now();
+ // Freeze the guard before removing it, then prove it has not begun its own
+ // escalation: it persists kill-pending before its first KILL. The child must
+ // still be live, so only the stop's admission snapshot can KILL it.
+ let guard = pid(&state, "guardPid");
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
+ let frozen = f.state("worker");
assert_eq!(
- unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
- 0
+ frozen["phase"], "stopping",
+ "guard escalated first: {frozen}"
);
+ assert!(
+ identity(child.pid).is_some_and(|current| current.birth == child.birth)
+ && !process_is_zombie(child.pid),
+ "TERM-ignoring child exited before guard loss"
+ );
+ let recovery_started = Instant::now();
+ // SAFETY: the frozen guard belongs to this fixture.
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
if !blocked {
assert!(wait(&mut caller, TIMEOUT).success());
}
@@ -1099,9 +1112,15 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
receipt["outcome"],
if blocked { "unknown" } else { "stopped" }
);
- // Scheduling may let the guard itself finish escalation. Either path must
- // confirm absence; do not mistake test observation latency for grace timing.
- assert!(recovery_started.elapsed() < TIMEOUT);
+ // Recovery starts its own TERM grace after detecting guard loss, so KILL of
+ // the captured child cannot precede a fresh five-second interval, even when
+ // the original guard deadline had almost expired.
+ let recovered_after = recovery_started.elapsed();
+ assert!(
+ recovered_after >= Duration::from_secs(5),
+ "recovery skipped its fresh grace: {recovered_after:?}"
+ );
+ assert!(recovered_after < TIMEOUT);
assert!(identity(child.pid).is_none() || process_is_zombie(child.pid));
if let Some(guard) = blocked_guard {
assert_eq!(unsafe { libc::kill(guard, libc::SIGCONT) }, 0);
From 98d0b0a401ba0864b86ded6257782a01c78eb80b Mon Sep 17 00:00:00 2001
From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com>
Date: Sat, 14 Nov 2026 12:15:09 -0500
Subject: [PATCH 24/45] test(supervisor): wait for kernel-observed guard stop
before guard loss
kill(SIGSTOP) only submits the signal. Wait until the kernel reports the
same guard incarnation stopped (Linux /proc state, Darwin SSTOP) before
checking its persisted phase and the child's liveness, so the guard cannot
escalate after those observations.
Co-Authored-By: Claude Opus 5.5
---
.../lifecycle-supervisor/tests/fake_caller.rs | 34 ++++++++++++++++---
1 file changed, 30 insertions(+), 4 deletions(-)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 420ded1da..ce7122580 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1088,7 +1088,15 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
// escalation: it persists kill-pending before its first KILL. The child must
// still be live, so only the stop's admission snapshot can KILL it.
let guard = pid(&state, "guardPid");
+ let guard_birth = identity(guard).expect("guard identity").birth;
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
+ // SIGSTOP delivery is asynchronous: wait until the kernel reports this
+ // guard incarnation stopped before trusting the observations below.
+ eventually(|| {
+ (identity(guard).is_some_and(|current| current.birth == guard_birth)
+ && process_is_stopped(guard))
+ .then_some(())
+ });
let frozen = f.state("worker");
assert_eq!(
frozen["phase"], "stopping",
@@ -1172,12 +1180,24 @@ fn orphan_recovery_keeps_observed_child_after_term_ends_witness() {
}
fn process_is_zombie(pid: i32) -> bool {
+ process_state(pid) == Some(b'Z')
+}
+
+/// Job-control stop observed by the kernel, not merely a submitted SIGSTOP.
+fn process_is_stopped(pid: i32) -> bool {
+ matches!(process_state(pid), Some(b'T' | b't'))
+}
+
+/// Linux /proc state letter; Darwin maps SZOMB to Z and SSTOP to T.
+fn process_state(pid: i32) -> Option {
#[cfg(target_os = "linux")]
{
fs::read_to_string(format!("/proc/{pid}/stat"))
.ok()
- .and_then(|s| s.rsplit_once(") ").map(|(_, rest)| rest.starts_with('Z')))
- .unwrap_or(false)
+ .and_then(|s| {
+ s.rsplit_once(") ")
+ .and_then(|(_, rest)| rest.bytes().next())
+ })
}
#[cfg(target_os = "macos")]
{
@@ -1192,8 +1212,14 @@ fn process_is_zombie(pid: i32) -> bool {
std::mem::size_of::() as i32,
)
};
- size as usize == std::mem::size_of::()
- && unsafe { info.assume_init() }.pbi_status == libc::SZOMB
+ if size as usize != std::mem::size_of::() {
+ return None;
+ }
+ Some(match unsafe { info.assume_init() }.pbi_status {
+ libc::SZOMB => b'Z',
+ libc::SSTOP => b'T',
+ _ => b'R',
+ })
}
}
From 416225380ef509b894f688894052f4523890fb35 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 02:29:07 +0000
Subject: [PATCH 25/45] fix(supervisor): treat pidfd_open EINVAL for an exited
group leader as absent
On Linux 6.8, pidfd_open returns EINVAL rather than ESRCH for a PID that
survives only as a process-group or session ID after its process exited.
A recovery stop whose wrapper died on TERM then failed before sending KILL
to the wrapper's TERM-ignoring child and reported unknown. Flags are zero,
so EINVAL means no thread-group leader exists; skip that member and let
the identity-checked liveness pass decide the outcome.
Four recovery fake-caller tests failed on a 6.8 host without this change
and pass with it.
Co-Authored-By: Claude Opus 5.5
---
packages/lifecycle-supervisor/src/os.rs | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index d9c708cc1..d739d344b 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -467,7 +467,11 @@ fn signal_member(expected: &Identity, signal: i32) -> Result<()> {
let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, expected.pid, 0u32) };
if raw < 0 {
let error = io::Error::last_os_error();
- if absent_error(&error) {
+ // Some kernels (observed on 6.8) return EINVAL, not ESRCH, for a PID
+ // kept alive only as a process-group or session ID after its process
+ // exited. Flags are zero, so EINVAL means no thread-group leader
+ // exists for that PID; liveness is still decided by `inspect` later.
+ if absent_error(&error) || error.raw_os_error() == Some(libc::EINVAL) {
return Ok(());
}
return Err(error);
From 8c23ab29e244d1fff079059cd7602deb638eca10 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 02:43:06 +0000
Subject: [PATCH 26/45] fix(supervisor): wait out unobserved descendants and
outlast a stop holder
Orphan recovery reported unknown on the first tick after its captured
members died while a descendant forked after the capture was still alive,
even when that descendant was about to exit by itself. Recovery now keeps
the target live until its own deadline. It still never signals members it
did not capture, and still reports unknown if they outlive the deadline.
A recovery stop can hold .stop.lock for two TERM-to-KILL bounds (about
14 s). Waiters on that lock gave up after 10 s with no receipt. The stop
lock wait is now derived from that worst case (16 s); the command lock
keeps its 10 s bound.
guard_lost_during_stop now freezes the stop request until the killed guard
has no process record, so a zombie guard cannot witness a fresh capture
and the tests exercise the admission-snapshot fallback.
Co-Authored-By: Claude Opus 5.5
---
packages/lifecycle-supervisor/src/os.rs | 21 ++-
packages/lifecycle-supervisor/src/storage.rs | 4 +-
.../lifecycle-supervisor/src/supervisor.rs | 31 +++--
.../lifecycle-supervisor/tests/fake_caller.rs | 129 +++++++++++++++++-
4 files changed, 169 insertions(+), 16 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index d739d344b..1cd463fc3 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -761,21 +761,34 @@ impl Recovery {
Ok(())
}
- pub fn live(&self) -> Result {
+ pub fn live(&self) -> Result {
for member in &self.members {
if inspect(member.pid)?.is_some_and(|p| !p.zombie && incarnation_matches(member, &p)) {
- return Ok(true);
+ return Ok(RecoveryLiveness::Captured);
}
}
// Unobserved descendants or an unrelated reused session cannot be
// signalled by this snapshot, and must not be claimed as stopped.
if live_group(self.session, Some(self.session))? {
- return Err(refused());
+ return Ok(RecoveryLiveness::Unobserved);
}
- Ok(false)
+ Ok(RecoveryLiveness::Gone)
}
}
+/// What a recovery snapshot can still see of its session.
+#[derive(Debug, PartialEq, Eq)]
+pub enum RecoveryLiveness {
+ /// A captured incarnation is alive; it may still be signalled.
+ Captured,
+ /// No captured incarnation is alive, but the session still has members this
+ /// snapshot never observed. They may exit by themselves; they are never
+ /// signalled, and stop reports unknown if they outlive its deadline.
+ Unobserved,
+ /// The session has no live member.
+ Gone,
+}
+
/// A successful census must contain our own readable identity. Known restricted
/// proc mounts are unsupported: hidden same-user members defeat absence proofs.
pub fn census_preflight() -> Result<()> {
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index 8a17d2514..e2d0ad6fa 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -117,9 +117,9 @@ impl Root {
Ok(file)
}
- pub fn lock_wait(&self, name: &str) -> Result {
+ pub fn lock_wait(&self, name: &str, wait: std::time::Duration) -> Result {
let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?;
- let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
+ let deadline = std::time::Instant::now() + wait;
loop {
// SAFETY: flock only borrows the live owned descriptor.
if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 39bad47f1..ab09de1db 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -21,6 +21,15 @@ use std::{
const GRACE: Duration = Duration::from_secs(5);
const IDLE_TICK: Duration = Duration::from_millis(500);
const TICK: Duration = Duration::from_millis(20);
+/// One stop holds `.stop.lock` for at most two TERM-to-KILL bounds: the guard
+/// path, then orphan recovery after a late guard loss. A waiter must outlast
+/// that holder instead of failing without a receipt.
+const STOP_LOCK_WAIT: Duration =
+ Duration::from_secs(2 * (GRACE.as_secs() + CONFIRM_BOUND.as_secs()) + 2);
+/// Bounded wait for a previous command owner to release its lock.
+const COMMAND_LOCK_WAIT: Duration = Duration::from_secs(10);
+/// Time after KILL for a stop to confirm absence.
+const CONFIRM_BOUND: Duration = Duration::from_secs(2);
const LINE_LIMIT: usize = 64 * 1024;
#[derive(Serialize)]
@@ -70,7 +79,7 @@ pub fn main() -> Result<()> {
protocol::validate_control(request.version, &request.root, &request.owner_token)?;
let root = Root::open(&request.root)?;
let _stop_lock = if args[0] == "stop" {
- Some(root.lock_wait(".stop.lock")?)
+ Some(root.lock_wait(".stop.lock", STOP_LOCK_WAIT)?)
} else {
None
};
@@ -135,7 +144,7 @@ fn run(request: Launch) -> Result<()> {
let root = Root::open(&request.root)?;
let _instance_lock = instance_lock(&root)?;
os::census_preflight()?;
- let launch_lock = root.lock_wait(".stop.lock")?;
+ let launch_lock = root.lock_wait(".stop.lock", STOP_LOCK_WAIT)?;
root.assert_not_retired(&request.owner_token)?;
// Validate every old claim before starting any new command. A guard survives
// project-owner death; its command lock also closes concurrent restart races.
@@ -199,7 +208,7 @@ fn run(request: Launch) -> Result<()> {
detach_stdio()?;
loop {
if os::STOP.load(Ordering::Relaxed) {
- match root.lock_wait(".stop.lock") {
+ match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) {
Ok(_lock) => {
stop(&root.states()?)?;
}
@@ -264,7 +273,7 @@ fn rollback(children: &mut [Child]) {
libc::kill(pid, libc::SIGTERM);
}
}
- let deadline = Instant::now() + GRACE + Duration::from_secs(2);
+ let deadline = Instant::now() + GRACE + CONFIRM_BOUND;
while Instant::now() < deadline {
if children
.iter_mut()
@@ -282,7 +291,8 @@ fn guard(request: GuardLaunch) -> Result<()> {
protocol::validate_control(request.version, &request.root, &request.owner_token)?;
request.process.validate()?;
let root = Root::open(&request.root)?;
- let _command_lock = root.lock_wait(&format!("{}.lock", request.process.name))?;
+ let _command_lock =
+ root.lock_wait(&format!("{}.lock", request.process.name), COMMAND_LOCK_WAIT)?;
if let Some(previous) = root.load(&request.process.name)?
&& os::recorded_live(&previous, false)?
{
@@ -518,7 +528,7 @@ fn stop(states: &[State]) -> Result<()> {
None => self.recover()?,
Some(false) => return Ok(false),
Some(true) => {
- if start.elapsed() >= GRACE + Duration::from_secs(2) {
+ if start.elapsed() >= GRACE + CONFIRM_BOUND {
return Err(refused());
}
return Ok(true);
@@ -526,14 +536,17 @@ fn stop(states: &[State]) -> Result<()> {
}
}
let recovery = self.recovery.as_ref().ok_or_else(refused)?;
- if !recovery.live()? {
+ let liveness = recovery.live()?;
+ if liveness == os::RecoveryLiveness::Gone {
return Ok(false);
}
let elapsed = self.orphan_term.ok_or_else(refused)?.elapsed();
- if elapsed >= GRACE + Duration::from_secs(2) {
+ if elapsed >= GRACE + CONFIRM_BOUND {
return Err(refused());
}
- if elapsed >= GRACE {
+ // Only captured incarnations are signalled. Unobserved members get
+ // the rest of this target's deadline to exit on their own.
+ if elapsed >= GRACE && liveness == os::RecoveryLiveness::Captured {
recovery.signal(libc::SIGKILL)?;
}
Ok(true)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index ce7122580..656716da7 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -8,7 +8,7 @@ use std::{
os::{fd::AsRawFd, unix::fs::MetadataExt},
path::{Path, PathBuf},
process::{Child, Command, Stdio},
- sync::atomic::{AtomicU64, Ordering},
+ sync::atomic::{AtomicBool, AtomicU64, Ordering},
thread,
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
@@ -1108,11 +1108,27 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
"TERM-ignoring child exited before guard loss"
);
let recovery_started = Instant::now();
+ // Freeze stop too, so it cannot observe the guard as an unreaped zombie:
+ // a zombie guard is still a witness, and a fresh capture would then succeed
+ // without the admission snapshot this test is about.
+ let stop_pid = stop.id() as i32;
+ // SAFETY: the stop request is this test's own child.
+ assert_eq!(unsafe { libc::kill(stop_pid, libc::SIGSTOP) }, 0);
+ eventually(|| process_is_stopped(stop_pid).then_some(()));
// SAFETY: the frozen guard belongs to this fixture.
assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
if !blocked {
assert!(wait(&mut caller, TIMEOUT).success());
}
+ // Resume stop only after the guard incarnation has no process record left,
+ // so neither a live nor a zombie guard can witness a fresh capture.
+ eventually(|| {
+ identity(guard)
+ .is_none_or(|current| current.birth != guard_birth)
+ .then_some(())
+ });
+ // SAFETY: resumes the stop request stopped above.
+ assert_eq!(unsafe { libc::kill(stop_pid, libc::SIGCONT) }, 0);
let receipt = receive(stop.stdout.as_mut().unwrap());
let stopped = wait(&mut stop, TIMEOUT).success();
assert_eq!(stopped, !blocked, "{receipt}");
@@ -1179,6 +1195,71 @@ fn orphan_recovery_keeps_observed_child_after_term_ends_witness() {
);
}
+/// Recovery captures the session before its TERM. A descendant forked after
+/// that capture is never signalled; stop waits until its own deadline for it.
+fn late_descendant_case(forever: bool) {
+ let f = Fixture::new();
+ let mode = if forever {
+ "fork-on-term-forever"
+ } else {
+ "fork-on-term"
+ };
+ let (mut caller, _) = f.start(vec![f.process("worker", mode)]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ // SAFETY: the captured guard belongs to this fixture; its command survives.
+ assert_eq!(
+ unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
+ 0
+ );
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let start = Instant::now();
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ let stopped = wait(&mut stop, TIMEOUT).success();
+ let elapsed = start.elapsed();
+ let late = std::fs::read_to_string(f.root.join("late-descendant"))
+ .ok()
+ .and_then(|value| value.parse::().ok())
+ .expect("command did not fork after recovery TERM");
+ let late_identity = identity(late);
+ if let Some(late) = &late_identity {
+ f.owned.borrow_mut().push(late.clone());
+ }
+ assert!(
+ identity(pid(&state, "commandPid")).is_none()
+ || process_is_zombie(pid(&state, "commandPid")),
+ "captured command survived recovery KILL"
+ );
+ if forever {
+ assert!(!stopped, "{receipt}");
+ assert_eq!(receipt["outcome"], "unknown");
+ assert!(
+ elapsed >= Duration::from_secs(7),
+ "stop gave up before its deadline: {elapsed:?}"
+ );
+ let late = late_identity.expect("unobserved descendant was signalled");
+ assert!(
+ !process_is_zombie(late.pid),
+ "unobserved descendant was signalled"
+ );
+ } else {
+ assert!(stopped, "{receipt}");
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(elapsed >= Duration::from_secs(5));
+ assert!(elapsed < TIMEOUT);
+ }
+}
+
+#[test]
+fn orphan_recovery_waits_for_unobserved_descendant_to_exit() {
+ late_descendant_case(false);
+}
+
+#[test]
+fn orphan_recovery_never_signals_unobserved_descendant() {
+ late_descendant_case(true);
+}
+
fn process_is_zombie(pid: i32) -> bool {
process_state(pid) == Some(b'Z')
}
@@ -1904,6 +1985,52 @@ fn fixture_process() {
}
println!("wrapper-child {child}");
}
+ "fork-on-term" | "fork-on-term-forever" => {
+ // Fork one TERM-ignoring descendant after each TERM, so recovery
+ // meets a session member forked after its capture. Its PID goes to
+ // a file: stdout belongs to the guard, which the test kills first.
+ static TERMINATED: AtomicBool = AtomicBool::new(false);
+ extern "C" fn on_term(_: libc::c_int) {
+ TERMINATED.store(true, Ordering::SeqCst);
+ }
+ let forever = mode == "fork-on-term-forever";
+ // SAFETY: the handler only stores an atomic flag.
+ unsafe {
+ libc::signal(
+ libc::SIGTERM,
+ on_term as extern "C" fn(libc::c_int) as libc::sighandler_t,
+ );
+ }
+ println!("fixture-ready");
+ std::io::stdout().flush().unwrap();
+ loop {
+ if TERMINATED.swap(false, Ordering::SeqCst) {
+ // SAFETY: the fork child uses only async-signal-safe libc
+ // calls and never returns into Rust state.
+ let child = unsafe { libc::fork() };
+ assert!(child >= 0);
+ if child == 0 {
+ unsafe {
+ libc::signal(libc::SIGTERM, libc::SIG_IGN);
+ if forever {
+ loop {
+ libc::pause();
+ }
+ }
+ // Outlive the parent's KILL at five seconds, then
+ // exit before the seven-second recovery deadline.
+ for _ in 0..12 {
+ libc::usleep(500_000);
+ }
+ libc::_exit(0);
+ }
+ }
+ std::fs::write("late-descendant.tmp", child.to_string()).unwrap();
+ std::fs::rename("late-descendant.tmp", "late-descendant").unwrap();
+ }
+ thread::sleep(Duration::from_millis(10));
+ }
+ }
"hold" => (),
_ => std::process::exit(99),
}
From c8564d09020fa6b44e61b432804e19a96b1a24e6 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 02:44:42 +0000
Subject: [PATCH 27/45] docs(supervisor): describe unobserved-descendant wait
and stop lock bound
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 9 ++++++---
packages/lifecycle-supervisor/README.md | 4 +++-
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 3221004d8..465e06287 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -51,8 +51,11 @@ a failed capture does not prevent that healthy guard from handling shutdown. If
is lost during stop, recovery prefers a fresh capture and falls back to those
admitted identities, rechecking each before TERM/KILL. Each recovered target gets
its own five-second TERM grace and two-second confirmation bound. Recovery begins
-at most once per target; late guard loss can extend stop to roughly fourteen seconds. Unobserved descendants
-still report unknown. A failed or expired target does not abandon other admitted
+at most once per target; late guard loss can extend stop to roughly fourteen seconds. A waiting
+stop or launch waits up to sixteen seconds for `.stop.lock`, longer than that worst case, so it
+gets its own receipt instead of a lock-wait failure. Descendants forked after the capture are never
+signalled: once no captured member is left, stop waits until the target's own deadline for them to
+exit and reports unknown only if they are still alive. A failed or expired target does not abandon other admitted
targets: stop drives them to their own outcomes before returning unknown.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
@@ -126,7 +129,7 @@ leader identity and each remaining member's UID, birth and session. Recovery
delivers TERM individually, grants a fresh five-second grace, then KILL to the captured incarnations and
confirms absence. The capture remains authoritative if TERM ends the witness;
every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux).
-Unobserved descendants never inherit this authority and retain an unknown outcome. Unverified identity never authorizes signals or a successful
+Unobserved descendants never inherit this authority; stop waits for them until its deadline and reports unknown if they remain. Unverified identity never authorizes signals or a successful
stop. Launch and stop share a bounded lock; a startup descriptor lease keeps
admission locked through guard publication even if the project owner dies.
The guard temporarily inherits this stop-lock lease on stderr (fd 2), then closes
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 138cbab5f..9fb0805bd 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -50,7 +50,9 @@ fallback session census detects surviving descendants. EOF descriptors leave the
finds descendants whose exit does not generate SIGCHLD for this guard. Stop uses
20 ms confirmation intervals and keeps its five-second TERM grace. Immediate
restart retries the instance lock for at most one second. Command-lock admission
-waits at most ten seconds for a previous owner to finish releasing its lock.
+waits at most ten seconds for a previous owner to finish releasing its lock. Stop
+and launch wait up to sixteen seconds for `.stop.lock`, which outlasts a stop that
+recovers after late guard loss (about fourteen seconds).
The project supervisor passes its launch/stop flock lease to each guard on startup
fd 2. The guard retains it through durable publication and receipt delivery, then
From c2c48699c805c6d5aa537052b8786c39ff8e8007 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 02:48:39 +0000
Subject: [PATCH 28/45] fix(supervisor): answer a stop whose lock wait expires
with an unknown receipt
The sixteen-second .stop.lock wait outlasts a stop's timers but not the
admission and census work outside them, so it is not a hard bound. A stop
whose wait still expires now writes an unknown receipt with the current
records and acts on nothing, instead of exiting without a frame.
The guard-loss helper now waits for the guard's record with a
zombie-aware birth lookup that matches production inspection on Darwin,
and resumes the frozen stop request if an assertion fails first.
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 6 +-
packages/lifecycle-supervisor/README.md | 5 +-
.../lifecycle-supervisor/src/supervisor.rs | 30 +++++-
.../lifecycle-supervisor/tests/fake_caller.rs | 100 ++++++++++++++++--
4 files changed, 125 insertions(+), 16 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 465e06287..ca0d289f5 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -52,8 +52,10 @@ is lost during stop, recovery prefers a fresh capture and falls back to those
admitted identities, rechecking each before TERM/KILL. Each recovered target gets
its own five-second TERM grace and two-second confirmation bound. Recovery begins
at most once per target; late guard loss can extend stop to roughly fourteen seconds. A waiting
-stop or launch waits up to sixteen seconds for `.stop.lock`, longer than that worst case, so it
-gets its own receipt instead of a lock-wait failure. Descendants forked after the capture are never
+stop or launch waits up to sixteen seconds for `.stop.lock`, longer than those timers, so it
+normally gets its own outcome. Admission and census work run outside the timers, so this is not
+a hard bound: if a stop's wait still expires, it returns an `unknown` receipt with the current
+records and acts on nothing. Descendants forked after the capture are never
signalled: once no captured member is left, stop waits until the target's own deadline for them to
exit and reports unknown only if they are still alive. A failed or expired target does not abandon other admitted
targets: stop drives them to their own outcomes before returning unknown.
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 9fb0805bd..29963cb78 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -51,8 +51,9 @@ finds descendants whose exit does not generate SIGCHLD for this guard. Stop uses
20 ms confirmation intervals and keeps its five-second TERM grace. Immediate
restart retries the instance lock for at most one second. Command-lock admission
waits at most ten seconds for a previous owner to finish releasing its lock. Stop
-and launch wait up to sixteen seconds for `.stop.lock`, which outlasts a stop that
-recovers after late guard loss (about fourteen seconds).
+and launch wait up to sixteen seconds for `.stop.lock`, longer than the timers of a
+stop that recovers after late guard loss (about fourteen seconds). A stop whose wait
+still expires returns an `unknown` receipt without acting.
The project supervisor passes its launch/stop flock lease to each guard on startup
fd 2. The guard retains it through durable publication and receipt delivery, then
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index ab09de1db..efb1bf471 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -21,9 +21,11 @@ use std::{
const GRACE: Duration = Duration::from_secs(5);
const IDLE_TICK: Duration = Duration::from_millis(500);
const TICK: Duration = Duration::from_millis(20);
-/// One stop holds `.stop.lock` for at most two TERM-to-KILL bounds: the guard
-/// path, then orphan recovery after a late guard loss. A waiter must outlast
-/// that holder instead of failing without a receipt.
+/// A stop's timers allow two TERM-to-KILL bounds: the guard path, then orphan
+/// recovery after a late guard loss. The wait outlasts those timers so a
+/// waiter normally gets its own outcome. It is not a hard bound: admission,
+/// census and signalling run outside the timers. If the wait still expires,
+/// stop returns an unknown receipt without acting.
const STOP_LOCK_WAIT: Duration =
Duration::from_secs(2 * (GRACE.as_secs() + CONFIRM_BOUND.as_secs()) + 2);
/// Bounded wait for a previous command owner to release its lock.
@@ -79,7 +81,27 @@ pub fn main() -> Result<()> {
protocol::validate_control(request.version, &request.root, &request.owner_token)?;
let root = Root::open(&request.root)?;
let _stop_lock = if args[0] == "stop" {
- Some(root.lock_wait(".stop.lock", STOP_LOCK_WAIT)?)
+ match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) {
+ Ok(lock) => Some(lock),
+ Err(error) if error.kind() == std::io::ErrorKind::TimedOut => {
+ // Another stop or launch still holds the lock. Report the
+ // current receipts as unknown without acting, so a waiter
+ // always gets a frame instead of a bare failure.
+ protocol::write(
+ &mut std::io::stdout(),
+ &Receipt::new(
+ 0,
+ "unknown",
+ root.states()?
+ .into_iter()
+ .filter(|state| state.owner_token == request.owner_token)
+ .collect(),
+ ),
+ )?;
+ return Err(error);
+ }
+ Err(error) => return Err(error),
+ }
} else {
None
};
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index 656716da7..df610f5ad 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1121,14 +1121,11 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
assert!(wait(&mut caller, TIMEOUT).success());
}
// Resume stop only after the guard incarnation has no process record left,
- // so neither a live nor a zombie guard can witness a fresh capture.
- eventually(|| {
- identity(guard)
- .is_none_or(|current| current.birth != guard_birth)
- .then_some(())
- });
- // SAFETY: resumes the stop request stopped above.
- assert_eq!(unsafe { libc::kill(stop_pid, libc::SIGCONT) }, 0);
+ // zombie included, so no guard can witness a fresh capture.
+ // Resume the stop request even if an assertion below fails first.
+ let resume = ResumeOnDrop(stop_pid);
+ eventually(|| (process_birth(guard) != Some(guard_birth)).then_some(()));
+ drop(resume);
let receipt = receive(stop.stdout.as_mut().unwrap());
let stopped = wait(&mut stop, TIMEOUT).success();
assert_eq!(stopped, !blocked, "{receipt}");
@@ -1260,6 +1257,93 @@ fn orphan_recovery_never_signals_unobserved_descendant() {
late_descendant_case(true);
}
+/// Sends SIGCONT to a stopped child of this test when dropped, so a failed
+/// assertion cannot leave the stop request suspended.
+struct ResumeOnDrop(i32);
+
+impl Drop for ResumeOnDrop {
+ fn drop(&mut self) {
+ // SAFETY: the PID is this test's own unreaped child.
+ unsafe {
+ libc::kill(self.0, libc::SIGCONT);
+ }
+ }
+}
+
+/// Birth of any process record at `pid`, zombies included, matching the
+/// production inspection (Darwin `proc_pidinfo` argument 1, no `getsid`).
+fn process_birth(pid: i32) -> Option {
+ #[cfg(target_os = "linux")]
+ {
+ let text = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
+ text.rsplit_once(") ")?
+ .1
+ .split_whitespace()
+ .nth(19)?
+ .parse()
+ .ok()
+ }
+ #[cfg(target_os = "macos")]
+ {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: native output buffer is writable and exactly sized.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDTBSDINFO,
+ 1,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ if size as usize != std::mem::size_of::() {
+ return None;
+ }
+ let info = unsafe { info.assume_init() };
+ Some(info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec)
+ }
+}
+
+#[test]
+fn stop_waiting_past_lock_bound_reports_unknown_without_acting() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ let worker = identity(pid(&state, "commandPid")).expect("command identity");
+ // Hold the fixture's stop lock as a stuck stop would.
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(f.root.join(".stop.lock"))
+ .unwrap();
+ // SAFETY: flock only borrows this live descriptor.
+ assert_eq!(
+ unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
+ 0
+ );
+ let start = Instant::now();
+ let mut stop = command(&f.root, "stop").spawn().unwrap();
+ send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ // The frame arrives only after the sixteen-second wait; the reader's own
+ // timeout is shorter, so start reading part of the way through.
+ thread::sleep(Duration::from_secs(10));
+ let receipt = receive(stop.stdout.as_mut().unwrap());
+ assert!(!wait(&mut stop, TIMEOUT).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "unknown");
+ assert!(start.elapsed() >= Duration::from_secs(16));
+ assert!(
+ identity(worker.pid).as_ref() == Some(&worker),
+ "stop acted while waiting"
+ );
+ assert_eq!(f.state("worker")["phase"], "running");
+ drop(lock);
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
fn process_is_zombie(pid: i32) -> bool {
process_state(pid) == Some(b'Z')
}
From a3905028a2bbbd27f629d68bff0ccb58c96c4b45 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 02:52:42 +0000
Subject: [PATCH 29/45] test(supervisor): resume the frozen stop request from
the moment it is stopped
Co-Authored-By: Claude Opus 5.5
---
packages/lifecycle-supervisor/tests/fake_caller.rs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index df610f5ad..cae7a11dc 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1114,6 +1114,8 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
let stop_pid = stop.id() as i32;
// SAFETY: the stop request is this test's own child.
assert_eq!(unsafe { libc::kill(stop_pid, libc::SIGSTOP) }, 0);
+ // Resume the stop request even if any assertion below fails first.
+ let resume = ResumeOnDrop(stop_pid);
eventually(|| process_is_stopped(stop_pid).then_some(()));
// SAFETY: the frozen guard belongs to this fixture.
assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
@@ -1122,8 +1124,6 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
}
// Resume stop only after the guard incarnation has no process record left,
// zombie included, so no guard can witness a fresh capture.
- // Resume the stop request even if an assertion below fails first.
- let resume = ResumeOnDrop(stop_pid);
eventually(|| (process_birth(guard) != Some(guard_birth)).then_some(()));
drop(resume);
let receipt = receive(stop.stdout.as_mut().unwrap());
From 3aa00588d6a65d1ccdc6ce9fa14cc011d8eacd55 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 03:00:06 +0000
Subject: [PATCH 30/45] test(supervisor): freeze or lock only after startup
releases the stop lock
A launch writes its ready receipt just before it drops .stop.lock, and a
guard keeps its startup lease on that lock until publication. Tests that
froze the project supervisor or a guard right after start could freeze a
lock holder, and the new lock test could find the lock still held. With
the sixteen-second stop lock wait, a stop behind a frozen holder outlived
the fifteen-second test deadline (hosted Linux:
unreaped_matching_guard_proves_legacy_session_ownership). Wait until the
fixture's stop lock is free before freezing a process or holding the lock.
Co-Authored-By: Claude Opus 5.5
---
.../lifecycle-supervisor/tests/fake_caller.rs | 42 +++++++++++++------
1 file changed, 30 insertions(+), 12 deletions(-)
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index cae7a11dc..a5d13bd64 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -103,6 +103,26 @@ impl Fixture {
}
}
+ /// Take the fixture's stop lock once its current holder releases it. A
+ /// launch writes its ready receipt just before it drops this lock, and a
+ /// guard keeps its startup lease until publication, so the lock can still
+ /// be held briefly after `start` returns.
+ fn take_stop_lock(&self) -> fs::File {
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(self.root.join(".stop.lock"))
+ .unwrap();
+ eventually(|| {
+ // SAFETY: flock only borrows this live descriptor.
+ (unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0)
+ .then_some(())
+ });
+ lock
+ }
+
fn state(&self, name: &str) -> Value {
eventually(|| {
fs::read(self.root.join(format!("{name}.json")))
@@ -1060,6 +1080,8 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
}
let (mut caller, _) = f.start(processes);
let blocked_guard = blocked.then(|| pid(&f.state("blocked"), "guardPid"));
+ // Freeze only after launch and guard startup leases released the stop lock.
+ drop(f.take_stop_lock());
if let Some(guard) = blocked_guard {
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
}
@@ -1312,18 +1334,7 @@ fn stop_waiting_past_lock_bound_reports_unknown_without_acting() {
let state = f.state("worker");
let worker = identity(pid(&state, "commandPid")).expect("command identity");
// Hold the fixture's stop lock as a stuck stop would.
- let lock = fs::OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .truncate(false)
- .open(f.root.join(".stop.lock"))
- .unwrap();
- // SAFETY: flock only borrows this live descriptor.
- assert_eq!(
- unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
- 0
- );
+ let lock = f.take_stop_lock();
let start = Instant::now();
let mut stop = command(&f.root, "stop").spawn().unwrap();
send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
@@ -1395,6 +1406,9 @@ fn unreaped_matching_guard_proves_legacy_session_ownership() {
let mut caller = ForeignChild(caller);
f.await_log("fixture-ready");
let mut state = f.state("worker");
+ // Freeze the parent only after launch has released the stop lock;
+ // otherwise the frozen parent would hold it and stop could never run.
+ drop(f.take_stop_lock());
// Keep the parent from reaping its guard, then exercise legacy birth absence.
assert_eq!(
unsafe { libc::kill(caller.0.id() as i32, libc::SIGSTOP) },
@@ -1440,6 +1454,8 @@ fn guard_dying_during_stop_gets_a_fresh_authored_term_grace() {
f.await_log("fixture-ready");
let guard = pid(&f.state("worker"), "guardPid");
let birth = identity(guard).unwrap();
+ // Freeze only after launch and guard startup leases released the stop lock.
+ drop(f.take_stop_lock());
// Suspend this owned guard so its pending TERM cannot reach the command.
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
let mut stop = command(&f.root, "stop").spawn().unwrap();
@@ -1687,6 +1703,8 @@ fn unresponsive_guard_stop_remains_unknown() {
f.await_log("fixture-ready");
let state = f.state("worker");
let guard = pid(&state, "guardPid");
+ // Freeze only after launch and guard startup leases released the stop lock.
+ drop(f.take_stop_lock());
// SAFETY: the guard belongs to this fixture and its receipt is immutable.
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
From 2d9c4cef9f5cd4e9b8a248325fe3e90b74305278 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 02:29:07 +0000
Subject: [PATCH 31/45] fix(supervisor): treat pidfd_open EINVAL for an exited
group leader as absent
On Linux 6.8, pidfd_open returns EINVAL rather than ESRCH for a PID that
survives only as a process-group or session ID after its process exited.
A recovery stop whose wrapper died on TERM then failed before sending KILL
to the wrapper's TERM-ignoring child and reported unknown. Flags are zero,
so EINVAL means no thread-group leader exists; skip that member and let
the identity-checked liveness pass decide the outcome.
Four recovery fake-caller tests failed on a 6.8 host without this change
and pass with it.
Co-Authored-By: Claude Opus 5.5
(cherry picked from commit 416225380ef509b894f688894052f4523890fb35)
---
packages/lifecycle-supervisor/src/os.rs | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index d9c708cc1..d739d344b 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -467,7 +467,11 @@ fn signal_member(expected: &Identity, signal: i32) -> Result<()> {
let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, expected.pid, 0u32) };
if raw < 0 {
let error = io::Error::last_os_error();
- if absent_error(&error) {
+ // Some kernels (observed on 6.8) return EINVAL, not ESRCH, for a PID
+ // kept alive only as a process-group or session ID after its process
+ // exited. Flags are zero, so EINVAL means no thread-group leader
+ // exists for that PID; liveness is still decided by `inspect` later.
+ if absent_error(&error) || error.raw_os_error() == Some(libc::EINVAL) {
return Ok(());
}
return Err(error);
From bf664de47f9e88489151b10ce01204aab953a417 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 03:02:29 +0000
Subject: [PATCH 32/45] test(supervisor): pin pidfd_open EINVAL handling behind
a seam
Hosted runners use kernels newer than 6.8 and never return EINVAL here,
so the four fake-caller recovery tests only exercise it on a 6.8 host.
Move the open into pidfd_open() with a test-only errno injection and add
unit tests with two real members: EINVAL or ESRCH from the first open
skips it without a signal and recovery still delivers KILL to the
survivor; any other errno refuses without signalling either. The comment
now states when 6.8 returns EINVAL and that the absent result can only
withhold a signal. Linux-only; Darwin code is unchanged.
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 4 +-
packages/lifecycle-supervisor/src/os.rs | 142 +++++++++++++++++++++---
2 files changed, 130 insertions(+), 16 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 3221004d8..0e0f3cff3 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -125,7 +125,9 @@ refused. Stop can recover the recorded session after verifying the boot,
leader identity and each remaining member's UID, birth and session. Recovery
delivers TERM individually, grants a fresh five-second grace, then KILL to the captured incarnations and
confirms absence. The capture remains authoritative if TERM ends the witness;
-every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux).
+every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux). A pidfd open that
+reports the process missing (`ESRCH`, or `EINVAL` on Linux 6.8 when the PID survives only as a group or
+session ID) skips that member without a signal; liveness is still decided by identity afterwards.
Unobserved descendants never inherit this authority and retain an unknown outcome. Unverified identity never authorizes signals or a successful
stop. Launch and stop share a bounded lock; a startup descriptor lease keeps
admission locked through guard publication even if the project owner dies.
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index d739d344b..5ae6c5f3a 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -459,24 +459,45 @@ pub fn session_signal(session: i32, signal: i32) -> Result<()> {
Ok(())
}
+/// Open a pidfd for `pid`, or `None` when no process with that thread-group
+/// leader exists any more.
+///
+/// With zero flags, `pidfd_open` reports a missing process as `ESRCH` on newer
+/// kernels, but as `EINVAL` on Linux 6.8 when the PID survives only as a
+/// process-group or session ID (its process exited while members of its group
+/// or session still run), because no thread-group leader exists for it. Both
+/// mean "no process to signal". Treating them as absent can only withhold a
+/// signal, never send one: identity-checked liveness decides the outcome later.
+#[cfg(target_os = "linux")]
+fn pidfd_open(pid: i32) -> Result> {
+ use std::os::fd::{FromRawFd, OwnedFd};
+ #[cfg(test)]
+ if let Some(errno) = tests::injected_pidfd_open_error(pid) {
+ return pidfd_open_failure(io::Error::from_raw_os_error(errno));
+ }
+ // SAFETY: pidfd_open returns a newly owned CLOEXEC descriptor.
+ let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0u32) };
+ if raw < 0 {
+ return pidfd_open_failure(io::Error::last_os_error());
+ }
+ // SAFETY: a non-negative result is a descriptor this call now owns.
+ Ok(Some(unsafe { OwnedFd::from_raw_fd(raw as i32) }))
+}
+
+#[cfg(target_os = "linux")]
+fn pidfd_open_failure(error: io::Error) -> Result> {
+ if absent_error(&error) || error.raw_os_error() == Some(libc::EINVAL) {
+ return Ok(None);
+ }
+ Err(error)
+}
+
fn signal_member(expected: &Identity, signal: i32) -> Result<()> {
#[cfg(target_os = "linux")]
{
- use std::os::fd::{FromRawFd, OwnedFd};
- // SAFETY: pidfd_open returns a newly owned CLOEXEC descriptor.
- let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, expected.pid, 0u32) };
- if raw < 0 {
- let error = io::Error::last_os_error();
- // Some kernels (observed on 6.8) return EINVAL, not ESRCH, for a PID
- // kept alive only as a process-group or session ID after its process
- // exited. Flags are zero, so EINVAL means no thread-group leader
- // exists for that PID; liveness is still decided by `inspect` later.
- if absent_error(&error) || error.raw_os_error() == Some(libc::EINVAL) {
- return Ok(());
- }
- return Err(error);
- }
- let fd = unsafe { OwnedFd::from_raw_fd(raw as i32) };
+ let Some(fd) = pidfd_open(expected.pid)? else {
+ return Ok(());
+ };
let Some(current) = inspect(expected.pid)? else {
return Ok(());
};
@@ -866,6 +887,97 @@ pub fn timestamp() -> Result {
#[cfg(test)]
mod tests {
+ #[cfg(target_os = "linux")]
+ thread_local! {
+ static PIDFD_OPEN_ERROR: std::cell::Cell> =
+ const { std::cell::Cell::new(None) };
+ }
+
+ /// Test seam: the errno `pidfd_open` reports for one PID on this thread.
+ #[cfg(target_os = "linux")]
+ pub(super) fn injected_pidfd_open_error(pid: i32) -> Option {
+ PIDFD_OPEN_ERROR
+ .with(std::cell::Cell::get)
+ .and_then(|(target, errno)| (target == pid).then_some(errno))
+ }
+
+ /// Two real members: the first reports `errno` from its pidfd open, the
+ /// second is an ordinary survivor. Returns the recovery KILL result and
+ /// whether each member was killed.
+ #[cfg(target_os = "linux")]
+ fn kill_with_failed_first_open(errno: i32) -> (std::io::Result<()>, bool, bool) {
+ use std::os::unix::process::ExitStatusExt;
+ let spawn = || {
+ std::process::Command::new("/bin/sleep")
+ .arg("60")
+ .spawn()
+ .expect("spawn member")
+ };
+ let (mut first, mut second) = (spawn(), spawn());
+ let identity = |child: &std::process::Child| {
+ super::inspect(child.id() as i32)
+ .unwrap()
+ .expect("member identity")
+ };
+ let recovery = super::Recovery {
+ session: 0,
+ members: vec![identity(&first), identity(&second)],
+ };
+ PIDFD_OPEN_ERROR.with(|cell| cell.set(Some((first.id() as i32, errno))));
+ let result = recovery.signal(libc::SIGKILL);
+ PIDFD_OPEN_ERROR.with(|cell| cell.set(None));
+ let killed = |child: &mut std::process::Child| {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
+ loop {
+ if let Some(status) = child.try_wait().unwrap() {
+ return status.signal() == Some(libc::SIGKILL);
+ }
+ if std::time::Instant::now() >= deadline {
+ return false;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ }
+ };
+ let second_killed = killed(&mut second);
+ let first_killed = first.try_wait().unwrap().is_some();
+ let _ = first.kill();
+ let _ = first.wait();
+ let _ = second.kill();
+ let _ = second.wait();
+ (result, first_killed, second_killed)
+ }
+
+ /// Linux 6.8 reports EINVAL for a PID that survives only as a group or
+ /// session ID. That member is absent: it gets no signal, and recovery still
+ /// delivers KILL to the remaining captured member. Pinned on every kernel.
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn pidfd_open_einval_is_absent_and_recovery_still_kills_survivor() {
+ let (result, first_killed, second_killed) = kill_with_failed_first_open(libc::EINVAL);
+ assert!(result.is_ok(), "{result:?}");
+ assert!(!first_killed, "absent member was signalled");
+ assert!(second_killed, "surviving member did not get KILL");
+ }
+
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn pidfd_open_esrch_is_absent_and_recovery_still_kills_survivor() {
+ let (result, first_killed, second_killed) = kill_with_failed_first_open(libc::ESRCH);
+ assert!(result.is_ok(), "{result:?}");
+ assert!(!first_killed);
+ assert!(second_killed);
+ }
+
+ /// Any other open failure stays an error and authorizes no signal.
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn pidfd_open_other_errors_refuse_without_signalling() {
+ let (result, first_killed, second_killed) = kill_with_failed_first_open(libc::EMFILE);
+ assert_eq!(result.unwrap_err().raw_os_error(), Some(libc::EMFILE));
+ assert!(!first_killed);
+ assert!(!second_killed);
+ }
+
#[test]
fn zombie_guard_birth_survives_discarded_exit_metadata() {
let state = crate::storage::State {
From 1fbf333371d68f85085cb4b70016feaf1c02c19d Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 03:09:21 +0000
Subject: [PATCH 33/45] test(supervisor): kill and reap pidfd seam members on
any failure
Co-Authored-By: Claude Opus 5.5
---
packages/lifecycle-supervisor/src/os.rs | 42 ++++++++++++++++---------
1 file changed, 27 insertions(+), 15 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 5ae6c5f3a..41fa44c69 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -901,6 +901,19 @@ mod tests {
.and_then(|(target, errno)| (target == pid).then_some(errno))
}
+ /// A member process that is killed and reaped when dropped, so a failed
+ /// assertion or spawn cannot leak it.
+ #[cfg(target_os = "linux")]
+ struct Member(std::process::Child);
+
+ #[cfg(target_os = "linux")]
+ impl Drop for Member {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+ }
+
/// Two real members: the first reports `errno` from its pidfd open, the
/// second is an ordinary survivor. Returns the recovery KILL result and
/// whether each member was killed.
@@ -908,14 +921,17 @@ mod tests {
fn kill_with_failed_first_open(errno: i32) -> (std::io::Result<()>, bool, bool) {
use std::os::unix::process::ExitStatusExt;
let spawn = || {
- std::process::Command::new("/bin/sleep")
- .arg("60")
- .spawn()
- .expect("spawn member")
+ Member(
+ std::process::Command::new("/bin/sleep")
+ .arg("60")
+ .spawn()
+ .expect("spawn member"),
+ )
};
- let (mut first, mut second) = (spawn(), spawn());
- let identity = |child: &std::process::Child| {
- super::inspect(child.id() as i32)
+ let mut first = spawn();
+ let mut second = spawn();
+ let identity = |member: &Member| {
+ super::inspect(member.0.id() as i32)
.unwrap()
.expect("member identity")
};
@@ -923,13 +939,13 @@ mod tests {
session: 0,
members: vec![identity(&first), identity(&second)],
};
- PIDFD_OPEN_ERROR.with(|cell| cell.set(Some((first.id() as i32, errno))));
+ PIDFD_OPEN_ERROR.with(|cell| cell.set(Some((first.0.id() as i32, errno))));
let result = recovery.signal(libc::SIGKILL);
PIDFD_OPEN_ERROR.with(|cell| cell.set(None));
- let killed = |child: &mut std::process::Child| {
+ let killed = |member: &mut Member| {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
- if let Some(status) = child.try_wait().unwrap() {
+ if let Some(status) = member.0.try_wait().unwrap() {
return status.signal() == Some(libc::SIGKILL);
}
if std::time::Instant::now() >= deadline {
@@ -939,11 +955,7 @@ mod tests {
}
};
let second_killed = killed(&mut second);
- let first_killed = first.try_wait().unwrap().is_some();
- let _ = first.kill();
- let _ = first.wait();
- let _ = second.kill();
- let _ = second.wait();
+ let first_killed = first.0.try_wait().unwrap().is_some();
(result, first_killed, second_killed)
}
From 033358ac72c29ede3e6597b479ce8125bfce222e Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 03:17:45 +0000
Subject: [PATCH 34/45] fix(supervisor): frame every stop and status reply and
count threaded leaders as live
Every stop and status request now ends with exactly one framed receipt,
also when it fails. A failure keeps the nonzero exit and reports outcome
refused (invalid-request) or unknown with a reason: stop-lock-wait-expired,
foreign-live-process, stop-incomplete, retired-owner-live or io-error.
Before, a lock wait that expired, a refusal or an I/O error exited with a
closed pipe, which a caller cannot tell from a crash.
The .stop.lock wait for stop and launch is now 16 s, derived from the two
TERM-to-KILL bounds a recovering stop can spend (about 14 s); the command
lock keeps 10 s.
On Linux, a thread-group leader whose main thread exited while worker
threads run shows Z in /proc//stat. It was treated as a zombie, so
recovery after guard loss neither captured nor signalled it and could
report stopped (and admit a duplicate) while the workers survived. A
leader now counts as a zombie only when no thread in /proc//task is
live. Regression: main_thread_exit_with_live_worker_is_still_stopped
(fails without the fix).
Tests: tests that freeze the project supervisor or a guard right after
start, or hold the stop lock, first wait until the fixture's .stop.lock is
free. A launch writes its ready receipt just before it drops that lock and
a guard keeps its startup lease until publication, so a frozen holder
could make stop wait out its bound (the Mac failures of
expired_target_does_not_abandon_other_recovery). eventually, wait, receive
and exact are #[track_caller] so a timeout names its call site.
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 23 ++-
packages/lifecycle-supervisor/README.md | 4 +-
packages/lifecycle-supervisor/src/os.rs | 30 ++-
packages/lifecycle-supervisor/src/storage.rs | 4 +-
.../lifecycle-supervisor/src/supervisor.rs | 175 ++++++++++++------
.../lifecycle-supervisor/tests/fake_caller.rs | 174 +++++++++++++++++
6 files changed, 347 insertions(+), 63 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 0e0f3cff3..95e0c8766 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -56,8 +56,25 @@ still report unknown. A failed or expired target does not abandon other admitted
targets: stop drives them to their own outcomes before returning unknown.
Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`,
-`observed`, `stopped`, or `unknown`). A stop whose confirmation times out emits
+`observed`, `stopped`, `unknown` or `refused`). A stop whose confirmation times out emits
`unknown` and exits nonzero. Callers must inspect both the receipt and exit status.
+
+Every `stop` and `status` request ends with exactly one framed receipt, also when it
+fails. A failed request exits nonzero and carries a `reason`:
+
+| `reason` | `outcome` | Meaning |
+|---|---|---|
+| `invalid-request` | `refused` | The control frame or root was rejected; nothing was read or changed. |
+| `stop-lock-wait-expired` | `unknown` | Another stop or a launch held `.stop.lock` for the whole wait (sixteen seconds); this request acted on nothing. |
+| `foreign-live-process` | `unknown` | A receipt from another owner token still has live members. |
+| `stop-incomplete` | `unknown` | Stop could not confirm that every member is gone. |
+| `retired-owner-live` | `unknown` | Status for a retired owner whose members are not proven absent. |
+| `io-error` | `unknown` | Any other local failure. |
+
+A successful reply has no `reason`. The sixteen-second stop-lock wait exceeds the
+timers of a stop that recovers after late guard loss (about fourteen seconds), so a
+waiter normally gets its own outcome; admission and census work run outside those
+timers, which is why an expired wait still returns its own receipt.
`run` must be spawned as a fresh child without making it a process-group leader:
the supervisor calls `setsid` itself. A caller using `detached: true` or
`setpgid(0, 0)` is refused before authored code starts.
@@ -125,7 +142,9 @@ refused. Stop can recover the recorded session after verifying the boot,
leader identity and each remaining member's UID, birth and session. Recovery
delivers TERM individually, grants a fresh five-second grace, then KILL to the captured incarnations and
confirms absence. The capture remains authoritative if TERM ends the witness;
-every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux). A pidfd open that
+every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux).
+On Linux a thread-group leader whose main thread exited while other threads run shows
+`Z` in `/proc//stat`; it counts as live until no thread in `/proc//task` is. A pidfd open that
reports the process missing (`ESRCH`, or `EINVAL` on Linux 6.8 when the PID survives only as a group or
session ID) skips that member without a signal; liveness is still decided by identity afterwards.
Unobserved descendants never inherit this authority and retain an unknown outcome. Unverified identity never authorizes signals or a successful
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 138cbab5f..7bfa522a4 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -50,7 +50,9 @@ fallback session census detects surviving descendants. EOF descriptors leave the
finds descendants whose exit does not generate SIGCHLD for this guard. Stop uses
20 ms confirmation intervals and keeps its five-second TERM grace. Immediate
restart retries the instance lock for at most one second. Command-lock admission
-waits at most ten seconds for a previous owner to finish releasing its lock.
+waits at most ten seconds for a previous owner to finish releasing its lock. Stop
+and launch wait up to sixteen seconds for `.stop.lock`; a stop whose wait expires
+still answers with an `unknown` receipt (reason `stop-lock-wait-expired`).
The project supervisor passes its launch/stop flock lease to each guard on startup
fd 2. The guard retains it through durable publication and receipt delivery, then
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 41fa44c69..e61767cc6 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -95,10 +95,38 @@ pub fn inspect(pid: i32) -> Result> {
session,
start,
uid,
- zombie: fields.first() == Some(&"Z") || fields.first() == Some(&"X"),
+ zombie: matches!(fields.first(), Some(&"Z" | &"X")) && !linux_live_thread(pid)?,
}))
}
+/// A thread-group leader that called `pthread_exit` shows `Z` in
+/// `/proc//stat` while its other threads keep running. Such a process is
+/// alive: it still owns its PID and receives process-directed signals. Only a
+/// group with no live thread left is a zombie.
+#[cfg(target_os = "linux")]
+fn linux_live_thread(pid: i32) -> Result {
+ let tasks = match fs::read_dir(format!("/proc/{pid}/task")) {
+ Ok(tasks) => tasks,
+ Err(error) if absent_error(&error) => return Ok(false),
+ Err(error) => return Err(error),
+ };
+ for task in tasks {
+ let task = task?;
+ let stat = match fs::read(task.path().join("stat")) {
+ Ok(stat) => stat,
+ Err(error) if absent_error(&error) => continue,
+ Err(error) => return Err(error),
+ };
+ if !matches!(
+ proc_stat_tail(&stat)?.split_whitespace().next(),
+ Some("Z" | "X")
+ ) {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+}
+
#[cfg(target_os = "linux")]
fn linux_uid(pid: i32) -> Result> {
let status = match fs::read(format!("/proc/{pid}/status")) {
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index 8a17d2514..e2d0ad6fa 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -117,9 +117,9 @@ impl Root {
Ok(file)
}
- pub fn lock_wait(&self, name: &str) -> Result {
+ pub fn lock_wait(&self, name: &str, wait: std::time::Duration) -> Result {
let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?;
- let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
+ let deadline = std::time::Instant::now() + wait;
loop {
// SAFETY: flock only borrows the live owned descriptor.
if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 39bad47f1..85ea68fff 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -21,6 +21,17 @@ use std::{
const GRACE: Duration = Duration::from_secs(5);
const IDLE_TICK: Duration = Duration::from_millis(500);
const TICK: Duration = Duration::from_millis(20);
+/// Time after KILL for a stop to confirm absence.
+const CONFIRM_BOUND: Duration = Duration::from_secs(2);
+/// A stop's timers allow two TERM-to-KILL bounds: the guard path, then orphan
+/// recovery after a late guard loss. The wait outlasts those timers, so a
+/// waiter normally gets its own outcome. Admission, census and signalling run
+/// outside the timers, so it is not a hard bound: an expired wait returns an
+/// `unknown` receipt with reason `stop-lock-wait-expired` and acts on nothing.
+const STOP_LOCK_WAIT: Duration =
+ Duration::from_secs(2 * (GRACE.as_secs() + CONFIRM_BOUND.as_secs()) + 2);
+/// Bounded wait for a previous command owner to release its lock.
+const COMMAND_LOCK_WAIT: Duration = Duration::from_secs(10);
const LINE_LIMIT: usize = 64 * 1024;
#[derive(Serialize)]
@@ -32,6 +43,9 @@ struct Receipt {
processes: Vec,
live_process_names: Vec,
unknown_process_names: Vec,
+ /// Why a stop or status request did not complete; absent on success.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reason: Option<&'static str>,
}
impl Receipt {
@@ -52,10 +66,106 @@ impl Receipt {
processes,
live_process_names,
unknown_process_names,
+ reason: None,
}
}
}
+/// Answer one `stop` or `status` request. Every request that reaches this
+/// point ends with exactly one framed receipt, including refusals and expired
+/// waits, so a caller can tell an incomplete request from a crash. A failure
+/// keeps the nonzero exit and reports outcome `refused` (no request was
+/// admitted) or `unknown`, with a `reason` code.
+fn control(action: &str) -> Result<()> {
+ let mut stdout = std::io::stdout();
+ let admitted = protocol::read::(&mut std::io::stdin()).and_then(|request| {
+ protocol::validate_control(request.version, &request.root, &request.owner_token)?;
+ let root = Root::open(&request.root)?;
+ Ok((request, root))
+ });
+ let (request, root) = match admitted {
+ Ok(admitted) => admitted,
+ Err(error) => {
+ let mut receipt = Receipt::new(0, "refused", Vec::new());
+ receipt.reason = Some("invalid-request");
+ protocol::write(&mut stdout, &receipt)?;
+ return Err(error);
+ }
+ };
+ let mut reason = "io-error";
+ let result = control_effects(action, &request, &root, &mut reason);
+ // Receipts are atomically replaced files, so this read is safe without the
+ // lock, also after a lock wait expired.
+ let processes = root
+ .states()
+ .map(|states| {
+ states
+ .into_iter()
+ .filter(|state| state.owner_token == request.owner_token)
+ .collect()
+ })
+ .unwrap_or_default();
+ let outcome = match (&result, action) {
+ (Err(_), _) => "unknown",
+ (Ok(()), "stop") => "stopped",
+ (Ok(()), _) => "observed",
+ };
+ let mut receipt = Receipt::new(0, outcome, processes);
+ if result.is_err() {
+ receipt.reason = Some(reason);
+ }
+ protocol::write(&mut stdout, &receipt)?;
+ result
+}
+
+fn control_effects(
+ action: &str,
+ request: &Control,
+ root: &Root,
+ reason: &mut &'static str,
+) -> Result<()> {
+ let _stop_lock = if action == "stop" {
+ match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) {
+ Ok(lock) => Some(lock),
+ Err(error) => {
+ if error.kind() == std::io::ErrorKind::TimedOut {
+ // Another stop or a launch still holds the lock; act on nothing.
+ *reason = "stop-lock-wait-expired";
+ }
+ return Err(error);
+ }
+ }
+ } else {
+ None
+ };
+ let mut states = Vec::new();
+ for state in root.states()? {
+ if state.owner_token == request.owner_token {
+ states.push(state);
+ } else if os::recorded_live(&state, false)? {
+ // Old terminal receipts survive renames and new generations.
+ // Guard-written terminal phases prove completed retirement;
+ // non-terminal receipts require native identity evidence.
+ *reason = "foreign-live-process";
+ return Err(refused());
+ }
+ }
+ if action == "stop" {
+ root.retire(&request.owner_token)?;
+ *reason = "stop-incomplete";
+ return stop(&states);
+ }
+ if root.assert_not_retired(&request.owner_token).is_err()
+ && states
+ .iter()
+ .any(|state| !matches!(os::observe_state(state), os::Observation::Absent))
+ {
+ *reason = "retired-owner-live";
+ return Err(refused());
+ }
+ Ok(())
+}
+
pub fn main() -> Result<()> {
os::close_inherited()?;
let args: Vec<_> = std::env::args_os().skip(1).collect();
@@ -65,57 +175,7 @@ pub fn main() -> Result<()> {
match args[0].to_str() {
Some("run") => run(protocol::read(&mut std::io::stdin())?),
Some("guard") => guard(protocol::read(&mut std::io::stdin())?),
- Some("stop") | Some("status") => {
- let request: Control = protocol::read(&mut std::io::stdin())?;
- protocol::validate_control(request.version, &request.root, &request.owner_token)?;
- let root = Root::open(&request.root)?;
- let _stop_lock = if args[0] == "stop" {
- Some(root.lock_wait(".stop.lock")?)
- } else {
- None
- };
- let mut states = Vec::new();
- for state in root.states()? {
- if state.owner_token == request.owner_token {
- states.push(state);
- } else if os::recorded_live(&state, false)? {
- // Old terminal receipts survive renames and new generations.
- // Guard-written terminal phases prove completed retirement;
- // non-terminal receipts require native identity evidence.
- return Err(refused());
- }
- }
- let result = if args[0] == "stop" {
- root.retire(&request.owner_token)?;
- stop(&states)
- } else if root.assert_not_retired(&request.owner_token).is_err()
- && states
- .iter()
- .any(|state| !matches!(os::observe_state(state), os::Observation::Absent))
- {
- Err(refused())
- } else {
- Ok(())
- };
- protocol::write(
- &mut std::io::stdout(),
- &Receipt::new(
- 0,
- if result.is_err() {
- "unknown"
- } else if args[0] == "stop" {
- "stopped"
- } else {
- "observed"
- },
- root.states()?
- .into_iter()
- .filter(|state| state.owner_token == request.owner_token)
- .collect(),
- ),
- )?;
- result
- }
+ Some(action @ ("stop" | "status")) => control(action),
_ => Err(refused()),
}
}
@@ -135,7 +195,7 @@ fn run(request: Launch) -> Result<()> {
let root = Root::open(&request.root)?;
let _instance_lock = instance_lock(&root)?;
os::census_preflight()?;
- let launch_lock = root.lock_wait(".stop.lock")?;
+ let launch_lock = root.lock_wait(".stop.lock", STOP_LOCK_WAIT)?;
root.assert_not_retired(&request.owner_token)?;
// Validate every old claim before starting any new command. A guard survives
// project-owner death; its command lock also closes concurrent restart races.
@@ -199,7 +259,7 @@ fn run(request: Launch) -> Result<()> {
detach_stdio()?;
loop {
if os::STOP.load(Ordering::Relaxed) {
- match root.lock_wait(".stop.lock") {
+ match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) {
Ok(_lock) => {
stop(&root.states()?)?;
}
@@ -264,7 +324,7 @@ fn rollback(children: &mut [Child]) {
libc::kill(pid, libc::SIGTERM);
}
}
- let deadline = Instant::now() + GRACE + Duration::from_secs(2);
+ let deadline = Instant::now() + GRACE + CONFIRM_BOUND;
while Instant::now() < deadline {
if children
.iter_mut()
@@ -282,7 +342,8 @@ fn guard(request: GuardLaunch) -> Result<()> {
protocol::validate_control(request.version, &request.root, &request.owner_token)?;
request.process.validate()?;
let root = Root::open(&request.root)?;
- let _command_lock = root.lock_wait(&format!("{}.lock", request.process.name))?;
+ let _command_lock =
+ root.lock_wait(&format!("{}.lock", request.process.name), COMMAND_LOCK_WAIT)?;
if let Some(previous) = root.load(&request.process.name)?
&& os::recorded_live(&previous, false)?
{
@@ -518,7 +579,7 @@ fn stop(states: &[State]) -> Result<()> {
None => self.recover()?,
Some(false) => return Ok(false),
Some(true) => {
- if start.elapsed() >= GRACE + Duration::from_secs(2) {
+ if start.elapsed() >= GRACE + CONFIRM_BOUND {
return Err(refused());
}
return Ok(true);
@@ -530,7 +591,7 @@ fn stop(states: &[State]) -> Result<()> {
return Ok(false);
}
let elapsed = self.orphan_term.ok_or_else(refused)?.elapsed();
- if elapsed >= GRACE + Duration::from_secs(2) {
+ if elapsed >= GRACE + CONFIRM_BOUND {
return Err(refused());
}
if elapsed >= GRACE {
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index ce7122580..d0d1f9f36 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -103,6 +103,27 @@ impl Fixture {
}
}
+ /// Take the fixture's stop lock once its current holder releases it. A
+ /// launch writes its ready receipt just before it drops this lock, and a
+ /// guard keeps its startup lease until publication, so the lock can still
+ /// be held briefly after `start` returns.
+ #[track_caller]
+ fn take_stop_lock(&self) -> fs::File {
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(self.root.join(".stop.lock"))
+ .unwrap();
+ eventually(|| {
+ // SAFETY: flock only borrows this live descriptor.
+ (unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0)
+ .then_some(())
+ });
+ lock
+ }
+
fn state(&self, name: &str) -> Value {
eventually(|| {
fs::read(self.root.join(format!("{name}.json")))
@@ -286,6 +307,7 @@ fn send(writer: &mut impl Write, value: &Value) {
writer.flush().unwrap();
}
+#[track_caller]
fn exact(reader: &mut (impl Read + AsRawFd), bytes: &mut [u8]) {
let end = Instant::now() + TIMEOUT;
let mut offset = 0;
@@ -308,6 +330,7 @@ fn exact(reader: &mut (impl Read + AsRawFd), bytes: &mut [u8]) {
}
}
+#[track_caller]
fn receive(reader: &mut (impl Read + AsRawFd)) -> Value {
let mut header = [0; 4];
exact(reader, &mut header);
@@ -318,6 +341,7 @@ fn receive(reader: &mut (impl Read + AsRawFd)) -> Value {
serde_json::from_slice(&bytes).unwrap()
}
+#[track_caller]
fn wait(child: &mut Child, timeout: Duration) -> std::process::ExitStatus {
let end = Instant::now() + timeout;
loop {
@@ -333,6 +357,7 @@ fn wait(child: &mut Child, timeout: Duration) -> std::process::ExitStatus {
}
}
+#[track_caller]
fn eventually(mut predicate: impl FnMut() -> Option) -> T {
let end = Instant::now() + TIMEOUT;
loop {
@@ -1060,6 +1085,8 @@ fn guard_lost_during_stop(delay: Duration, blocked: bool) {
}
let (mut caller, _) = f.start(processes);
let blocked_guard = blocked.then(|| pid(&f.state("blocked"), "guardPid"));
+ // Freeze only after launch and guard startup leases released the stop lock.
+ drop(f.take_stop_lock());
if let Some(guard) = blocked_guard {
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
}
@@ -1179,6 +1206,108 @@ fn orphan_recovery_keeps_observed_child_after_term_ends_witness() {
);
}
+#[test]
+fn stop_waiting_past_lock_bound_reports_unknown_without_acting() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ let worker = identity(pid(&state, "commandPid")).expect("command identity");
+ // Hold the fixture's stop lock as a stuck stop would.
+ let lock = f.take_stop_lock();
+ let start = Instant::now();
+ let mut stop = command(&f.root, "stop").spawn().unwrap();
+ send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ // The frame arrives only after the sixteen-second wait; the reader's own
+ // timeout is shorter, so start reading part of the way through.
+ thread::sleep(Duration::from_secs(10));
+ let receipt = receive(stop.stdout.as_mut().unwrap());
+ assert!(!wait(&mut stop, TIMEOUT).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "unknown");
+ assert_eq!(receipt["reason"], "stop-lock-wait-expired");
+ assert!(start.elapsed() >= Duration::from_secs(16));
+ assert!(
+ identity(worker.pid).as_ref() == Some(&worker),
+ "stop acted while waiting"
+ );
+ assert_eq!(f.state("worker")["phase"], "running");
+ drop(lock);
+ f.stop();
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
+#[test]
+fn invalid_control_requests_still_get_a_framed_refusal() {
+ let f = Fixture::new();
+ for action in ["stop", "status"] {
+ let (mut child, receipt) = f.request(
+ action,
+ &json!({"version": 99, "root": f.root, "ownerToken": TOKEN}),
+ );
+ assert!(!wait(&mut child, TIMEOUT).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "refused");
+ assert_eq!(receipt["reason"], "invalid-request");
+ assert_eq!(receipt["processes"], json!([]));
+ }
+}
+
+/// Linux shows a leader whose main thread called `pthread_exit` as `Z` while
+/// its worker threads run. It must count as live. After guard loss, recovery
+/// signals captured members one by one, so a member misread as a zombie is
+/// never captured: stop then reported absence while its worker survived.
+#[cfg(target_os = "linux")]
+#[test]
+fn main_thread_exit_with_live_worker_is_still_stopped() {
+ for lose_guard in [false, true] {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "pthread-main-exit")]);
+ f.await_log("fixture-ready");
+ let state = f.state("worker");
+ let child = eventually(|| {
+ f.log().lines().find_map(|line| {
+ line.split("pthread-child ")
+ .nth(1)
+ .and_then(|pid| pid.parse::().ok())
+ })
+ });
+ f.owned
+ .borrow_mut()
+ .push(identity(child).expect("pthread child identity"));
+ eventually(|| (process_state(child) == Some(b'Z') && live_task(child)).then_some(()));
+ if lose_guard {
+ drop(f.take_stop_lock());
+ // SAFETY: the captured guard belongs to this fixture.
+ assert_eq!(
+ unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) },
+ 0
+ );
+ assert!(wait(&mut caller, TIMEOUT).success());
+ }
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "stopped");
+ eventually(|| (!live_task(child)).then_some(()));
+ if !lose_guard {
+ assert!(wait(&mut caller, TIMEOUT).success());
+ }
+ }
+}
+
+#[cfg(target_os = "linux")]
+fn live_task(pid: i32) -> bool {
+ fs::read_dir(format!("/proc/{pid}/task"))
+ .into_iter()
+ .flatten()
+ .flatten()
+ .any(|task| {
+ fs::read_to_string(task.path().join("stat"))
+ .ok()
+ .and_then(|stat| stat.rsplit_once(") ").map(|(_, rest)| rest.bytes().next()))
+ .flatten()
+ .is_some_and(|state| state != b'Z' && state != b'X')
+ })
+}
+
fn process_is_zombie(pid: i32) -> bool {
process_state(pid) == Some(b'Z')
}
@@ -1230,6 +1359,9 @@ fn unreaped_matching_guard_proves_legacy_session_ownership() {
let mut caller = ForeignChild(caller);
f.await_log("fixture-ready");
let mut state = f.state("worker");
+ // Freeze the parent only after launch has released the stop lock;
+ // otherwise the frozen parent would hold it and stop could never run.
+ drop(f.take_stop_lock());
// Keep the parent from reaping its guard, then exercise legacy birth absence.
assert_eq!(
unsafe { libc::kill(caller.0.id() as i32, libc::SIGSTOP) },
@@ -1275,6 +1407,8 @@ fn guard_dying_during_stop_gets_a_fresh_authored_term_grace() {
f.await_log("fixture-ready");
let guard = pid(&f.state("worker"), "guardPid");
let birth = identity(guard).unwrap();
+ // Freeze only after launch and guard startup leases released the stop lock.
+ drop(f.take_stop_lock());
// Suspend this owned guard so its pending TERM cannot reach the command.
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
let mut stop = command(&f.root, "stop").spawn().unwrap();
@@ -1522,6 +1656,8 @@ fn unresponsive_guard_stop_remains_unknown() {
f.await_log("fixture-ready");
let state = f.state("worker");
let guard = pid(&state, "guardPid");
+ // Freeze only after launch and guard startup leases released the stop lock.
+ drop(f.take_stop_lock());
// SAFETY: the guard belongs to this fixture and its receipt is immutable.
assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0);
let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
@@ -1904,6 +2040,44 @@ fn fixture_process() {
}
println!("wrapper-child {child}");
}
+ #[cfg(target_os = "linux")]
+ "pthread-main-exit" => {
+ extern "C" fn worker(_: *mut libc::c_void) -> *mut libc::c_void {
+ loop {
+ // SAFETY: pause has no preconditions.
+ unsafe {
+ libc::pause();
+ }
+ }
+ }
+ // The fork child is single-threaded, so its main thread is the
+ // thread-group leader. It starts one worker and ends only its main
+ // thread (as `pthread_exit` does); Linux then shows the leader as
+ // `Z` while the worker runs.
+ // SAFETY: the child calls only libc thread and exit functions and
+ // never returns into Rust/libtest state.
+ let child = unsafe { libc::fork() };
+ assert!(child >= 0);
+ if child == 0 {
+ unsafe {
+ let mut thread: libc::pthread_t = std::mem::zeroed();
+ if libc::pthread_create(
+ &mut thread,
+ std::ptr::null(),
+ worker,
+ std::ptr::null_mut(),
+ ) != 0
+ {
+ libc::_exit(96);
+ }
+ // End only the main thread, as pthread_exit would, without
+ // its unwinder: the raw exit syscall is thread-scoped.
+ libc::syscall(libc::SYS_exit, 0);
+ libc::_exit(95);
+ }
+ }
+ println!("pthread-child {child}");
+ }
"hold" => (),
_ => std::process::exit(99),
}
From b4317ec2713ac4b2eda5757fd70e74c0e420f369 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 03:20:34 +0000
Subject: [PATCH 35/45] fix(supervisor): never report success when the reply's
receipt read fails
Co-Authored-By: Claude Opus 5.5
---
.../lifecycle-supervisor/src/supervisor.rs | 27 +++++++++++--------
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 85ea68fff..162c309fb 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -93,18 +93,23 @@ fn control(action: &str) -> Result<()> {
}
};
let mut reason = "io-error";
- let result = control_effects(action, &request, &root, &mut reason);
+ let mut result = control_effects(action, &request, &root, &mut reason);
// Receipts are atomically replaced files, so this read is safe without the
- // lock, also after a lock wait expired.
- let processes = root
- .states()
- .map(|states| {
- states
- .into_iter()
- .filter(|state| state.owner_token == request.owner_token)
- .collect()
- })
- .unwrap_or_default();
+ // lock, also after a lock wait expired. A failed read never reports
+ // success: it keeps an earlier error, or becomes `io-error` itself.
+ let processes = match root.states() {
+ Ok(states) => states
+ .into_iter()
+ .filter(|state| state.owner_token == request.owner_token)
+ .collect(),
+ Err(error) => {
+ if result.is_ok() {
+ reason = "io-error";
+ result = Err(error);
+ }
+ Vec::new()
+ }
+ };
let outcome = match (&result, action) {
(Err(_), _) => "unknown",
(Ok(()), "stop") => "stopped",
From 60d737b59ddebddb7f052afc375a45852de7a83a Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 03:57:09 +0000
Subject: [PATCH 36/45] fix(supervisor): reread a receipt replaced while it was
being opened
Root::file opened a name and then required nlink == 1. A guard replaces
its receipt atomically (temporary file, then rename), so a stop or status
listing receipts at that moment could open the old inode after it was
unlinked, see nlink 0 and refuse. The stop then failed with an io-error
receipt. That is the Intel macOS CI failure of
unresponsive_guard_stop_remains_unknown and, by the same path, the
cleanup-stop failure of expired_target_does_not_abandon_other_recovery:
both resume a frozen guard and stop at once while the guard rewrites its
receipt. An unlinked inode now means "reopen the name"; any other link
count is still refused.
Regression: storage::tests::receipt_reads_survive_concurrent_atomic_
replacement lists receipts against a concurrent writer for 3 s; it fails
after about 2,900 reads without the fix.
Failure reasons are more specific: stop-lock-failed, receipt-read-failed
and retire-failed now separate the steps that io-error covered.
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 5 +
packages/lifecycle-supervisor/src/storage.rs | 134 ++++++++++++++----
.../lifecycle-supervisor/src/supervisor.rs | 14 +-
3 files changed, 125 insertions(+), 28 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 95e0c8766..d021bde12 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -69,6 +69,9 @@ fails. A failed request exits nonzero and carries a `reason`:
| `foreign-live-process` | `unknown` | A receipt from another owner token still has live members. |
| `stop-incomplete` | `unknown` | Stop could not confirm that every member is gone. |
| `retired-owner-live` | `unknown` | Status for a retired owner whose members are not proven absent. |
+| `stop-lock-failed` | `unknown` | Taking `.stop.lock` failed for a reason other than the wait expiring. |
+| `receipt-read-failed` | `unknown` | The process receipts could not be read. |
+| `retire-failed` | `unknown` | Stop could not durably retire the owner token; nothing was signalled. |
| `io-error` | `unknown` | Any other local failure. |
A successful reply has no `reason`. The sixteen-second stop-lock wait exceeds the
@@ -143,6 +146,8 @@ leader identity and each remaining member's UID, birth and session. Recovery
delivers TERM individually, grants a fresh five-second grace, then KILL to the captured incarnations and
confirms absence. The capture remains authoritative if TERM ends the witness;
every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux).
+Readers that open a receipt while a guard atomically replaces it reopen the name, so a
+concurrent update never reads as a refusal.
On Linux a thread-group leader whose main thread exited while other threads run shows
`Z` in `/proc//stat`; it counts as live until no thread in `/proc//task` is. A pidfd open that
reports the process missing (`ESRCH`, or `EINVAL` on Linux 6.8 when the PID survives only as a group or
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index e2d0ad6fa..3108dfa1b 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -81,31 +81,41 @@ impl Root {
pub fn file(&self, name: &str, flags: i32) -> Result {
let name = std::ffi::CString::new(name).map_err(|_| refused())?;
- // SAFETY: directory descriptor is live; name is NUL-terminated; returned
- // descriptor is newly owned and immediately wrapped in RAII.
- let fd = unsafe {
- libc::openat(
- self.directory.as_raw_fd(),
- name.as_ptr(),
- flags | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK,
- 0o600,
- )
- };
- if fd < 0 {
- return Err(std::io::Error::last_os_error());
- }
- // SAFETY: fd was just returned by openat and has no other owner.
- let file = File::from(unsafe { OwnedFd::from_raw_fd(fd) });
- let metadata = file.metadata()?;
- // SAFETY: geteuid has no preconditions.
- if !metadata.is_file()
- || metadata.uid() != unsafe { libc::geteuid() }
- || metadata.nlink() != 1
- || metadata.mode() & 0o777 != 0o600
- {
- return Err(refused());
+ // A receipt is replaced atomically (temporary file, then rename). An open
+ // that races that rename can return the replaced inode, which is already
+ // unlinked (nlink 0). Reopen the name to read its current inode; any
+ // other link count is still refused. The bound only stops a livelock.
+ for _ in 0..64 {
+ // SAFETY: directory descriptor is live; name is NUL-terminated;
+ // returned descriptor is newly owned and immediately wrapped in RAII.
+ let fd = unsafe {
+ libc::openat(
+ self.directory.as_raw_fd(),
+ name.as_ptr(),
+ flags | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK,
+ 0o600,
+ )
+ };
+ if fd < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ // SAFETY: fd was just returned by openat and has no other owner.
+ let file = File::from(unsafe { OwnedFd::from_raw_fd(fd) });
+ let metadata = file.metadata()?;
+ if metadata.nlink() == 0 {
+ continue;
+ }
+ // SAFETY: geteuid has no preconditions.
+ if !metadata.is_file()
+ || metadata.uid() != unsafe { libc::geteuid() }
+ || metadata.nlink() != 1
+ || metadata.mode() & 0o777 != 0o600
+ {
+ return Err(refused());
+ }
+ return Ok(file);
}
- Ok(file)
+ Err(refused())
}
pub fn lock(&self, name: &str) -> Result {
@@ -241,3 +251,79 @@ impl Root {
Ok(states)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::{Root, State};
+ use std::os::unix::fs::DirBuilderExt;
+
+ fn state(phase: &str) -> State {
+ State {
+ version: 1,
+ name: "worker".into(),
+ owner_token: "synthetic".into(),
+ guard_pid: 44,
+ process_group_id: 44,
+ guard_start_time: 100,
+ boot_id: String::new(),
+ uid: 501,
+ command_pid: None,
+ command_start_time: None,
+ phase: phase.into(),
+ exit_code: None,
+ signal: None,
+ }
+ }
+
+ /// A guard atomically replaces its receipt while a stop or status lists
+ /// receipts. A reader that opened the old inode just before the rename sees
+ /// it unlinked; that must read as the current receipt, never as a refusal.
+ #[test]
+ fn receipt_reads_survive_concurrent_atomic_replacement() {
+ let base = std::env::temp_dir().join(format!(
+ "storage-replace-{}-{}",
+ std::process::id(),
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ ));
+ std::fs::DirBuilder::new()
+ .mode(0o700)
+ .create(&base)
+ .unwrap();
+ let root = Root::open(&base).unwrap();
+ root.save(&state("running")).unwrap();
+ let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let writer = {
+ let base = base.clone();
+ let stop = stop.clone();
+ std::thread::spawn(move || {
+ let root = Root::open(&base).unwrap();
+ let mut flip = false;
+ while !stop.load(std::sync::atomic::Ordering::Relaxed) {
+ flip = !flip;
+ root.save(&state(if flip { "stopping" } else { "running" }))
+ .unwrap();
+ }
+ })
+ };
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3);
+ let mut reads = 0u64;
+ let mut failure = None;
+ while std::time::Instant::now() < deadline {
+ match root.states() {
+ Ok(states) => assert_eq!(states.len(), 1),
+ Err(error) => {
+ failure = Some(error);
+ break;
+ }
+ }
+ reads += 1;
+ }
+ stop.store(true, std::sync::atomic::Ordering::Relaxed);
+ writer.join().unwrap();
+ let _ = std::fs::remove_dir_all(&base);
+ assert!(failure.is_none(), "read {reads} failed: {failure:?}");
+ }
+}
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 162c309fb..e06499614 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -104,7 +104,7 @@ fn control(action: &str) -> Result<()> {
.collect(),
Err(error) => {
if result.is_ok() {
- reason = "io-error";
+ reason = "receipt-read-failed";
result = Err(error);
}
Vec::new()
@@ -133,10 +133,12 @@ fn control_effects(
match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) {
Ok(lock) => Some(lock),
Err(error) => {
- if error.kind() == std::io::ErrorKind::TimedOut {
+ *reason = if error.kind() == std::io::ErrorKind::TimedOut {
// Another stop or a launch still holds the lock; act on nothing.
- *reason = "stop-lock-wait-expired";
- }
+ "stop-lock-wait-expired"
+ } else {
+ "stop-lock-failed"
+ };
return Err(error);
}
}
@@ -144,7 +146,9 @@ fn control_effects(
None
};
let mut states = Vec::new();
+ *reason = "receipt-read-failed";
for state in root.states()? {
+ *reason = "io-error";
if state.owner_token == request.owner_token {
states.push(state);
} else if os::recorded_live(&state, false)? {
@@ -155,7 +159,9 @@ fn control_effects(
return Err(refused());
}
}
+ *reason = "io-error";
if action == "stop" {
+ *reason = "retire-failed";
root.retire(&request.owner_token)?;
*reason = "stop-incomplete";
return stop(&states);
From ca0fa30779793c6c0db262506899adb211a584eb Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 06:38:32 +0000
Subject: [PATCH 37/45] fix(supervisor): append log records without a lock so
no guard waits on a sibling
Every guard took a blocking flock on the shared lifecycle.log around each
record. A guard stopped (SIGSTOP, a debugger, a hung write) while holding
it stalled every sibling guard's loop in flock: no output, no exit
detection, no stop handling. Captured on macOS as the 9/100 failure of
expired_target_does_not_abandon_other_recovery (worker guard blocked in
Lines::flush -> flock while the frozen blocked guard held the lock).
Each record is now one write on the O_APPEND descriptor with no lock. The
kernel places each append atomically, so whole records from different
guards never interleave on a local file system; only a short write (for
example a full disk) can split one. The unused flock retry helper and its
unit test are removed.
Regressions: held_log_lock_never_stalls_a_guard holds the log flock from
outside for the whole run and requires output to reach the log and stop
to answer stopped within the normal bound (fails with the old lock);
concurrent_guards_append_whole_records_without_a_lock checks two guards
bursting 64 KiB records leave only whole, unmixed records. Docs state the
log's ordering and integrity and list every lock, who can hold it across
a blocking call, and the bound.
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 24 +++++++
packages/lifecycle-supervisor/README.md | 2 +-
packages/lifecycle-supervisor/src/os.rs | 39 -----------
.../lifecycle-supervisor/src/supervisor.rs | 14 ++--
.../lifecycle-supervisor/tests/fake_caller.rs | 69 +++++++++++++++++++
5 files changed, 101 insertions(+), 47 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index d021bde12..637dbce34 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -100,6 +100,30 @@ longer than 64 KiB become consecutive records so memory remains bounded. An
unterminated final line is retained. Records from different streams can interleave;
there is no total ordering guarantee between stdout and stderr.
+Each record is written with one `write` on an `O_APPEND` descriptor and no lock. On a
+local file system the kernel places each append atomically at the end of the file, so
+whole records from different guards never interleave; records from different guards
+appear in the order their appends happened. A short write (for example a full disk)
+can split one record. No guard waits for another guard to log, so a stopped or hung
+guard cannot stall a sibling guard's output, exit detection or stop handling.
+
+## Locks and waits
+
+A guard stopped at any instruction (SIGSTOP, a debugger) can hold only what it already
+had. No sibling guard waits on it, and every request that can wait on it has a bound:
+
+| Lock | Held by | Held across a blocking call? | Who waits, and the bound |
+|---|---|---|---|
+| `.instance.lock` | The project supervisor (`run`), for its lifetime | Yes, its whole run | Another `run` for the root: retries 1 s, then refused. Guards, `stop` and `status` never take it. |
+| `.stop.lock` | `run` from admission until its ready receipt; each new guard (inherited on fd 2) until it publishes and detaches; `stop` for its duration; the project supervisor while it stops on a signal | Yes. A guard frozen before it publishes keeps the lease until it is continued or killed | `stop` and launches wait up to 16 s, then answer `unknown` with reason `stop-lock-wait-expired` and act on nothing. `run` waits up to 10 s for each guard's receipt, then rolls back. |
+| `.lock` | The guard of that command, for its lifetime | Yes | Only a new guard for the same command name: up to 10 s, then refused. Never a sibling, `stop` or `status`. |
+| Receipts, retirement marker, `lifecycle.log` | No lock | No | Receipts are replaced atomically; a reader that races a replacement reopens the name. Log records are single appends (above). |
+
+`status` takes no lock. `stop` handles each target separately: a frozen live guard costs
+its own target the five-second grace plus the two-second confirmation bound (and the
+same again if the guard is lost), then that target reports `unknown` while the others
+complete.
+
All descriptors the supervisor opens are close-on-exec. The command receives only
its intended standard descriptors: null stdin, piped stdout and stderr. On macOS,
creation and setting close-on-exec are not always atomic. Delivery uses explicit
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index 7bfa522a4..fce258562 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -60,7 +60,7 @@ replaces stdio with `/dev/null`. Supervisor death therefore cannot let stop pass
unpublished guard that might later spawn. Authored children get their own stderr
pipe and never inherit that lease. Startup closes inherited descriptors with Linux
`close_range` or enumeration of actual `/proc/self/fd` / `/dev/fd` entries; its cost
-is independent of the descriptor limit. Log flock acquisition and release retry EINTR.
+is independent of the descriptor limit. Log records are single unlocked `O_APPEND` writes, so no guard waits on a sibling to log.
A usable process census is required before authored spawn. Linux preflight rejects
known `hidepid` mounts and requires a readable self identity in the census. An
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index e61767cc6..2d1c3f26d 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -857,26 +857,6 @@ fn readable_self(members: &[Identity]) -> Result<()> {
Ok(())
}
-pub fn flock_retry(fd: i32, operation: i32) -> Result<()> {
- retry_interrupted(|| {
- // SAFETY: caller retains ownership of the live descriptor across retries.
- if unsafe { libc::flock(fd, operation) } == 0 {
- Ok(())
- } else {
- Err(io::Error::last_os_error())
- }
- })
-}
-
-fn retry_interrupted(mut operation: impl FnMut() -> Result<()>) -> Result<()> {
- loop {
- match operation() {
- Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
- result => return result,
- }
- }
-}
-
pub fn nonblocking(fd: &impl AsRawFd) -> Result<()> {
// SAFETY: fcntl borrows a live descriptor and supplies integer flags only.
let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) };
@@ -1057,25 +1037,6 @@ mod tests {
assert!(super::recorded_guard_matches(&state, &identity));
}
- #[test]
- fn flock_interruption_retries_but_other_errors_do_not() {
- let mut calls = 0;
- super::retry_interrupted(|| {
- calls += 1;
- if calls < 3 {
- Err(std::io::Error::from_raw_os_error(libc::EINTR))
- } else {
- Ok(())
- }
- })
- .unwrap();
- assert_eq!(calls, 3);
- let error =
- super::retry_interrupted(|| Err(std::io::Error::from_raw_os_error(libc::EPERM)))
- .unwrap_err();
- assert_eq!(error.raw_os_error(), Some(libc::EPERM));
- }
-
#[test]
fn unusable_census_refuses_before_launch() {
assert!(super::readable_self(&[]).is_err());
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index e06499614..075979dac 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -746,13 +746,13 @@ impl Lines {
};
let message = String::from_utf8_lossy(bytes).replace('\t', " ");
let record = format!("{}\t{name}\t{stream}\t{message}\n", os::timestamp()?);
- // One append per complete record; concurrent guards share the same
- // flock so records larger than PIPE_BUF never interleave.
- os::flock_retry(log.as_raw_fd(), libc::LOCK_EX)?;
- let result = log.write_all(record.as_bytes());
- let unlocked = os::flock_retry(log.as_raw_fd(), libc::LOCK_UN);
- result?;
- unlocked?;
+ // One write per complete record on an O_APPEND descriptor, with no
+ // lock: the kernel positions and writes each append atomically, so
+ // whole records from sibling guards do not interleave on a local file
+ // system. A guard never waits on another guard to log, so a stopped or
+ // hung sibling cannot stall this guard's loop (STOP, exit, census).
+ // Only a short write (for example a full disk) splits a record.
+ log.write_all(record.as_bytes())?;
self.pending.clear();
Ok(())
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index d0d1f9f36..a37a5d649 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -136,6 +136,7 @@ impl Fixture {
fs::read_to_string(self.root.join("lifecycle.log")).unwrap_or_default()
}
+ #[track_caller]
fn await_log(&self, needle: &str) {
eventually(|| self.log().contains(needle).then_some(()));
}
@@ -529,6 +530,46 @@ fn output_bursts_and_long_lines_include_partial_tail() {
}
}
+/// Without a log lock, each record is one O_APPEND write. Two guards bursting
+/// at once, including 64 KiB records far above PIPE_BUF, must still leave only
+/// whole records: every line parses, belongs to one guard and is not mixed.
+#[test]
+fn concurrent_guards_append_whole_records_without_a_lock() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![
+ f.process("alpha", "burst"),
+ f.process("beta", "burst"),
+ ]);
+ assert!(wait(&mut caller, TIMEOUT).success());
+ let text = f.log();
+ for name in ["alpha", "beta"] {
+ let own: Vec<_> = text
+ .lines()
+ .filter(|line| line.split('\t').nth(1) == Some(name))
+ .collect();
+ assert_eq!(
+ own.iter().filter(|l| l.contains("\tburst-record-")).count(),
+ 2000
+ );
+ let xs: usize = own
+ .iter()
+ .filter_map(|line| line.splitn(4, '\t').nth(3))
+ .filter(|message| message.starts_with('X'))
+ .map(|message| {
+ assert!(message.bytes().all(|b| b == b'X'), "mixed record");
+ message.len()
+ })
+ .sum();
+ assert_eq!(xs, 300_000);
+ }
+ for line in text.lines() {
+ let fields: Vec<_> = line.splitn(4, '\t').collect();
+ assert_eq!(fields.len(), 4, "split record: {line:.80}");
+ assert!(matches!(fields[1], "alpha" | "beta"), "{line:.80}");
+ assert!(fields[0].ends_with('Z'));
+ }
+}
+
#[test]
fn zombies_do_not_hold_liveness() {
let f = Fixture::new();
@@ -1308,6 +1349,34 @@ fn live_task(pid: i32) -> bool {
})
}
+/// A guard stopped while it held a lock on the shared log (SIGSTOP between
+/// lock and unlock) must not stall sibling guards. The test holds an exclusive
+/// flock on the log from outside for the whole run, as such a frozen guard
+/// would: a guard's output must still reach the log and its project must
+/// still stop with a framed `stopped` receipt within the normal bounds.
+#[test]
+fn held_log_lock_never_stalls_a_guard() {
+ use std::os::unix::fs::OpenOptionsExt;
+ let f = Fixture::new();
+ let log = fs::OpenOptions::new()
+ .append(true)
+ .create(true)
+ .mode(0o600)
+ .open(f.root.join("lifecycle.log"))
+ .unwrap();
+ // SAFETY: flock only borrows this live descriptor.
+ assert_eq!(unsafe { libc::flock(log.as_raw_fd(), libc::LOCK_EX) }, 0);
+ let (mut caller, _) = f.start(vec![f.process("worker", "hold")]);
+ f.await_log("fixture-ready");
+ let start = Instant::now();
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(start.elapsed() < Duration::from_secs(7));
+ assert!(wait(&mut caller, TIMEOUT).success());
+ drop(log);
+}
+
fn process_is_zombie(pid: i32) -> bool {
process_state(pid) == Some(b'Z')
}
From 15bb1d01262824dd7db806d0c48de94db8d026b8 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 06:40:09 +0000
Subject: [PATCH 38/45] docs(supervisor): separate stop and launch lock-wait
outcomes; name short-write causes
Co-Authored-By: Claude Opus 5.5
---
docs/lifecycle-supervisor.md | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index 637dbce34..d3920c32b 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -103,19 +103,21 @@ there is no total ordering guarantee between stdout and stderr.
Each record is written with one `write` on an `O_APPEND` descriptor and no lock. On a
local file system the kernel places each append atomically at the end of the file, so
whole records from different guards never interleave; records from different guards
-appear in the order their appends happened. A short write (for example a full disk)
-can split one record. No guard waits for another guard to log, so a stopped or hung
+appear in the order their appends happened. A short write (a full disk, a file-size
+limit, or a signal after partial progress) is completed by a second write and can split
+one record. No guard waits for another guard to log, so a stopped or hung
guard cannot stall a sibling guard's output, exit detection or stop handling.
## Locks and waits
A guard stopped at any instruction (SIGSTOP, a debugger) can hold only what it already
-had. No sibling guard waits on it, and every request that can wait on it has a bound:
+had. No sibling guard waits on it, and every request that can wait on it has a bound.
+The bounds below limit each wait, not the total duration of a request:
| Lock | Held by | Held across a blocking call? | Who waits, and the bound |
|---|---|---|---|
| `.instance.lock` | The project supervisor (`run`), for its lifetime | Yes, its whole run | Another `run` for the root: retries 1 s, then refused. Guards, `stop` and `status` never take it. |
-| `.stop.lock` | `run` from admission until its ready receipt; each new guard (inherited on fd 2) until it publishes and detaches; `stop` for its duration; the project supervisor while it stops on a signal | Yes. A guard frozen before it publishes keeps the lease until it is continued or killed | `stop` and launches wait up to 16 s, then answer `unknown` with reason `stop-lock-wait-expired` and act on nothing. `run` waits up to 10 s for each guard's receipt, then rolls back. |
+| `.stop.lock` | `run` from admission until its ready receipt; each new guard (inherited on fd 2) until it publishes and detaches; `stop` for its duration; the project supervisor while it stops on a signal | Yes. A guard frozen before it publishes keeps the lease until it is continued or killed | `stop` waits up to 16 s, then answers `unknown` with reason `stop-lock-wait-expired` and acts on nothing. A launch (`run`) waits up to 16 s, then exits nonzero without a ready receipt and starts nothing. `run` waits up to 10 s for each guard's receipt, then rolls back. |
| `.lock` | The guard of that command, for its lifetime | Yes | Only a new guard for the same command name: up to 10 s, then refused. Never a sibling, `stop` or `status`. |
| Receipts, retirement marker, `lifecycle.log` | No lock | No | Receipts are replaced atomically; a reader that races a replacement reopens the name. Log records are single appends (above). |
From 1e7b55e1699a42a8930120aa8a4c645db7b761c7 Mon Sep 17 00:00:00 2001
From: Kit <344842+roodboi@users.noreply.github.com>
Date: Sun, 15 Nov 2026 09:18:32 +0000
Subject: [PATCH 39/45] fix(supervisor): stop healthy targets past an
unverifiable receipt; recheck Linux identities
Round-4 finding 4: one receipt of the owner that could not be verified
(guard gone, no witness for its session) returned from stop admission
before any target was signalled, so healthy commands of the same owner
kept running. Admission now records that failure and skips only that
receipt, which is never signalled; the other targets get their normal
stop, and the reply is still unknown with stop-incomplete. Regression:
unverifiable_receipt_does_not_block_healthy_targets (fails with the old
abort).
Round-5 finding 1 (Linux): a single listing of /proc//task could read
a process whose main thread exited as a zombie if every listed thread was
replaced during the scan. "No live thread" now needs two consecutive
listings to agree. inspect also re-reads the leader's start time after
assembling an identity from stat, status and the task list, and repeats
the reads if the PID was reused in between, instead of combining two
processes.
Co-Authored-By: Claude Opus 5.5
---
packages/lifecycle-supervisor/src/os.rs | 78 ++++++++++++++-----
.../lifecycle-supervisor/src/supervisor.rs | 59 ++++++++------
.../lifecycle-supervisor/tests/fake_caller.rs | 45 +++++++++++
3 files changed, 138 insertions(+), 44 deletions(-)
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index f6c54b95f..31d9c3d28 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -63,48 +63,86 @@ pub struct Identity {
pub zombie: bool,
}
+/// One `/proc//stat` read: state letter, group, session and start time.
#[cfg(target_os = "linux")]
-pub fn inspect(pid: i32) -> Result