diff --git a/.github/workflows/lifecycle-supervisor.yml b/.github/workflows/lifecycle-supervisor.yml new file mode 100644 index 000000000..54bd02643 --- /dev/null +++ b/.github/workflows/lifecycle-supervisor.yml @@ -0,0 +1,46 @@ +name: Lifecycle supervisor + +on: + pull_request: + paths: + - packages/lifecycle-supervisor/** + - .github/workflows/lifecycle-supervisor.yml + push: + branches: [next, main] + paths: + - packages/lifecycle-supervisor/** + - .github/workflows/lifecycle-supervisor.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: lifecycle-supervisor-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + contract: + name: Supervisor (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-15, macos-15-intel] + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + env: + HACK_DAEMON_AUTOSTART_DISABLE: '1' + CARGO_TARGET_DIR: ${{ github.workspace }}/.hack-local/target/lifecycle-supervisor + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.97.1 + with: + components: rustfmt, clippy + - name: Check formatting + run: cargo fmt --manifest-path packages/lifecycle-supervisor/Cargo.toml --check + - name: Lint standalone supervisor + run: cargo clippy --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --all-targets -- -D warnings + - name: Test standalone supervisor with private fake callers + run: cargo test --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml -- --test-threads=4 + - name: Build standalone release executable + run: cargo build --locked --release --manifest-path packages/lifecycle-supervisor/Cargo.toml diff --git a/docs/diagrams/lifecycle-supervisor.mmd b/docs/diagrams/lifecycle-supervisor.mmd new file mode 100644 index 000000000..c9fa975bf --- /dev/null +++ b/docs/diagrams/lifecycle-supervisor.mmd @@ -0,0 +1,31 @@ +--- +title: Lifecycle supervisor · ownership and recovery +--- +stateDiagram-v2 + direction TB + [*] --> Absent + Absent --> Starting : Reserve [locks acquired] + Starting --> Running : Spawn [identity durable] + Starting --> Exited : StartFailure + Running --> Recoverable : SupervisorDeath + Recoverable --> Recoverable : Retry [duplicate refused] + Running --> Orphaned : GuardDeath [members live] + Recoverable --> Orphaned : GuardDeath [members live] + Recoverable --> Stopping : Stop [identity matches] + Orphaned --> Stopping : RecoverStop [exact witness; capture members] + Orphaned --> StopUnknown : RecoverStop [identity unconfirmed] + Running --> Stopping : Stop [identity matches] + Running --> Exited : Reap [no live members] + Stopping --> Exited : Reap [no live members] + Stopping --> KillPending : Kill [grace elapsed] + Stopping --> Stopping : WitnessExit [captured identities retained] + KillPending --> Exited : Reap [no live members] + KillPending --> StopUnknown : StopUnknown [members live] + note right of Starting + INV identity precedes command + end note + note right of Orphaned + Unknown health retains ownership + No exact witness refuses capture + Live session refuses duplicate + end note diff --git a/docs/diagrams/lifecycle-supervisor.svg b/docs/diagrams/lifecycle-supervisor.svg new file mode 100644 index 000000000..91c811ee9 --- /dev/null +++ b/docs/diagrams/lifecycle-supervisor.svg @@ -0,0 +1,66 @@ +Lifecycle supervisor · ownership and recoveryReserve [locks acquired]Spawn [identity durable]StartFailureSupervisorDeathRetry [duplicate refused]GuardDeath [members live]GuardDeath [members live]Stop [identity matches]RecoverStop [exact witnessRecoverStop [identityunconfirmed]Stop [identity matches]Reap [no live members]Reap [no live members]Kill [grace elapsed]WitnessExit [capturedidentities retained]Reap [no live members]StopUnknown [members live]AbsentRunningExitedRecoverableStopping;capturemembers]StopUnknownKillPendingStartingOrphanedINV identity precedes commandUnknown health retains ownershipNo exact witness refuses captureLive session refuses duplicate \ No newline at end of file diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md new file mode 100644 index 000000000..d3920c32b --- /dev/null +++ b/docs/lifecycle-supervisor.md @@ -0,0 +1,336 @@ +# Lifecycle supervisor: standalone contract + +This first slice builds a standalone Rust supervisor with a fake caller. It does +not change `hack up`, configuration, or the default runner. CLI integration follows +lifecycle retirement changes; packaging is a separate change. The current default +continues to use the lifecycle mux session described in [lifecycle](./lifecycle.md). + +## Artifact choice + +`packages/lifecycle-supervisor` builds `hack-lifecycle-supervisor`. A separate crate +adds one executable, but compiles only its small process-management dependency set. +A config-compiler subcommand would save that artifact while mixing pure compilation +with process, signal and recovery code. A runtime-core subcommand would bring the +broader native runtime dependency graph into Compose installations. The separate +crate keeps those build costs and failure boundaries independent. + +The supported build targets are `aarch64-apple-darwin`, `x86_64-apple-darwin`, and +`x86_64-unknown-linux-gnu`. A successful Linux run does not qualify macOS behavior. + +## Protocol and private state + +The binary accepts `run`, `stop`, and `status`. Input and output are one four-byte +big-endian length followed by that many JSON bytes. Frames are bounded to 1 MiB; +each frame read or write has one 10-second deadline. A complete frame, rather than descriptor close, +commits delivery. Environment values travel only in the input frame and process +memory. They are never serialized into state, command-line arguments, or errors. +Authored commands can themselves print values; output remains unredacted and private. + +`run` accepts `{version: 1, root, ownerToken, processes}`. `root` is an absolute, +existing, owner-only directory for one project instance. `ownerToken` is a fresh +opaque ownership token supplied by the caller. Each process has `name`, `argv`, +absolute `cwd`, and `env`. Names are safe file identifiers and must also be unique +without regard to ASCII case, including on case-sensitive filesystems. Commands execute argv +directly. A shell-authored command can explicitly use `sh -c`; the supervisor adds +no shell wrapper. The full environment is explicit, with no ambient inheritance. + +`stop` and `status` accept `{version: 1, root, ownerToken}`. Read-only `status` +returns saved phases plus kernel-verified `liveProcessNames` and +`unknownProcessNames`; a saved phase alone is not a health check. It creates no files, +processes or ownership. Stop +requires matching ownership and a live identity before sending a signal. Unknown +identity is a refusal, never proof that a process is absent. A different boot or a +leader with a different start identity proves that the old session ended; it +grants no authority over the replacement. Guard-written terminal receipts are retained as proof that the old session ended. +A missing leader plus a numeric SID is not ownership: orphan recovery requires +the exact recorded command PID/birth or a matching guard, including a zombie. +Only unrecorded members, or legacy receipts without a surviving matching guard, report +unknown without signals. Descendants alone may require manual recovery when both recorded processes are gone. +Stop attempts to capture verified members before signaling even a live guard; +a failed capture does not prevent that healthy guard from handling shutdown. If that guard +is lost during stop, recovery prefers a fresh capture and falls back to those +admitted identities, rechecking each before TERM/KILL. Each recovered target gets +its own five-second TERM grace and two-second confirmation bound. Recovery begins +at most once per target; late guard loss can extend stop to roughly fourteen seconds. Unobserved descendants +still report unknown. A failed or expired target does not abandon other admitted +targets: stop drives them to their own outcomes before returning unknown. + +Replies contain `version`, `supervisorPid`, `processes`, and `outcome` (`ready`, +`observed`, `stopped`, `unknown` or `refused`). A stop whose confirmation times out emits +`unknown` and exits nonzero. Callers must inspect both the receipt and exit status. + +Every `stop` and `status` request ends with exactly one framed receipt, also when it +fails. A failed request exits nonzero and carries a `reason`: + +| `reason` | `outcome` | Meaning | +|---|---|---| +| `invalid-request` | `refused` | The control frame or root was rejected; nothing was read or changed. | +| `stop-lock-wait-expired` | `unknown` | Another stop or a launch held `.stop.lock` for the whole wait (sixteen seconds); this request acted on nothing. | +| `foreign-live-process` | `unknown` | A receipt from another owner token still has live members. | +| `stop-incomplete` | `unknown` | Stop could not confirm that every member is gone. | +| `retired-owner-live` | `unknown` | Status for a retired owner whose members are not proven absent. | +| `stop-lock-failed` | `unknown` | Taking `.stop.lock` failed for a reason other than the wait expiring. | +| `receipt-read-failed` | `unknown` | The process receipts could not be read. | +| `retire-failed` | `unknown` | Stop could not durably retire the owner token; nothing was signalled. | +| `io-error` | `unknown` | Any other local failure. | + +A successful reply has no `reason`. The sixteen-second stop-lock wait exceeds the +timers of a stop that recovers after late guard loss (about fourteen seconds), so a +waiter normally gets its own outcome; admission and census work run outside those +timers, which is why an expired wait still returns its own receipt. +`run` must be spawned as a fresh child without making it a process-group leader: +the supervisor calls `setsid` itself. A caller using `detached: true` or +`setpgid(0, 0)` is refused before authored code starts. + +The root contains per-command state and lock files, an instance lock and +`lifecycle.log`. Directories are mode 0700; files are mode 0600. Symlinks, +hard-linked files, foreign owners and broad permissions are refused. State contains +name, ownership token, guard PID, process group, OS start identity, boot identity, +UID, command PID and start identity when known, phase, and the direct command's exit code or signal +when observed. It contains no command or environment. +State is atomically published and synchronized before authored code can run. +The guard leads a new session in its own process group. The authored command +starts in a separate group within that session. Cleanup covers every live group +in the session, so a command moving into another process group stays owned. + +The log uses the existing lifecycle reader contract: +`UTC timestampservicestdout|stderr|metamessage`. +Stdout and stderr drain independently; tabs in messages become four spaces. Lines +longer than 64 KiB become consecutive records so memory remains bounded. An +unterminated final line is retained. Records from different streams can interleave; +there is no total ordering guarantee between stdout and stderr. + +Each record is written with one `write` on an `O_APPEND` descriptor and no lock. On a +local file system the kernel places each append atomically at the end of the file, so +whole records from different guards never interleave; records from different guards +appear in the order their appends happened. A short write (a full disk, a file-size +limit, or a signal after partial progress) is completed by a second write and can split +one record. No guard waits for another guard to log, so a stopped or hung +guard cannot stall a sibling guard's output, exit detection or stop handling. + +## Locks and waits + +A guard stopped at any instruction (SIGSTOP, a debugger) can hold only what it already +had. No sibling guard waits on it, and every request that can wait on it has a bound. +The bounds below limit each wait, not the total duration of a request: + +| Lock | Held by | Held across a blocking call? | Who waits, and the bound | +|---|---|---|---| +| `.instance.lock` | The project supervisor (`run`), for its lifetime | Yes, its whole run | Another `run` for the root: retries 1 s, then refused. Guards, `stop` and `status` never take it. | +| `.stop.lock` | `run` from admission until its ready receipt; each new guard (inherited on fd 2) until it publishes and detaches; `stop` for its duration; the project supervisor while it stops on a signal | Yes. A guard frozen before it publishes keeps the lease until it is continued or killed | `stop` waits up to 16 s, then answers `unknown` with reason `stop-lock-wait-expired` and acts on nothing. A launch (`run`) waits up to 16 s, then exits nonzero without a ready receipt and starts nothing. `run` waits up to 10 s for each guard's receipt, then rolls back. | +| `.lock` | The guard of that command, for its lifetime | Yes | Only a new guard for the same command name: up to 10 s, then refused. Never a sibling, `stop` or `status`. | +| Receipts, retirement marker, `lifecycle.log` | No lock | No | Receipts are replaced atomically; a reader that races a replacement reopens the name. Log records are single appends (above). | + +`status` takes no lock. `stop` handles each target separately: a frozen live guard costs +its own target the five-second grace plus the two-second confirmation bound (and the +same again if the guard is lost), then that target reports `unknown` while the others +complete. + +All descriptors the supervisor opens are close-on-exec. The command receives only +its intended standard descriptors: null stdin, piped stdout and stderr. On macOS, +creation and setting close-on-exec are not always atomic. Delivery uses explicit +frames with deadlines; process liveness comes from an OS census, and post-exit +output draining is bounded. None depends on another process closing an accidentally +inherited descriptor. + +## State model + +This transition table is the first model. Each row maps to a fake-caller regression +in `packages/lifecycle-supervisor/tests/fake_caller.rs`. `guard` means a resident Rust +child that owns the command's session and process group. The project supervisor +owns and reaps guards; each guard owns and reaps its direct command child. +The table's labels describe abstract control states and request outcomes. Persisted +phases use lowercase names; `Orphaned` and `StopUnknown` are recovery observations, +not claims that a dead or paused guard rewrote its receipt. + +| Before | Event and guard | Effect | After | Regression | +| --- | --- | --- | --- | --- | +| Absent | Start, instance and command locks acquired | Guard creates session; durable identity precedes command spawn | Starting | `starts_and_records_identity` | +| Starting | Command spawn succeeds | Publish command PID; deliver ready receipt | Running | `starts_and_records_identity` | +| Starting | Invalid executable or publication fails | No unrecorded command; terminate owned partial start | FailedStart/refused | `failed_start_runs_no_later_command` | +| Running | Stop with exact owner, boot and start identity | TERM authored session members | Stopping | `stop_after_caller_closed` | +| Stopping | No non-zombie members remain | Reap child; retain terminal evidence | Exited | `stop_after_caller_closed` | +| Stopping | Five-second grace expires, identity still matches | KILL authored session members, keeping the guard alive | KillPending | `term_ignoring_command_is_killed_after_grace` | +| KillPending | Census confirms no live members | Guard reaps and releases command lock | Exited | `term_ignoring_command_is_killed_after_grace` | +| Stopping/KillPending | Confirmation deadline expires with live members | Retain identity and command lock; report uncertainty | StopUnknown | `unresponsive_guard_stop_remains_unknown` | +| Running | Project supervisor dies | Guard and durable identity remain | Running/recoverable | `supervisor_kill_then_recover_stop` | +| Recoverable | Another start sees held command lock or live old identity | Refuse duplicate; preserve original | Recoverable | `supervisor_kill_then_recover_stop` | +| Running/recoverable | Guard itself is separately killed with live session members | Health unknown; refuse duplicate, permit verified session recovery stop | Orphaned | `dead_guard_with_live_session_recovers_term_then_kill` | +| Any live recorded | Owner differs or identity is unknown | Refuse; signal nothing | Unchanged | `foreign_owner_cannot_stop` | +| Any recorded | Boot differs or leader PID is reused | Treat old session as absent; signal no replacement | Unchanged | `stale_reused_session_leader_does_not_block_fresh_launch` | +| Running | Command exits; only zombies remain | Reap direct child; zombies do not hold liveness | Exited | `zombies_do_not_hold_liveness` | + +![Supervisor state transitions](./diagrams/lifecycle-supervisor.svg) + +A resident guard closes the publication gap: command ownership is durable before +spawn, and the group leader survives the project supervisor's death. The guard +keeps the command lock until its session has no live members, so a restarted project +supervisor cannot launch a second copy. If a guard itself is separately killed +while its command survives, health becomes unknown and duplicate launch is +refused. Stop can recover the recorded session after verifying the boot, +leader identity and each remaining member's UID, birth and session. Recovery +delivers TERM individually, grants a fresh five-second grace, then KILL to the captured incarnations and +confirms absence. The capture remains authoritative if TERM ends the witness; +every signal still rechecks that member's PID, birth, UID and session (pidfd on Linux). +Readers that open a receipt while a guard atomically replaces it reopen the name, so a +concurrent update never reads as a refusal. +On Linux a thread-group leader whose main thread exited while other threads run shows +`Z` in `/proc//stat`; it counts as live until no thread in `/proc//task` is. A pidfd open that +reports the process missing (`ESRCH`, or `EINVAL` on Linux 6.8 when the PID survives only as a group or +session ID) skips that member without a signal; liveness is still decided by identity afterwards. +Unobserved descendants never inherit this authority and retain an unknown outcome. Unverified identity never authorizes signals or a successful +stop. Launch and stop share a bounded lock; a startup descriptor lease keeps +admission locked through guard publication even if the project owner dies. +The guard temporarily inherits this stop-lock lease on stderr (fd 2), then closes +it by detaching stdio after publication. This descriptor is an internal ownership +lease, not a diagnostic stream; authored commands receive a separate stderr pipe. +On Darwin, processes of a different UID may be hidden by kernel inspection +permissions. Such descendants are unsupported: a readable census cannot prove +absence of a hidden different-UID member, and an ordinary owner cannot signal it. + +The table models process death and retry, not filesystem hardware failure, hostile +same-user mutation, commands that deliberately create new sessions, or power-loss +recovery of live processes. Commands that daemonize out of their session are outside +this ownership contract. Native tests qualify only their host OS. Darwin has no +pidfd: its identity-to-signal interval, and numeric group signaling on both +platforms, retain a small kernel observation/effect race. The model assumes the +observed identity remains valid across that interval. + +## Terminal behavior and later work + +The current mux runner supplies a terminal, though its FIFO wrapper redirects +command output. A command that prompts or requires a controlling terminal can +therefore depend on that mux environment today. The standalone supervisor provides +no PTY: stdin is `/dev/null`, output is piped, and there is no controlling terminal. +Such a command sees EOF or its own non-TTY failure. It must use a noninteractive +mode or remain in an interactive workspace until PTY support exists. + +Later work can add an explicit PTY mode and a small attach client so a persistent +interactive workspace can exist without tmux. An outside-tool provider interface +should deliver environment on stdin, with explicit framing and launch receipts. +Neither feature is part of this unit; interactive workspaces keep tmux by default. + +## Delivery gates + +The CLI opt-in is planned as `lifecycle.runner = "mux" | "supervisor"`, default +`mux`, plus a test environment override. Both runners must pass existing lifecycle +unit and e2e scenarios before that integration is ready. Packaging must include the +binary wherever the Compose CLI installs: stable archives, prerelease bundles, +Homebrew, self-upgrade and supported container distributions. Existing prerelease +packaging is macOS arm64 only; Intel/Linux prerelease support needs its own explicit +qualification rather than assuming the stable archive matrix covers it. + +The packaging PR inventory is: + +- New `scripts/build-lifecycle-supervisor.ts`, root `package.json`, and CLI build + wiring in `packages/cli/package.json` where needed. +- `scripts/build-release.ts`, `scripts/update-homebrew-tap.ts`, + `src/lib/release-install.ts`, and `src/lib/release-staging.ts`. +- `scripts/build-native-candidate.sh`, `scripts/prerelease-plan.ts`, + `scripts/installer/integrity.ts`, `scripts/installer/bundle.ts`, and retained + `scripts/install-prerelease.py` compatibility inventories. +- `.github/workflows/ci.yml`, `.github/workflows/release.yml`, + `.github/workflows/prerelease.yml`, and `docker/slim-runtime/Dockerfile`. +- `.hack/toolchain/run.sh`, `.hack/README.md`, and install/prerelease docs. +- Archive, Homebrew, upgrade and prerelease inventory regression tests, plus an + installed supervisor artifact check analogous to the config compiler check. + +Default migration removes the Python/FIFO wrapper in +`src/lib/project-lifecycle-controller.ts` and the shell execution gate in +`src/lib/shared-host-lifecycle.ts`. Finite shell-authored hooks continue to use +POSIX `sh -c` intentionally. Interactive tmux environment delivery stays separate. +Zellij removal, its breaking-change notice, and the workspace-only `MuxBackend` +contract belong to the third unit. + +## Resource and qualification boundaries + +A live direct child proves its guard is needed without a full-system census. +Guards wait for output or SIGCHLD; once only descendants remain, a 500 ms +fallback checks the session. Stop uses the short retirement tick. Startup closes +actual open descriptors rather than iterating the open-file limit. A known +restricted process view is refused before authored code; silently hidden +same-user processes remain unsupported. + +Linux debug-build measurements with one quiet guard and owner recorded less than +10 ms CPU accounting resolution per five-second sample and two voluntary context +switches per second, across three samples. These are neither hardware wakeup +counts nor macOS measurements. The host's hard descriptor limit was 524,288; +the exact 1,048,576 test requires a host that permits that limit. +Darwin compilation does not qualify execution, group-signal EPERM recovery or +idle cost on a real Mac. + +The 426-state TLA+ artifact is an abstract phase/ownership exploration. It models +SID reuse, an exact recorded-incarnation witness for orphan signaling, witness loss +after capture and the retained-snapshot KILL step. It does +not model actual kernel census observations, real flock behavior, recovery timing, +startup descriptor leases or signal errors. Some safety predicates hold by +construction. Native regressions, rather than that model, are the evidence for +those boundaries; see the model coverage document for the reduced claims. + +Stop durably retires its owner generation before confirming teardown, even if no +command receipt exists. A delayed run checks that fence under the admission lock +before spawning. New generations must use fresh owner tokens; markers persist in +the private root and contain no command environment. Retired-but-live work remains +unconfirmed. Linux guard spawning uses `/proc/self/exe`, preserving an executable +reference even after atomic helper replacement. + +Linux ownership checks read the process's effective UID from `/proc//status`. +They do not infer UID from proc-file ownership, which changes when a command +disables dumpability. Such a command remains recoverable after guard loss. + +Restart policy and restart backoff belong to the CLI integration in #811, not +this binary's command ownership loop. This crate does not restart commands. + +Logs are append-only and have no automatic rotation. Stop all generations and +confirm native absence before archiving the private root/log and starting with +a fresh root. Crash-left private .tmp files can be removed only while no writer +holds the root. Retirement markers must not be pruned individually: they fence +arbitrarily delayed frames. Retire the entire quiescent root only after all old +callers and guards have exited. These files have no automatic size/count bound; +long-running installations must arrange a maintenance window and offline rotation. + +Native crash tests currently cover prepublication command-lock blocking, incomplete +guard launch frames, and blocked ready delivery. Deterministic tests at the two +smaller instruction windows (starting-state save to spawn, and spawn to running-state +save) remain unqualified; polling a receipt cannot reliably stop a process in either +window. They require a dedicated synchronization seam, not a timing-only test. + +## Manual recovery after an unknown stop + +Stop retires the owner token before validating all receipts. One unknown receipt +can therefore leave otherwise healthy guards running and block every new start +in that private root. Repeating up with a new token does not bypass the uncertainty. +This also applies when a guard dies after spawn but before publishing running: +the receipt has no command birth evidence, so an unanchored session is unknown. + +1. Keep the private root and all receipts. Run status with the recorded token; + use the reported guard/command PIDs and session as leads for inspection, never + as authority to kill. Inspect current processes with the OS process tools + (ps on macOS; ps and /proc on Linux), including PID, parent PID, SID, UID, + start time and command. Identify the actual application processes yourself. +2. For each process you confirm belongs to this failed launch, record its current + start identity. Send TERM to those individual PIDs, wait at least five seconds, + and recheck each identity before sending KILL to any remaining process. Never + signal a numeric process group/session from the stale receipt alone. Leave + unfamiliar or inaccessible processes untouched and seek operator help. +3. Retry stop with the original token (or hack down through the same persisted + project owner). Once the uncertain session is absent, normal stop can retire + the remaining healthy guards. If another receipt is unknown, inspect it in the + same way. Confirm stopped/status absence before a fresh up with a fresh token. +4. Do not delete receipts, locks, retirement markers or the root to unblock up. + That discards the ownership evidence without stopping anything. Offline root + rotation is allowed only after all original callers/guards and authored work + have been independently confirmed gone. + +Darwin libproc requires a nonzero PROC_PIDTBSDINFO argument to include unreaped +zombies. The inspector requests it explicitly. A matching boot, PID, birth and +UID prove the original zombie guard has not been reused even when getsid rejects +it; this zombie never counts as live work and is never signalled. Live guards +and recovery members still require the recorded session and UID. + +Darwin start timestamps come from the wall clock and can decrease between guard +and command creation. Recovery therefore never orders those timestamps: it matches +PID and the recorded birth exactly. A backward step between those starts does not +invalidate a still-matching command witness. The native identity contract still +assumes a PID and exact recorded birth value identify one incarnation; Darwin has +no pidfd, so its short inspection-to-signal race remains as documented above. diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 5f2e3017c..769b34f44 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -4,6 +4,10 @@ Many projects need host-side setup around `hack up` (auth, local proxies, tunnel Lifecycle is configured in `.hack/hack.config.json` under `lifecycle`. +The [standalone Rust supervisor contract](./lifecycle-supervisor.md) describes +the separately tested supervisor work. It does not change the current mux runner +or expose a new configuration setting in this slice. + With the Compose backend, the first Ctrl-C during foreground `hack up` asks Compose to stop the project containers. Hack waits for Compose to finish stopping them and drains its logs before exiting with code 130. A second Ctrl-C is forwarded diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index dfdfd14df..33423b881 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,6 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | +| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 426 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. | | Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 74 states, two steps per phase, deferred Compose repair, separate publication and receipt, safe pre-removal cleanup refusal; unsafe controls fail `NoSameRunCleanup` and `FreshRunProof` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, admits later explicit Compose repair without republishing env layers, and requires a new successful project run plus fresh snapshot/delivery/live-input proofs for cleanup. Real-file tests check preview parity, sentinels, backup damage, consent, tighter modes, refused cleanup retry and rollback; owned child processes receive SIGKILL at initial and later journal/publication boundaries | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. | | Interactive exec write-ahead admission, crash, EOF, replacement boot and retirement | interactive-session (459 states); no-replay, EOF-retirement, failed-inspect and wrong-boot negative controls | `interactive_session.rs` atomic intent/exec records, exact stopped-boot proof, known-undispatched withdrawal, restore-before-busy-retirement and run-scoped cleanup-report tests, and lock-free observation/revalidation tests; Engine effect fences; authored owner start and SCM_RIGHTS handoff | One session, two boot identities, two possible starts. Atomic durable publication and exact identity/absence proofs are assumptions; real fsync/power loss, real owner/Engine handoff and VM loss need Mac acceptance; receipt lock timing and terminal restoration are regression-covered, not modeled. Validated exact exec exit permits retirement; it makes no descendant cleanup claim. | | Private interactive envelope, echo state and bytes still in flight on failure | interactive-envelope (15 states); restore-on-error negative control | HKEE2 launcher and host gate, Linux PTY seven-case matrix, bounded marker/ACK and coalesced-output tests | Two abstract private bytes, one handshake. PTY fixtures prove local termios ordering; Docker detach scanning, real guest wiring, same-user access and VM effects remain separate. Source/compiled Bun PTY and native-process fixtures cover lifetime/terminal boundaries, not model liveness. | @@ -83,6 +84,7 @@ liveness/reachability controls have their own documented assumptions. | Model | Positive distinct states | Configurations | | --- | ---: | ---: | +| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 426 | 4 | | [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 | | [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 | | [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 | diff --git a/packages/lifecycle-supervisor/Cargo.lock b/packages/lifecycle-supervisor/Cargo.lock new file mode 100644 index 000000000..665862df1 --- /dev/null +++ b/packages/lifecycle-supervisor/Cargo.lock @@ -0,0 +1,114 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "hack-lifecycle-supervisor" +version = "0.1.0" +dependencies = [ + "libc", + "serde", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/packages/lifecycle-supervisor/Cargo.toml b/packages/lifecycle-supervisor/Cargo.toml new file mode 100644 index 000000000..1fdb83721 --- /dev/null +++ b/packages/lifecycle-supervisor/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "hack-lifecycle-supervisor" +version = "0.1.0" +edition = "2024" +rust-version = "1.97.1" +publish = false + +[dependencies] +libc = "0.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +[profile.release] +lto = "thin" +strip = true diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md new file mode 100644 index 000000000..fce258562 --- /dev/null +++ b/packages/lifecycle-supervisor/README.md @@ -0,0 +1,95 @@ +# Standalone lifecycle supervisor + +This private crate builds `hack-lifecycle-supervisor`, an internal host-process +owner for Linux and macOS. It has no dependency on runtime-core, a terminal +multiplexer, Python, or a shell wrapper. Authored shell commands can still +explicitly select a shell in their argv. + +The first slice is exercised by a fake caller. It does not select a new CLI runner +or install the executable. See [the protocol, state model and delivery gates](../../docs/lifecycle-supervisor.md). + +Build and verify from the repository root with the pinned toolchain: + +```sh +cargo +1.97.1 fmt --manifest-path packages/lifecycle-supervisor/Cargo.toml --check +cargo +1.97.1 clippy --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor --all-targets -- -D warnings +cargo +1.97.1 test --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor +cargo +1.97.1 build --locked --release --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor +``` + +Tests create private temporary roots and explicit fake command environments. They +need no network, Docker, user registry or running Hack daemon. Each test owns its +processes and cleans them up. CI runs the same crate on Linux x86_64, macOS arm64, +and macOS x86_64. A cross-target `cargo check` checks compilation only; it does not +qualify platform signal, process identity, output or recovery behavior. + +`run` must be spawned as a fresh child without the caller making it a process-group +leader. The supervisor calls `setsid` itself and detaches its standard descriptors +after the framed ready receipt. `stop` and `status` are finite framed requests. +Frames, not pipe closure, delimit delivery. The per-command guard publishes kernel +identity before executing authored argv and retains ownership after the project +supervisor dies. A killed guard with surviving session members prevents a fresh +launch. Stop can recover that session after validating its recorded boot, leader +birth, UID and SID: TERM, five seconds of grace, then KILL if necessary, with +individual incarnation checks before delivery. A different boot or reused leader +proves the old session absent and never authorizes signaling the replacement. +A command that creates another session with `setsid` leaves this ownership boundary and is unsupported. + +The caller supplies a fresh ownership token and an existing private instance root. +No command/environment is persisted. Log output is unredacted and mode 0600. +There is no PTY: stdin is null and stdout/stderr are pipes. + +Status returns `liveProcessNames` and `unknownProcessNames` separately from the +persisted `processes[].phase`. A missing/zombie guard with live descendants is +unknown until recovery confirms absence; persisted `running` alone proves nothing. +These are additive version-1 receipt fields; they do not alter state files. + +Idle guards poll stdout/stderr and react to SIGCHLD/stop signals. A live direct +child proves work remains without a machine-wide census. After its exit, a 500 ms +fallback session census detects surviving descendants. EOF descriptors leave the poll set. A slow census still +finds descendants whose exit does not generate SIGCHLD for this guard. Stop uses +20 ms confirmation intervals and keeps its five-second TERM grace. Immediate +restart retries the instance lock for at most one second. Command-lock admission +waits at most ten seconds for a previous owner to finish releasing its lock. Stop +and launch wait up to sixteen seconds for `.stop.lock`; a stop whose wait expires +still answers with an `unknown` receipt (reason `stop-lock-wait-expired`). + +The project supervisor passes its launch/stop flock lease to each guard on startup +fd 2. The guard retains it through durable publication and receipt delivery, then +replaces stdio with `/dev/null`. Supervisor death therefore cannot let stop pass an +unpublished guard that might later spawn. Authored children get their own stderr +pipe and never inherit that lease. Startup closes inherited descriptors with Linux +`close_range` or enumeration of actual `/proc/self/fd` / `/dev/fd` entries; its cost +is independent of the descriptor limit. Log records are single unlocked `O_APPEND` writes, so no guard waits on a sibling to log. + +A usable process census is required before authored spawn. Linux preflight rejects +known `hidepid` mounts and requires a readable self identity in the census. An +unreadable process is skipped only after proving its UID is foreign; inaccessible +same-user identities fail closed. Darwin uses the short BSD identity query to +prove a protected process has a foreign UID; failure to establish that also refuses. +A post-launch census error retains the guard and command lock during cleanup. +This is not qualification of hidepid, PID namespaces, sandboxed/filtered process +views, or a process view that silently omits same-user members. Those views remain +unsupported. Same-user hostile receipt mutation and commands escaping their SID +remain outside the contract. Darwin lacks pidfd, so identity recheck and numeric +signal delivery still have a kernel observation/effect interval. + +The ignored `idle_guard_cpu_and_wakeup_measurement` fixture samples one guard and +its supervisor for three five-second idle intervals. Linux CPU ticks and voluntary +context switches are process accounting, not hardware wakeup telemetry. Run it +alone with `--ignored --exact idle_guard_cpu_and_wakeup_measurement --nocapture`. +The ignored `million_descriptor_limit_qualification` fixture requires a host hard +RLIMIT_NOFILE of at least 1048576; the ordinary raised-limit fixture uses the host's +available limit. A host refusing that raise does not qualify the million-fd case. + +Orphan recovery requires an exact command PID/birth or matching guard, including +a zombie. It captures verified members before TERM and retains those incarnations +through KILL even after the witness exits. Darwin birth timestamps are matched, +never ordered across processes. A reused numeric SID alone never authorizes a signal. +Terminal receipts prove the old guard completed cleanup; uncertain legacy or +newer-member-only sessions remain unknown without effects. See +[resource and qualification boundaries](../../docs/lifecycle-supervisor.md#resource-and-qualification-boundaries) +for offline rotation, Darwin different-UID limits and remaining crash-test gaps. +The ignored idle measurement has a native Darwin variant using proc_pid_rusage +over 60 seconds; its kernel idle/interrupt wakeups differ from Linux context switches. +Descriptor-limit and functional prepublication lease tests run on both platforms. diff --git a/packages/lifecycle-supervisor/src/main.rs b/packages/lifecycle-supervisor/src/main.rs new file mode 100644 index 000000000..8799f6ed9 --- /dev/null +++ b/packages/lifecycle-supervisor/src/main.rs @@ -0,0 +1,42 @@ +//! Standalone lifecycle owner; no terminal, shell wrapper, or environment file. +//! +//! Transition table (specified before implementation): +//! | From | Guard / action | To | Durable consequence | +//! | absent | command lock acquired, record guard identity | starting | identity before execution | +//! | starting | authored spawn succeeds | running | command PID, no environment | +//! | starting | authored spawn fails | exited | no authored process | +//! | running | verified owner requests TERM | stopping | retained ownership | +//! | stopping | no live authored session members | exited | retained terminal receipt | +//! | stopping | five-second grace expires | kill-pending | KILL authored session groups | +//! | kill-pending | no live authored session members | killed | confirmed terminal receipt | +//! | kill-pending | live members remain | kill-pending | retain guard and lock; stop unknown | +//! | starting/running | project owner dies | same | resident guard retains lock and identity | +//! | any live | duplicate command / lock busy | refused | no second execution | +//! | any | token or identity mismatch | refused | no signal | +//! | any | guard dead and authored session live | orphaned | duplicate refused; verified session stop permitted | +//! +//! A resident guard is the session leader in its own process group. Authored +//! commands receive a different group in that session. It publishes its kernel +//! identity before starting authored code, and remains present while any live +//! session member remains. A project-owner crash therefore leaves a verifiable +//! leader. Killing that guard separately makes health unknown; stop can recover +//! the recorded session only after boot, identity and member-incarnation checks. + +mod os; +mod protocol; +mod storage; +mod supervisor; + +fn main() { + if supervisor::main().is_err() { + // Inputs can contain credentials. Never include parser or OS error text. + eprintln!("Lifecycle supervisor refused or failed; values omitted."); + std::process::exit(1); + } +} + +type Result = std::result::Result; + +fn refused() -> std::io::Error { + std::io::Error::other("Lifecycle operation refused; values omitted") +} diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs new file mode 100644 index 000000000..2d1c3f26d --- /dev/null +++ b/packages/lifecycle-supervisor/src/os.rs @@ -0,0 +1,1142 @@ +use crate::{Result, refused}; +use std::fs; +use std::{ + io, + os::fd::AsRawFd, + sync::atomic::{AtomicBool, Ordering}, +}; + +pub static CHILD_CHANGED: AtomicBool = AtomicBool::new(false); + +extern "C" fn child_signal(_: i32) { + CHILD_CHANGED.store(true, Ordering::Relaxed); +} + +pub static STOP: AtomicBool = AtomicBool::new(false); + +extern "C" fn stop_signal(_: i32) { + STOP.store(true, Ordering::Relaxed); +} + +pub fn signals() -> Result<()> { + // SAFETY: sigaction is initialized, handler has C ABI and only stores an + // atomic flag; the kernel copies the action and retains no Rust pointer. + unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = child_signal as *const () as usize; + libc::sigemptyset(&mut action.sa_mask); + if libc::sigaction(libc::SIGCHLD, &action, std::ptr::null_mut()) != 0 { + return Err(io::Error::last_os_error()); + } + // A caller's blocked signal mask survives exec. Supervision must still + // receive stop requests and must be able to reap its direct children. + if libc::sigprocmask(libc::SIG_SETMASK, &action.sa_mask, std::ptr::null_mut()) != 0 { + return Err(io::Error::last_os_error()); + } + action.sa_sigaction = stop_signal as *const () as usize; + libc::sigemptyset(&mut action.sa_mask); + for signal in [libc::SIGTERM, libc::SIGINT, libc::SIGHUP] { + if libc::sigaction(signal, &action, std::ptr::null_mut()) != 0 { + return Err(io::Error::last_os_error()); + } + } + } + Ok(()) +} + +pub fn session() -> Result<()> { + // SAFETY: setsid has no pointer preconditions; this process is the caller's + // freshly spawned child, never an interactive terminal owner. + if unsafe { libc::setsid() } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub struct Identity { + pub pid: i32, + pub group: i32, + pub session: i32, + pub start: u64, + pub uid: u32, + pub zombie: bool, +} + +#[cfg(target_os = "linux")] +pub fn inspect(pid: i32) -> Result> { + let path = format!("/proc/{pid}/stat"); + let bytes = match fs::read(&path) { + Ok(value) => value, + Err(error) if absent_error(&error) => return Ok(None), + Err(error) => return Err(error), + }; + let tail = proc_stat_tail(&bytes)?; + let fields: Vec<_> = tail.split_whitespace().collect(); + let group = fields + .get(2) + .and_then(|v| v.parse().ok()) + .ok_or_else(refused)?; + let start = fields + .get(19) + .and_then(|v| v.parse().ok()) + .ok_or_else(refused)?; + let session = fields + .get(3) + .and_then(|v| v.parse().ok()) + .ok_or_else(refused)?; + let uid = match linux_uid(pid)? { + Some(uid) => uid, + None => return Ok(None), + }; + Ok(Some(Identity { + pid, + group, + session, + start, + uid, + zombie: matches!(fields.first(), Some(&"Z" | &"X")) && !linux_live_thread(pid)?, + })) +} + +/// A thread-group leader that called `pthread_exit` shows `Z` in +/// `/proc//stat` while its other threads keep running. Such a process is +/// alive: it still owns its PID and receives process-directed signals. Only a +/// group with no live thread left is a zombie. +#[cfg(target_os = "linux")] +fn linux_live_thread(pid: i32) -> Result { + let tasks = match fs::read_dir(format!("/proc/{pid}/task")) { + Ok(tasks) => tasks, + Err(error) if absent_error(&error) => return Ok(false), + Err(error) => return Err(error), + }; + for task in tasks { + let task = task?; + let stat = match fs::read(task.path().join("stat")) { + Ok(stat) => stat, + Err(error) if absent_error(&error) => continue, + Err(error) => return Err(error), + }; + if !matches!( + proc_stat_tail(&stat)?.split_whitespace().next(), + Some("Z" | "X") + ) { + return Ok(true); + } + } + Ok(false) +} + +#[cfg(target_os = "linux")] +fn linux_uid(pid: i32) -> Result> { + let status = match fs::read(format!("/proc/{pid}/status")) { + Ok(status) => status, + Err(error) if absent_error(&error) => return Ok(None), + Err(error) => return Err(error), + }; + let line = status + .split(|byte| *byte == b'\n') + .find_map(|line| line.strip_prefix(b"Uid:")) + .ok_or_else(refused)?; + let uid = std::str::from_utf8(line) + .map_err(|_| refused())? + .split_ascii_whitespace() + .nth(1) + .and_then(|uid| uid.parse().ok()) + .ok_or_else(refused)?; + Ok(Some(uid)) +} + +#[cfg(target_os = "linux")] +fn proc_stat_tail(bytes: &[u8]) -> Result<&str> { + // comm is arbitrary bytes and may itself contain parentheses. Only the + // fields following its final delimiter have an ASCII/UTF-8 contract. + let end = bytes + .windows(2) + .rposition(|v| v == b") ") + .ok_or_else(refused)?; + std::str::from_utf8(&bytes[end + 2..]).map_err(|_| refused()) +} + +#[cfg(target_os = "linux")] +pub fn census() -> Result> { + let mut result = Vec::new(); + for entry in fs::read_dir("/proc")? { + let entry = entry?; + if let Some(pid) = entry.file_name().to_str().and_then(|v| v.parse().ok()) { + match inspect(pid) { + Ok(Some(identity)) => result.push(identity), + Ok(None) => (), + Err(error) + if matches!(error.raw_os_error(), Some(libc::EPERM) | Some(libc::EACCES)) => + { + match linux_uid(pid)? { + // SAFETY: geteuid is an observation with no preconditions. + Some(uid) if uid != unsafe { libc::geteuid() } => (), + None => (), + _ => return Err(error), + } + } + Err(error) => return Err(error), + } + } + } + readable_self(&result)?; + Ok(result) +} + +#[cfg(target_os = "linux")] +pub fn boot() -> Result { + Ok(fs::read_to_string("/proc/sys/kernel/random/boot_id")? + .trim() + .into()) +} + +#[cfg(target_os = "macos")] +pub fn inspect(pid: i32) -> Result> { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: buffer is writable for its exact size, and no pointer is retained. + let size = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 1, // XNU requires nonzero arg to search unreaped zombie records. + info.as_mut_ptr().cast(), + std::mem::size_of::() as i32, + ) + }; + if size as usize != std::mem::size_of::() { + let error = io::Error::last_os_error(); + if size == 0 && error.raw_os_error() == Some(libc::ESRCH) { + return Ok(None); + } + if size == 0 && error.raw_os_error() == Some(libc::EPERM) { + return Err(error); + } + return Err(refused()); + } + // SAFETY: the kernel initialized exactly the expected struct size above. + let info = unsafe { info.assume_init() }; + // SAFETY: getsid only observes the named PID. ESRCH is an exit race. + let mut session = unsafe { libc::getsid(pid) }; + if session < 0 + && info.pbi_status == libc::SZOMB + && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + { + // Darwin getsid can reject zombies even when libproc returns birth/UID. + // Preserve its immutable PID/birth, but never invent live membership. + session = 0; + } + if session < 0 { + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(None); + } + return Err(error); + } + Ok(Some(Identity { + pid, + group: info.pbi_pgid as i32, + session, + start: info + .pbi_start_tvsec + .checked_mul(1_000_000) + .and_then(|v| v.checked_add(info.pbi_start_tvusec)) + .ok_or_else(refused)?, + uid: info.pbi_uid, + zombie: info.pbi_status == libc::SZOMB, + })) +} + +#[cfg(target_os = "macos")] +pub fn census() -> Result> { + const PROC_ALL_PIDS: u32 = 1; + // SAFETY: querying required bytes uses a null pointer and zero buffer size. + let size = unsafe { libc::proc_listpids(PROC_ALL_PIDS, 0, std::ptr::null_mut(), 0) }; + if size <= 0 { + return Err(refused()); + } + let mut pids = vec![0i32; size as usize / 4 + 1024]; + // SAFETY: vector is initialized and live for the byte size supplied. + let written = unsafe { + libc::proc_listpids( + PROC_ALL_PIDS, + 0, + pids.as_mut_ptr().cast(), + (pids.len() * 4) as i32, + ) + }; + if written <= 0 || written as usize >= pids.len() * 4 { + return Err(refused()); + } + let mut result = Vec::new(); + for pid in pids + .into_iter() + .take(written as usize / 4) + .filter(|pid| *pid > 0) + { + match inspect(pid) { + Ok(Some(identity)) => result.push(identity), + Ok(None) => (), + // Unrelated protected processes cannot belong to this user's + // authored session. Exact guard reads remain strict in inspect. + Err(error) if error.raw_os_error() == Some(libc::EPERM) && foreign_uid(pid)? => (), + Err(error) => return Err(error), + } + } + readable_self(&result)?; + Ok(result) +} + +#[cfg(target_os = "macos")] +fn foreign_uid(pid: i32) -> Result { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: bounded writable struct buffer, retained only for this call. + let size = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDT_SHORTBSDINFO, + 0, + info.as_mut_ptr().cast(), + std::mem::size_of::() as i32, + ) + }; + if size as usize != std::mem::size_of::() { + let error = io::Error::last_os_error(); + if absent_error(&error) { + return Ok(true); + } + return Err(refused()); + } + // SAFETY: exact struct size was initialized; geteuid only observes identity. + Ok(unsafe { info.assume_init().pbsi_uid != libc::geteuid() }) +} + +#[cfg(target_os = "macos")] +pub fn boot() -> Result { + let mut bytes = [0u8; 64]; + let mut length = bytes.len(); + // SAFETY: read-only sysctl with writable bounded buffer, no retained pointer. + if unsafe { + libc::sysctlbyname( + c"kern.bootsessionuuid".as_ptr(), + bytes.as_mut_ptr().cast(), + &mut length, + std::ptr::null_mut(), + 0, + ) + } != 0 + { + return Err(refused()); + } + if length == 0 || length > bytes.len() { + return Err(refused()); + } + Ok(std::str::from_utf8(&bytes[..length]) + .map_err(|_| refused())? + .trim_end_matches('\0') + .to_ascii_lowercase()) +} + +pub fn same(state: &crate::storage::State, identity: &Identity) -> Result<()> { + if state.guard_pid != identity.pid + || state.process_group_id != identity.group + || state.guard_pid != identity.session + || state.guard_start_time != identity.start + || state.uid != identity.uid + || state.boot_id != boot()? + || identity.zombie + { + return Err(refused()); + } + Ok(()) +} + +pub fn live_group(group: i32, excluding: Option) -> Result { + Ok(census()? + .iter() + .any(|p| p.session == group && !p.zombie && Some(p.pid) != excluding)) +} + +pub fn absent_error(error: &io::Error) -> bool { + error.kind() == io::ErrorKind::NotFound + || matches!(error.raw_os_error(), Some(libc::ESRCH) | Some(libc::ENOENT)) +} + +fn incarnation_matches(expected: &Identity, current: &Identity) -> bool { + expected.pid == current.pid + && expected.start == current.start + && expected.uid == current.uid + && expected.session == current.session +} + +/// PID/birth/UID pins the original zombie guard when Darwin getsid rejects it. +/// Live leaders additionally require the recorded session. +fn recorded_guard_matches(state: &crate::storage::State, p: &Identity) -> bool { + p.pid == state.guard_pid + && p.start == state.guard_start_time + && p.uid == state.uid + && (p.zombie || p.session == state.guard_pid) +} + +/// Return one census with session identity proved across that census. A matching +/// leader (including zombie) or exact recorded command birth is the witness. +/// No ordering comparison uses Darwin's adjustable wall-clock start timestamps. +fn recorded_members(state: &crate::storage::State) -> Result> { + for _ in 0..2 { + if matches!(state.phase.as_str(), "exited" | "killed" | "failed-start") + || state.boot_id != boot()? + { + return Ok(Vec::new()); + } + let leader = inspect(state.guard_pid)?; + if let Some(p) = &leader + && !recorded_guard_matches(state, p) + { + return Ok(Vec::new()); + } + let members: Vec<_> = census()? + .into_iter() + .filter(|p| p.session == state.guard_pid && !p.zombie) + .collect(); + if members.is_empty() { + return Ok(members); + } + let witness = leader + .as_ref() + .or_else(|| { + members.iter().find(|p| { + Some(p.pid) == state.command_pid + && Some(p.start) == state.command_start_time + && p.uid == state.uid + }) + }) + .ok_or_else(refused)?; + // Recheck after enumeration: the witness held the SID throughout the census. + if !inspect(witness.pid)?.as_ref().is_some_and(|p| { + if witness.pid == state.guard_pid { + recorded_guard_matches(state, p) + } else { + incarnation_matches(witness, p) + } + }) { + continue; + } + return Ok(members); + } + Err(refused()) +} + +pub fn recorded_live(state: &crate::storage::State, excluding_guard: bool) -> Result { + Ok(recorded_members(state)? + .iter() + .any(|p| !excluding_guard || p.pid != state.guard_pid)) +} + +pub enum Observation { + Absent, + Live, + Unknown, +} + +/// Health observation is stricter than a persisted running phase. A lost guard +/// with remaining session members is unknown, never healthy or absent. +pub fn observe_state(state: &crate::storage::State) -> Observation { + let observed = (|| -> Result { + if !recorded_live(state, true)? { + return Ok(Observation::Absent); + } + let leader = inspect(state.guard_pid)?; + if let Some(identity) = &leader + && !recorded_guard_matches(state, identity) + { + return Ok(Observation::Absent); + } + if !live_group(state.guard_pid, Some(state.guard_pid))? { + return Ok(Observation::Absent); + } + Ok( + if leader.is_some_and(|p| !p.zombie && p.group == state.guard_pid) { + Observation::Live + } else { + Observation::Unknown + }, + ) + })(); + observed.unwrap_or(Observation::Unknown) +} + +/// The guard is never a member of an authored process group. It retains the +/// command lock while KILL is confirmed, including uninterruptible processes. +pub fn session_signal(session: i32, signal: i32) -> Result<()> { + let members: Vec<_> = census()? + .into_iter() + .filter(|p| p.session == session && p.pid != session && !p.zombie) + .collect(); + let groups: std::collections::BTreeSet<_> = members + .iter() + .filter(|p| p.group != session) + .map(|p| p.group) + .collect(); + for member in members.iter().filter(|p| p.group == session) { + signal_member(member, signal)?; + } + for group in groups { + group_signal(group, signal)?; + } + Ok(()) +} + +/// Open a pidfd for `pid`, or `None` when no process with that thread-group +/// leader exists any more. +/// +/// With zero flags, `pidfd_open` reports a missing process as `ESRCH` on newer +/// kernels, but as `EINVAL` on Linux 6.8 when the PID survives only as a +/// process-group or session ID (its process exited while members of its group +/// or session still run), because no thread-group leader exists for it. Both +/// mean "no process to signal". Treating them as absent can only withhold a +/// signal, never send one: identity-checked liveness decides the outcome later. +#[cfg(target_os = "linux")] +fn pidfd_open(pid: i32) -> Result> { + use std::os::fd::{FromRawFd, OwnedFd}; + #[cfg(test)] + if let Some(errno) = tests::injected_pidfd_open_error(pid) { + return pidfd_open_failure(io::Error::from_raw_os_error(errno)); + } + // SAFETY: pidfd_open returns a newly owned CLOEXEC descriptor. + let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0u32) }; + if raw < 0 { + return pidfd_open_failure(io::Error::last_os_error()); + } + // SAFETY: a non-negative result is a descriptor this call now owns. + Ok(Some(unsafe { OwnedFd::from_raw_fd(raw as i32) })) +} + +#[cfg(target_os = "linux")] +fn pidfd_open_failure(error: io::Error) -> Result> { + if absent_error(&error) || error.raw_os_error() == Some(libc::EINVAL) { + return Ok(None); + } + Err(error) +} + +fn signal_member(expected: &Identity, signal: i32) -> Result<()> { + #[cfg(target_os = "linux")] + { + let Some(fd) = pidfd_open(expected.pid)? else { + return Ok(()); + }; + let Some(current) = inspect(expected.pid)? else { + return Ok(()); + }; + if current.zombie + || current.start != expected.start + || current.session != expected.session + || current.uid != expected.uid + { + return Ok(()); + } + // SAFETY: the pidfd pins the verified process; null siginfo is ordinary delivery. + if unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + fd.as_raw_fd(), + signal, + std::ptr::null::(), + 0u32, + ) + } < 0 + { + let error = io::Error::last_os_error(); + if !absent_error(&error) { + return Err(error); + } + } + Ok(()) + } + #[cfg(target_os = "macos")] + { + let Some(current) = inspect(expected.pid)? else { + return Ok(()); + }; + if current.zombie + || current.start != expected.start + || current.session != expected.session + || current.uid != expected.uid + { + return Ok(()); + } + // SAFETY: exact incarnation and session were rechecked immediately above. + if unsafe { libc::kill(expected.pid, signal) } != 0 { + let error = io::Error::last_os_error(); + if !absent_error(&error) { + return Err(error); + } + } + Ok(()) + } +} + +/// Authored commands start a separate group and restore inherited ignored +/// dispositions, which exec itself is not required to reset. +pub fn command_setup() -> Result<()> { + // SAFETY: async-signal-safe calls in the post-fork, single-threaded child. + unsafe { + if libc::setpgid(0, 0) != 0 { + return Err(io::Error::last_os_error()); + } + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = libc::SIG_DFL; + libc::sigemptyset(&mut action.sa_mask); + #[cfg(target_os = "linux")] + let highest = libc::SIGRTMAX(); + #[cfg(target_os = "macos")] + let highest = 31; + for signal in 1..=highest { + if signal != libc::SIGKILL + && signal != libc::SIGSTOP + && libc::sigaction(signal, &action, std::ptr::null_mut()) != 0 + { + #[cfg(target_os = "linux")] + if (signal == 32 || signal == 33) + && io::Error::last_os_error().raw_os_error() == Some(libc::EINVAL) + { + continue; + } + return Err(io::Error::last_os_error()); + } + } + let mut mask = std::mem::zeroed(); + libc::sigemptyset(&mut mask); + if libc::sigprocmask(libc::SIG_SETMASK, &mask, std::ptr::null_mut()) != 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) +} + +pub fn subreaper() -> Result<()> { + #[cfg(target_os = "linux")] + // SAFETY: process-local prctl enabling orphan adoption, integer options only. + if unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) } != 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +pub fn reap() -> Result> { + use std::os::unix::process::ExitStatusExt; + let mut reaped = Vec::new(); + loop { + let mut status = 0; + // SAFETY: waitpid writes one live integer; WNOHANG never blocks. + let pid = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) }; + if pid == 0 { + return Ok(reaped); + } + if pid < 0 { + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ECHILD) { + return Ok(reaped); + } + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + reaped.push((pid, std::process::ExitStatus::from_raw(status))); + } +} + +pub fn close_inherited() -> Result<()> { + #[cfg(target_os = "linux")] + { + // SAFETY: called before opening application descriptors. Only inherited + // descriptors >=3 are closed; the three protocol/std descriptors remain. + if unsafe { libc::syscall(libc::SYS_close_range, 3u32, u32::MAX, 0u32) } == 0 { + return Ok(()); + } + } + // Collect first, so the directory iterator's descriptor is closed before + // the sweep. Work is proportional to open descriptors, not RLIMIT_NOFILE. + #[cfg(target_os = "linux")] + let directory = "/proc/self/fd"; + #[cfg(target_os = "macos")] + let directory = "/dev/fd"; + let descriptors: Vec = fs::read_dir(directory)? + .map(|entry| { + entry? + .file_name() + .to_str() + .and_then(|v| v.parse().ok()) + .ok_or_else(refused) + }) + .collect::>()?; + for fd in descriptors.into_iter().filter(|fd| *fd >= 3) { + // SAFETY: startup owns no application descriptors >=3. The iterator's + // former descriptor may already be closed; that gap is harmless. + unsafe { + libc::close(fd); + } + } + Ok(()) +} + +pub fn signal_guard(state: &crate::storage::State) -> Result<()> { + #[cfg(target_os = "linux")] + { + use std::os::fd::{FromRawFd, OwnedFd}; + // SAFETY: pidfd_open pins this PID incarnation and returns a fresh CLOEXEC fd. + let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, state.guard_pid, 0u32) }; + if raw < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw as i32) }; + same(state, &inspect(state.guard_pid)?.ok_or_else(refused)?)?; + // SAFETY: pidfd is live; null siginfo requests ordinary signal delivery. + if unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + fd.as_raw_fd(), + libc::SIGTERM, + std::ptr::null::(), + 0u32, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + Ok(()) + } + #[cfg(target_os = "macos")] + { + same(state, &inspect(state.guard_pid)?.ok_or_else(refused)?)?; + // SAFETY: guarded by the exact kernel identity check above. Darwin has no pidfd. + if unsafe { libc::kill(state.guard_pid, libc::SIGTERM) } != 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } +} + +pub fn group_signal(group: i32, signal: i32) -> Result<()> { + if group <= 1 { + return Err(refused()); + } + // SAFETY: validated positive group number is negated for group delivery. + if unsafe { libc::kill(-group, signal) } != 0 { + let error = io::Error::last_os_error(); + group_signal_result(group, error, census)?; + } + Ok(()) +} + +/// EPERM may race the last live member becoming an unreaped zombie. Only a +/// fresh, successful group census proving no non-zombie member makes it benign. +fn group_signal_result( + group: i32, + error: io::Error, + census: impl FnOnce() -> Result>, +) -> Result<()> { + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(()); + } + if error.raw_os_error() == Some(libc::EPERM) + && !census()?.iter().any(|p| p.group == group && !p.zombie) + { + return Ok(()); + } + Err(error) +} + +/// Poll only pipe readiness between slow session checks. SIGCHLD and stop +/// signals interrupt poll; flags also cover signals arriving before this call. +pub fn wait_activity(fds: &[i32], timeout: std::time::Duration) -> Result<()> { + if STOP.load(Ordering::Relaxed) || CHILD_CHANGED.swap(false, Ordering::Relaxed) { + return Ok(()); + } + let mut polls: Vec<_> = fds + .iter() + .map(|fd| libc::pollfd { + fd: *fd, + events: libc::POLLIN, + revents: 0, + }) + .collect(); + // SAFETY: initialized pollfd slice is live for its supplied length. + if unsafe { + libc::poll( + polls.as_mut_ptr(), + polls.len() as _, + timeout.as_millis() as i32, + ) + } < 0 + { + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } + Ok(()) +} + +/// In-memory authority captured while a witness holds the old SID. It survives +/// witness exit but never grants authority over a later, unobserved incarnation. +/// A new stop invocation must establish its own witness again. +pub struct Recovery { + session: i32, + members: Vec, +} + +impl Recovery { + pub fn capture(state: &crate::storage::State) -> Result { + let members: Vec<_> = recorded_members(state)? + .into_iter() + .filter(|p| p.pid != state.guard_pid) + .collect(); + if members.iter().any(|p| p.uid != state.uid) { + return Err(refused()); + } + Ok(Self { + session: state.guard_pid, + members, + }) + } + + pub fn signal(&self, signal: i32) -> Result<()> { + for member in &self.members { + signal_member(member, signal)?; + } + Ok(()) + } + + pub fn live(&self) -> Result { + for member in &self.members { + if inspect(member.pid)?.is_some_and(|p| !p.zombie && incarnation_matches(member, &p)) { + return Ok(true); + } + } + // Unobserved descendants or an unrelated reused session cannot be + // signalled by this snapshot, and must not be claimed as stopped. + if live_group(self.session, Some(self.session))? { + return Err(refused()); + } + Ok(false) + } +} + +/// A successful census must contain our own readable identity. Known restricted +/// proc mounts are unsupported: hidden same-user members defeat absence proofs. +pub fn census_preflight() -> Result<()> { + #[cfg(target_os = "linux")] + { + let mounts = fs::read_to_string("/proc/self/mountinfo")?; + if restricted_mount(&mounts) { + return Err(refused()); + } + } + readable_self(&census()?) +} + +#[cfg(target_os = "linux")] +fn restricted_mount(mounts: &str) -> bool { + mounts.lines().any(|line| { + line.split_whitespace().any(|field| { + field + .split(',') + .any(|option| option.starts_with("hidepid=") && option != "hidepid=0") + }) + }) +} + +fn readable_self(members: &[Identity]) -> Result<()> { + let pid = std::process::id() as i32; + if !members.iter().any(|p| p.pid == pid && !p.zombie) { + return Err(refused()); + } + Ok(()) +} + +pub fn nonblocking(fd: &impl AsRawFd) -> Result<()> { + // SAFETY: fcntl borrows a live descriptor and supplies integer flags only. + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + if flags < 0 + || unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 + { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +pub fn timestamp() -> Result { + // SAFETY: time returns current seconds; gmtime_r writes one live tm struct, + // strftime writes at most the supplied buffer length and retains no pointer. + unsafe { + let now = libc::time(std::ptr::null_mut()); + let mut tm: libc::tm = std::mem::zeroed(); + if libc::gmtime_r(&now, &mut tm).is_null() { + return Err(refused()); + } + let mut bytes = [0u8; 32]; + let count = libc::strftime( + bytes.as_mut_ptr().cast(), + bytes.len(), + c"%Y-%m-%dT%H:%M:%SZ".as_ptr(), + &tm, + ); + if count == 0 { + return Err(refused()); + } + Ok(std::str::from_utf8(&bytes[..count]) + .map_err(|_| refused())? + .into()) + } +} + +#[cfg(test)] +mod tests { + #[cfg(target_os = "linux")] + thread_local! { + static PIDFD_OPEN_ERROR: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + } + + /// Test seam: the errno `pidfd_open` reports for one PID on this thread. + #[cfg(target_os = "linux")] + pub(super) fn injected_pidfd_open_error(pid: i32) -> Option { + PIDFD_OPEN_ERROR + .with(std::cell::Cell::get) + .and_then(|(target, errno)| (target == pid).then_some(errno)) + } + + /// A member process that is killed and reaped when dropped, so a failed + /// assertion or spawn cannot leak it. + #[cfg(target_os = "linux")] + struct Member(std::process::Child); + + #[cfg(target_os = "linux")] + impl Drop for Member { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + /// Two real members: the first reports `errno` from its pidfd open, the + /// second is an ordinary survivor. Returns the recovery KILL result and + /// whether each member was killed. + #[cfg(target_os = "linux")] + fn kill_with_failed_first_open(errno: i32) -> (std::io::Result<()>, bool, bool) { + use std::os::unix::process::ExitStatusExt; + let spawn = || { + Member( + std::process::Command::new("/bin/sleep") + .arg("60") + .spawn() + .expect("spawn member"), + ) + }; + let mut first = spawn(); + let mut second = spawn(); + let identity = |member: &Member| { + super::inspect(member.0.id() as i32) + .unwrap() + .expect("member identity") + }; + let recovery = super::Recovery { + session: 0, + members: vec![identity(&first), identity(&second)], + }; + PIDFD_OPEN_ERROR.with(|cell| cell.set(Some((first.0.id() as i32, errno)))); + let result = recovery.signal(libc::SIGKILL); + PIDFD_OPEN_ERROR.with(|cell| cell.set(None)); + let killed = |member: &mut Member| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + loop { + if let Some(status) = member.0.try_wait().unwrap() { + return status.signal() == Some(libc::SIGKILL); + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + }; + let second_killed = killed(&mut second); + let first_killed = first.0.try_wait().unwrap().is_some(); + (result, first_killed, second_killed) + } + + /// Linux 6.8 reports EINVAL for a PID that survives only as a group or + /// session ID. That member is absent: it gets no signal, and recovery still + /// delivers KILL to the remaining captured member. Pinned on every kernel. + #[cfg(target_os = "linux")] + #[test] + fn pidfd_open_einval_is_absent_and_recovery_still_kills_survivor() { + let (result, first_killed, second_killed) = kill_with_failed_first_open(libc::EINVAL); + assert!(result.is_ok(), "{result:?}"); + assert!(!first_killed, "absent member was signalled"); + assert!(second_killed, "surviving member did not get KILL"); + } + + #[cfg(target_os = "linux")] + #[test] + fn pidfd_open_esrch_is_absent_and_recovery_still_kills_survivor() { + let (result, first_killed, second_killed) = kill_with_failed_first_open(libc::ESRCH); + assert!(result.is_ok(), "{result:?}"); + assert!(!first_killed); + assert!(second_killed); + } + + /// Any other open failure stays an error and authorizes no signal. + #[cfg(target_os = "linux")] + #[test] + fn pidfd_open_other_errors_refuse_without_signalling() { + let (result, first_killed, second_killed) = kill_with_failed_first_open(libc::EMFILE); + assert_eq!(result.unwrap_err().raw_os_error(), Some(libc::EMFILE)); + assert!(!first_killed); + assert!(!second_killed); + } + + #[test] + fn zombie_guard_birth_survives_discarded_exit_metadata() { + let state = crate::storage::State { + version: 1, + name: "worker".into(), + owner_token: "synthetic".into(), + guard_pid: 44, + process_group_id: 44, + guard_start_time: 100, + boot_id: String::new(), + uid: 501, + command_pid: None, + command_start_time: None, + phase: "running".into(), + exit_code: None, + signal: None, + }; + let mut identity = super::Identity { + pid: 44, + group: 0, + session: 0, + start: 100, + uid: 501, + zombie: true, + }; + assert!(super::recorded_guard_matches(&state, &identity)); + identity.start += 1; + assert!(!super::recorded_guard_matches(&state, &identity)); + identity.start = 100; + identity.uid = 0; + assert!(!super::recorded_guard_matches(&state, &identity)); + identity.uid = state.uid; + identity.zombie = false; + assert!(!super::recorded_guard_matches(&state, &identity)); + identity.uid = state.uid; + identity.session = state.guard_pid; + assert!(super::recorded_guard_matches(&state, &identity)); + } + + #[test] + fn unusable_census_refuses_before_launch() { + assert!(super::readable_self(&[]).is_err()); + assert!(super::readable_self(&super::census().unwrap()).is_ok()); + #[cfg(target_os = "linux")] + { + assert!(super::restricted_mount( + "1 2 0:1 / /proc rw - proc proc rw,hidepid=2" + )); + assert!(super::restricted_mount( + "1 2 0:1 / /proc rw - proc proc rw,hidepid=invisible" + )); + assert!(!super::restricted_mount( + "1 2 0:1 / /proc rw - proc proc rw,hidepid=0" + )); + } + } + + #[test] + fn eperm_requires_fresh_zombie_only_group_proof() { + use super::*; + let member = Identity { + pid: 123, + group: 123, + session: 123, + start: 1, + uid: 0, + zombie: true, + }; + let denied = || io::Error::from_raw_os_error(libc::EPERM); + assert!(group_signal_result(123, denied(), || Ok(vec![member.clone()])).is_ok()); + let mut live = member.clone(); + live.zombie = false; + assert!(group_signal_result(123, denied(), || Ok(vec![member.clone(), live])).is_err()); + assert!(group_signal_result(123, denied(), || Err(refused())).is_err()); + assert!( + group_signal_result(123, io::Error::from_raw_os_error(libc::EIO), || Ok(vec![])) + .is_err() + ); + } + + #[test] + fn real_unreaped_group_is_not_live() { + use super::*; + // SAFETY: child uses only async-signal-safe calls and _exit, never Rust. + let pid = unsafe { libc::fork() }; + assert!(pid >= 0); + if pid == 0 { + unsafe { + libc::setsid(); + libc::_exit(0); + } + } + // Darwin getsid may return ESRCH for zombies, so inspect intentionally + // reports absence. Observe exit without reaping through the wait API + // instead of requiring a live-process identity for an exited child. + let mut event = std::mem::MaybeUninit::::zeroed(); + // SAFETY: this is our direct child, event is a writable kernel buffer, + // and WNOWAIT deliberately retains the child for the assertions below. + assert_eq!( + unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + event.as_mut_ptr(), + libc::WEXITED | libc::WNOWAIT, + ) + }, + 0 + ); + assert!(!live_group(pid, None).unwrap()); + assert!( + group_signal_result(pid, io::Error::from_raw_os_error(libc::EPERM), census).is_ok() + ); + group_signal(pid, libc::SIGTERM).unwrap(); + let mut status = 0; + // SAFETY: reap only our own still-unreaped child, with a valid status pointer. + assert_eq!(unsafe { libc::waitpid(pid, &mut status, 0) }, pid); + } + + #[cfg(target_os = "linux")] + #[test] + fn non_utf8_process_name_does_not_invalidate_identity_fields() { + assert_eq!( + super::proc_stat_tail(b"42 (name\xff) nested) S 1 2 3").unwrap(), + "S 1 2 3" + ); + assert!(super::proc_stat_tail(b"42 (name) S \xff").is_err()); + } + #[test] + fn proc_exit_during_open_or_read_is_absent_not_census_failure() { + for errno in [libc::ENOENT, libc::ESRCH] { + assert!(super::absent_error(&std::io::Error::from_raw_os_error( + errno + ))); + } + for errno in [libc::EPERM, libc::EIO] { + assert!(!super::absent_error(&std::io::Error::from_raw_os_error( + errno + ))); + } + } +} diff --git a/packages/lifecycle-supervisor/src/protocol.rs b/packages/lifecycle-supervisor/src/protocol.rs new file mode 100644 index 000000000..f12c63fbd --- /dev/null +++ b/packages/lifecycle-supervisor/src/protocol.rs @@ -0,0 +1,250 @@ +use crate::{Result, refused}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use std::{ + collections::BTreeMap, + io::{Read, Write}, + os::fd::AsRawFd, + path::PathBuf, + time::{Duration, Instant}, +}; + +pub const LIMIT: usize = 1024 * 1024; +pub const DEADLINE: Duration = Duration::from_secs(10); + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Launch { + pub version: u32, + pub root: PathBuf, + pub owner_token: String, + pub processes: Vec, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Process { + pub name: String, + pub argv: Vec, + pub cwd: PathBuf, + pub env: BTreeMap, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Control { + pub version: u32, + pub root: PathBuf, + pub owner_token: String, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GuardLaunch { + pub version: u32, + pub root: PathBuf, + pub owner_token: String, + pub process: Process, +} + +pub fn valid_name(name: &str) -> bool { + !name.is_empty() + && name.as_bytes()[0].is_ascii_alphanumeric() + && name.len() <= 96 + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b)) + && name != "." + && name != ".." +} + +pub fn validate_control(version: u32, root: &std::path::Path, token: &str) -> Result<()> { + if version != 1 + || !root.is_absolute() + || token.is_empty() + || token.len() > 256 + || token.contains('\0') + { + return Err(refused()); + } + Ok(()) +} + +impl Process { + pub fn validate(&self) -> Result<()> { + if !valid_name(&self.name) + || !self.cwd.is_absolute() + || self.argv.is_empty() + || self.argv[0].is_empty() + || self.argv.iter().any(|v| v.contains('\0')) + || self.env.iter().any(|(key, value)| { + key.is_empty() || key.contains(['\0', '=']) || value.contains('\0') + }) + { + return Err(refused()); + } + Ok(()) + } +} + +/// Length and deadline, never descriptor closure, delimit a handoff. Polling +/// handles Darwin's brief inheritable-descriptor window without waiting on EOF. +pub fn read(input: &mut (impl Read + AsRawFd)) -> Result { + let deadline = Instant::now() + DEADLINE; + let mut header = [0; 4]; + exact(input, &mut header, deadline)?; + let length = u32::from_be_bytes(header) as usize; + if length == 0 || length > LIMIT { + return Err(refused()); + } + let mut bytes = vec![0; length]; + exact(input, &mut bytes, deadline)?; + let parsed = serde_json::from_slice(&bytes).map_err(|_| refused()); + bytes.fill(0); + parsed +} + +fn exact(input: &mut (impl Read + AsRawFd), bytes: &mut [u8], deadline: Instant) -> Result<()> { + let mut offset = 0; + while offset < bytes.len() { + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or_else(refused)?; + let mut fd = libc::pollfd { + fd: input.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: poll borrows one initialized descriptor record for this call. + let result = unsafe { + libc::poll( + &mut fd, + 1, + remaining.as_millis().min(i32::MAX as u128) as i32, + ) + }; + if result < 0 && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted { + continue; + } + if result <= 0 { + return Err(refused()); + } + // Read exactly the polled kernel descriptor. Stdin's high-level Read + // buffers ahead: polling after a four-byte header would then wait for + // data already held inside its user-space buffer. + // SAFETY: the destination slice is writable for its supplied length; + // read borrows a live descriptor and retains no pointer. + let count = unsafe { + libc::read( + input.as_raw_fd(), + bytes[offset..].as_mut_ptr().cast(), + bytes.len() - offset, + ) + }; + let result = if count < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(count as usize) + }; + match result { + Ok(0) => return Err(refused()), + Ok(count) => offset += count, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + } + } + Ok(()) +} + +pub fn write(output: &mut (impl Write + AsRawFd), value: &impl Serialize) -> Result<()> { + let mut bytes = serde_json::to_vec(value).map_err(|_| refused())?; + if bytes.is_empty() || bytes.len() > LIMIT { + return Err(refused()); + } + let fd = output.as_raw_fd(); + // SAFETY: fcntl borrows this live, exclusively used protocol descriptor. + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if flags < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: only integer descriptor flags are changed for bounded writes. + if unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 { + return Err(std::io::Error::last_os_error()); + } + let deadline = Instant::now() + DEADLINE; + let result = send(fd, &(bytes.len() as u32).to_be_bytes(), deadline) + .and_then(|()| send(fd, &bytes, deadline)); + // SAFETY: restore the original flags before returning the borrowed fd. + let restore = unsafe { libc::fcntl(fd, libc::F_SETFL, flags) }; + bytes.fill(0); + result?; + if restore < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +fn send(fd: i32, bytes: &[u8], deadline: Instant) -> Result<()> { + let mut offset = 0; + while offset < bytes.len() { + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or_else(refused)?; + let mut record = libc::pollfd { + fd, + events: libc::POLLOUT, + revents: 0, + }; + // SAFETY: poll borrows one initialized descriptor record. + let ready = unsafe { + libc::poll( + &mut record, + 1, + remaining.as_millis().min(i32::MAX as u128) as i32, + ) + }; + if ready < 0 && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted { + continue; + } + if ready <= 0 { + return Err(refused()); + } + // SAFETY: bytes remain live for the supplied length; fd is nonblocking. + let count = + unsafe { libc::write(fd, bytes[offset..].as_ptr().cast(), bytes.len() - offset) }; + if count < 0 { + let error = std::io::Error::last_os_error(); + if matches!( + error.kind(), + std::io::ErrorKind::Interrupted | std::io::ErrorKind::WouldBlock + ) { + continue; + } + return Err(error); + } + if count == 0 { + return Err(refused()); + } + offset += count as usize; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn inherited_environment_uses_exec_constraints_not_shell_identifiers() { + let mut process: super::Process = serde_json::from_value(serde_json::json!({ + "name": "worker", "argv": ["/usr/bin/true"], "cwd": "/", + "env": {"BASH_FUNC_fixture%%": "() { :; }", "1.name": "synthetic"} + })) + .unwrap(); + assert!(process.validate().is_ok()); + for key in ["", "bad=name", "bad\0name"] { + process.env.insert(key.into(), "synthetic".into()); + assert!(process.validate().is_err()); + process.env.remove(key); + } + process.env.insert("VALID".into(), "bad\0value".into()); + assert!(process.validate().is_err()); + } +} diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs new file mode 100644 index 000000000..3108dfa1b --- /dev/null +++ b/packages/lifecycle-supervisor/src/storage.rs @@ -0,0 +1,329 @@ +use crate::{Result, refused}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{File, OpenOptions}, + io::{Read, Write}, + os::{ + fd::{AsRawFd, FromRawFd, OwnedFd}, + unix::fs::{MetadataExt, OpenOptionsExt}, + }, + path::{Path, PathBuf}, +}; + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct State { + pub version: u32, + pub name: String, + pub owner_token: String, + pub guard_pid: i32, + pub process_group_id: i32, + pub guard_start_time: u64, + pub boot_id: String, + pub uid: u32, + pub command_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command_start_time: Option, + pub phase: String, + #[serde(default)] + pub exit_code: Option, + #[serde(default)] + pub signal: Option, +} + +/// Operations use an opened O_NOFOLLOW directory and relative openat/renameat; +/// changing a pathname after admission cannot redirect state/log writes. +pub struct Root { + directory: File, + pub path: PathBuf, +} + +impl Root { + fn retirement_key(token: &str) -> String { + // Stable non-authentication index. A collision can only refuse launch. + let hash = token.bytes().fold(0xcbf29ce484222325_u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3) + }); + format!(".retired-{hash:016x}") + } + + pub fn retire(&self, token: &str) -> Result<()> { + self.file(&Self::retirement_key(token), libc::O_WRONLY | libc::O_CREAT)? + .sync_all()?; + self.directory.sync_all() + } + + pub fn assert_not_retired(&self, token: &str) -> Result<()> { + match self.file(&Self::retirement_key(token), libc::O_RDONLY) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + _ => Err(refused()), + } + } + + pub fn open(path: &Path) -> Result { + let directory = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?; + let metadata = directory.metadata()?; + // SAFETY: geteuid has no preconditions. + if !metadata.is_dir() + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.mode() & 0o777 != 0o700 + { + return Err(refused()); + } + Ok(Self { + directory, + path: path.into(), + }) + } + + pub fn file(&self, name: &str, flags: i32) -> Result { + let name = std::ffi::CString::new(name).map_err(|_| refused())?; + // A receipt is replaced atomically (temporary file, then rename). An open + // that races that rename can return the replaced inode, which is already + // unlinked (nlink 0). Reopen the name to read its current inode; any + // other link count is still refused. The bound only stops a livelock. + for _ in 0..64 { + // SAFETY: directory descriptor is live; name is NUL-terminated; + // returned descriptor is newly owned and immediately wrapped in RAII. + let fd = unsafe { + libc::openat( + self.directory.as_raw_fd(), + name.as_ptr(), + flags | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: fd was just returned by openat and has no other owner. + let file = File::from(unsafe { OwnedFd::from_raw_fd(fd) }); + let metadata = file.metadata()?; + if metadata.nlink() == 0 { + continue; + } + // SAFETY: geteuid has no preconditions. + if !metadata.is_file() + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.nlink() != 1 + || metadata.mode() & 0o777 != 0o600 + { + return Err(refused()); + } + return Ok(file); + } + Err(refused()) + } + + pub fn lock(&self, name: &str) -> Result { + let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?; + // SAFETY: flock operates only on this live descriptor. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 { + return Err(refused()); + } + Ok(file) + } + + pub fn lock_wait(&self, name: &str, wait: std::time::Duration) -> Result { + let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?; + let deadline = std::time::Instant::now() + wait; + loop { + // SAFETY: flock only borrows the live owned descriptor. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + return Ok(file); + } + let error = std::io::Error::last_os_error(); + if !matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) { + return Err(error); + } + if std::time::Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "Lifecycle lock wait expired", + )); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + } + + pub fn save(&self, state: &State) -> Result<()> { + let final_name = format!("{}.json", state.name); + // Refuse unsafe pre-existing targets before atomically replacing an owned + // regular receipt. Never truncate through a link. + match self.file(&final_name, libc::O_RDONLY) { + Ok(_) => (), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return Err(error), + } + static SERIAL: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let mut attempts = 0; + let (temporary_name, mut temporary) = loop { + attempts += 1; + if attempts > 16 { + return Err(refused()); + } + let clock = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| refused())? + .as_nanos(); + let serial = SERIAL.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let name = format!( + ".{}.{}.{clock}.{serial}.tmp", + state.name, + std::process::id() + ); + match self.file(&name, libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL) { + Ok(file) => break (name, file), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + }; + let bytes = serde_json::to_vec(state).map_err(|_| refused())?; + temporary.write_all(&bytes)?; + temporary.sync_all()?; + let temporary_name = std::ffi::CString::new(temporary_name).map_err(|_| refused())?; + let final_name = std::ffi::CString::new(final_name).map_err(|_| refused())?; + // SAFETY: both names are NUL terminated and relative to this live root. + if unsafe { + libc::renameat( + self.directory.as_raw_fd(), + temporary_name.as_ptr(), + self.directory.as_raw_fd(), + final_name.as_ptr(), + ) + } != 0 + { + return Err(std::io::Error::last_os_error()); + } + self.directory.sync_all() + } + + pub fn load(&self, name: &str) -> Result> { + let mut file = match self.file(&format!("{name}.json"), libc::O_RDONLY) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if file.metadata()?.len() > 8192 { + return Err(refused()); + } + let mut bytes = Vec::new(); + Read::by_ref(&mut file).take(8193).read_to_end(&mut bytes)?; + let state: State = serde_json::from_slice(&bytes).map_err(|_| refused())?; + if state.version != 1 + // Case-insensitive filesystems may return a retained receipt under + // its previous spelling after a case-only command rename. + || !state.name.eq_ignore_ascii_case(name) + || state.guard_pid <= 1 + || state.process_group_id != state.guard_pid + || state.guard_start_time == 0 + { + return Err(refused()); + } + Ok(Some(state)) + } + + pub fn states(&self) -> Result> { + let mut states = Vec::new(); + // Enumeration is followed by descriptor-relative checked reads. Root path + // replacement can at most change the candidate names, never read location. + for entry in std::fs::read_dir(&self.path)? { + let entry = entry?; + if let Some(name) = entry + .file_name() + .to_str() + .and_then(|v| v.strip_suffix(".json")) + { + if !crate::protocol::valid_name(name) { + return Err(refused()); + } + if let Some(state) = self.load(name)? { + states.push(state); + } + } + } + Ok(states) + } +} + +#[cfg(test)] +mod tests { + use super::{Root, State}; + use std::os::unix::fs::DirBuilderExt; + + fn state(phase: &str) -> State { + State { + version: 1, + name: "worker".into(), + owner_token: "synthetic".into(), + guard_pid: 44, + process_group_id: 44, + guard_start_time: 100, + boot_id: String::new(), + uid: 501, + command_pid: None, + command_start_time: None, + phase: phase.into(), + exit_code: None, + signal: None, + } + } + + /// A guard atomically replaces its receipt while a stop or status lists + /// receipts. A reader that opened the old inode just before the rename sees + /// it unlinked; that must read as the current receipt, never as a refusal. + #[test] + fn receipt_reads_survive_concurrent_atomic_replacement() { + let base = std::env::temp_dir().join(format!( + "storage-replace-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&base) + .unwrap(); + let root = Root::open(&base).unwrap(); + root.save(&state("running")).unwrap(); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let writer = { + let base = base.clone(); + let stop = stop.clone(); + std::thread::spawn(move || { + let root = Root::open(&base).unwrap(); + let mut flip = false; + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + flip = !flip; + root.save(&state(if flip { "stopping" } else { "running" })) + .unwrap(); + } + }) + }; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + let mut reads = 0u64; + let mut failure = None; + while std::time::Instant::now() < deadline { + match root.states() { + Ok(states) => assert_eq!(states.len(), 1), + Err(error) => { + failure = Some(error); + break; + } + } + reads += 1; + } + stop.store(true, std::sync::atomic::Ordering::Relaxed); + writer.join().unwrap(); + let _ = std::fs::remove_dir_all(&base); + assert!(failure.is_none(), "read {reads} failed: {failure:?}"); + } +} diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs new file mode 100644 index 000000000..075979dac --- /dev/null +++ b/packages/lifecycle-supervisor/src/supervisor.rs @@ -0,0 +1,805 @@ +use crate::{ + Result, os, + protocol::{self, Control, GuardLaunch, Launch}, + refused, + storage::{Root, State}, +}; +use serde::Serialize; +use std::{ + fs::File, + io::{Read, Write}, + os::{ + fd::AsRawFd, + unix::process::{CommandExt, ExitStatusExt}, + }, + process::{Child, Command, Stdio}, + sync::atomic::Ordering, + thread, + time::{Duration, Instant}, +}; + +const GRACE: Duration = Duration::from_secs(5); +const IDLE_TICK: Duration = Duration::from_millis(500); +const TICK: Duration = Duration::from_millis(20); +/// Time after KILL for a stop to confirm absence. +const CONFIRM_BOUND: Duration = Duration::from_secs(2); +/// A stop's timers allow two TERM-to-KILL bounds: the guard path, then orphan +/// recovery after a late guard loss. The wait outlasts those timers, so a +/// waiter normally gets its own outcome. Admission, census and signalling run +/// outside the timers, so it is not a hard bound: an expired wait returns an +/// `unknown` receipt with reason `stop-lock-wait-expired` and acts on nothing. +const STOP_LOCK_WAIT: Duration = + Duration::from_secs(2 * (GRACE.as_secs() + CONFIRM_BOUND.as_secs()) + 2); +/// Bounded wait for a previous command owner to release its lock. +const COMMAND_LOCK_WAIT: Duration = Duration::from_secs(10); +const LINE_LIMIT: usize = 64 * 1024; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Receipt { + version: u32, + supervisor_pid: u32, + outcome: &'static str, + processes: Vec, + live_process_names: Vec, + unknown_process_names: Vec, + /// Why a stop or status request did not complete; absent on success. + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option<&'static str>, +} + +impl Receipt { + fn new(supervisor_pid: u32, outcome: &'static str, processes: Vec) -> Self { + let mut live_process_names = Vec::new(); + let mut unknown_process_names = Vec::new(); + for state in &processes { + match os::observe_state(state) { + os::Observation::Live => live_process_names.push(state.name.clone()), + os::Observation::Absent => (), + os::Observation::Unknown => unknown_process_names.push(state.name.clone()), + } + } + Self { + version: 1, + supervisor_pid, + outcome, + processes, + live_process_names, + unknown_process_names, + reason: None, + } + } +} + +/// Answer one `stop` or `status` request. Every request that reaches this +/// point ends with exactly one framed receipt, including refusals and expired +/// waits, so a caller can tell an incomplete request from a crash. A failure +/// keeps the nonzero exit and reports outcome `refused` (no request was +/// admitted) or `unknown`, with a `reason` code. +fn control(action: &str) -> Result<()> { + let mut stdout = std::io::stdout(); + let admitted = protocol::read::(&mut std::io::stdin()).and_then(|request| { + protocol::validate_control(request.version, &request.root, &request.owner_token)?; + let root = Root::open(&request.root)?; + Ok((request, root)) + }); + let (request, root) = match admitted { + Ok(admitted) => admitted, + Err(error) => { + let mut receipt = Receipt::new(0, "refused", Vec::new()); + receipt.reason = Some("invalid-request"); + protocol::write(&mut stdout, &receipt)?; + return Err(error); + } + }; + let mut reason = "io-error"; + let mut result = control_effects(action, &request, &root, &mut reason); + // Receipts are atomically replaced files, so this read is safe without the + // lock, also after a lock wait expired. A failed read never reports + // success: it keeps an earlier error, or becomes `io-error` itself. + let processes = match root.states() { + Ok(states) => states + .into_iter() + .filter(|state| state.owner_token == request.owner_token) + .collect(), + Err(error) => { + if result.is_ok() { + reason = "receipt-read-failed"; + result = Err(error); + } + Vec::new() + } + }; + let outcome = match (&result, action) { + (Err(_), _) => "unknown", + (Ok(()), "stop") => "stopped", + (Ok(()), _) => "observed", + }; + let mut receipt = Receipt::new(0, outcome, processes); + if result.is_err() { + receipt.reason = Some(reason); + } + protocol::write(&mut stdout, &receipt)?; + result +} + +fn control_effects( + action: &str, + request: &Control, + root: &Root, + reason: &mut &'static str, +) -> Result<()> { + let _stop_lock = if action == "stop" { + match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) { + Ok(lock) => Some(lock), + Err(error) => { + *reason = if error.kind() == std::io::ErrorKind::TimedOut { + // Another stop or a launch still holds the lock; act on nothing. + "stop-lock-wait-expired" + } else { + "stop-lock-failed" + }; + return Err(error); + } + } + } else { + None + }; + let mut states = Vec::new(); + *reason = "receipt-read-failed"; + for state in root.states()? { + *reason = "io-error"; + if state.owner_token == request.owner_token { + states.push(state); + } else if os::recorded_live(&state, false)? { + // Old terminal receipts survive renames and new generations. + // Guard-written terminal phases prove completed retirement; + // non-terminal receipts require native identity evidence. + *reason = "foreign-live-process"; + return Err(refused()); + } + } + *reason = "io-error"; + if action == "stop" { + *reason = "retire-failed"; + root.retire(&request.owner_token)?; + *reason = "stop-incomplete"; + return stop(&states); + } + if root.assert_not_retired(&request.owner_token).is_err() + && states + .iter() + .any(|state| !matches!(os::observe_state(state), os::Observation::Absent)) + { + *reason = "retired-owner-live"; + return Err(refused()); + } + Ok(()) +} + +pub fn main() -> Result<()> { + os::close_inherited()?; + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.len() != 1 { + return Err(refused()); + } + match args[0].to_str() { + Some("run") => run(protocol::read(&mut std::io::stdin())?), + Some("guard") => guard(protocol::read(&mut std::io::stdin())?), + Some(action @ ("stop" | "status")) => control(action), + _ => Err(refused()), + } +} + +fn run(request: Launch) -> Result<()> { + protocol::validate_control(request.version, &request.root, &request.owner_token)?; + if request.processes.is_empty() || request.processes.len() > 128 { + return Err(refused()); + } + let mut names = std::collections::BTreeSet::new(); + for process in &request.processes { + process.validate()?; + if !names.insert(process.name.to_ascii_lowercase()) { + return Err(refused()); + } + } + let root = Root::open(&request.root)?; + let _instance_lock = instance_lock(&root)?; + os::census_preflight()?; + let launch_lock = root.lock_wait(".stop.lock", STOP_LOCK_WAIT)?; + root.assert_not_retired(&request.owner_token)?; + // Validate every old claim before starting any new command. A guard survives + // project-owner death; its command lock also closes concurrent restart races. + for state in root.states()? { + if os::recorded_live(&state, false)? { + return Err(refused()); + } + } + os::session()?; + os::signals()?; + let mut children = Vec::new(); + let outcome = (|| { + let mut states = Vec::new(); + for process in request.processes { + if os::STOP.load(Ordering::Relaxed) { + return Err(refused()); + } + #[cfg(target_os = "linux")] + let executable = std::path::PathBuf::from("/proc/self/exe"); + #[cfg(not(target_os = "linux"))] + let executable = std::env::current_exe()?; + let mut command = Command::new(executable); + command + .arg("guard") + .env_clear() + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + // Keep stop admission leased even if this supervisor dies + // before the guard publishes its durable identity. fd 2 is + // reserved until guard receipt delivery/detach (or exit). + .stderr(Stdio::from(launch_lock.try_clone()?)); + // SAFETY: setsid is async-signal-safe, and this closure allocates no + // application state. It runs in the freshly forked child only. + unsafe { + command.pre_exec(os::session); + } + let child = command.spawn()?; + children.push(child); + let child = children.last_mut().ok_or_else(refused)?; + let plan = GuardLaunch { + version: 1, + root: request.root.clone(), + owner_token: request.owner_token.clone(), + process, + }; + protocol::write(&mut child.stdin.take().ok_or_else(refused)?, &plan)?; + let state: State = protocol::read(&mut child.stdout.take().ok_or_else(refused)?)?; + if state.guard_pid != child.id() as i32 || state.owner_token != request.owner_token { + return Err(refused()); + } + if state.phase == "failed-start" { + return Err(refused()); + } + states.push(state); + } + protocol::write( + &mut std::io::stdout(), + &Receipt::new(std::process::id(), "ready", states), + )?; + drop(launch_lock); + detach_stdio()?; + loop { + if os::STOP.load(Ordering::Relaxed) { + match root.lock_wait(".stop.lock", STOP_LOCK_WAIT) { + Ok(_lock) => { + stop(&root.states()?)?; + } + Err(error) if error.kind() == std::io::ErrorKind::TimedOut => (), + Err(error) => return Err(error), + } + } + let mut running = false; + for child in &mut children { + if child.try_wait()?.is_none() { + running = true; + } + } + if !running { + return Ok(()); + } + os::wait_activity(&[], IDLE_TICK)?; + } + })(); + if outcome.is_err() { + // Rollback only this start's unreaped children. Never infer authority from + // another root entry while handling a launch failure. + rollback(&mut children); + } + outcome +} + +fn instance_lock(root: &Root) -> Result { + // Preserve the uncontended fast path; a bounded retry below distinguishes + // contention from persistent file/OS failures with the actual errno. + if let Ok(file) = root.lock(".instance.lock") { + return Ok(file); + } + let file = root.file(".instance.lock", libc::O_RDWR | libc::O_CREAT)?; + let deadline = Instant::now() + Duration::from_secs(1); + loop { + // SAFETY: file remains owned throughout this bounded retry. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + return Ok(file); + } + let error = std::io::Error::last_os_error(); + if !matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) || Instant::now() >= deadline + { + return Err(error); + } + thread::sleep(TICK); + } +} + +fn rollback(children: &mut [Child]) { + for child in children.iter_mut() { + if child.try_wait().ok().flatten().is_some() { + continue; + } + let pid = child.id() as i32; + // An unreaped direct Child pins the PID; only the guard receives TERM. + // SAFETY: the child has not been reaped, so this PID cannot be reused. + unsafe { + libc::kill(pid, libc::SIGTERM); + } + } + let deadline = Instant::now() + GRACE + CONFIRM_BOUND; + while Instant::now() < deadline { + if children + .iter_mut() + .all(|child| child.try_wait().ok().flatten().is_some()) + { + break; + } + thread::sleep(TICK); + } + // A guard still alive after KILL confirmation remains the owner of unknown + // or uninterruptible work. Do not kill it and discard its lock. +} + +fn guard(request: GuardLaunch) -> Result<()> { + protocol::validate_control(request.version, &request.root, &request.owner_token)?; + request.process.validate()?; + let root = Root::open(&request.root)?; + let _command_lock = + root.lock_wait(&format!("{}.lock", request.process.name), COMMAND_LOCK_WAIT)?; + if let Some(previous) = root.load(&request.process.name)? + && os::recorded_live(&previous, false)? + { + return Err(refused()); + } + let pid = std::process::id() as i32; + let identity = os::inspect(pid)?.ok_or_else(refused)?; + if identity.group != pid { + return Err(refused()); + } + os::signals()?; + os::subreaper()?; + os::census_preflight()?; + let mut state = State { + version: 1, + name: request.process.name.clone(), + owner_token: request.owner_token, + guard_pid: pid, + process_group_id: pid, + guard_start_time: identity.start, + boot_id: os::boot()?, + uid: identity.uid, + command_pid: None, + command_start_time: None, + phase: "starting".into(), + exit_code: None, + signal: None, + }; + let mut log = root.file( + "lifecycle.log", + libc::O_WRONLY | libc::O_APPEND | libc::O_CREAT, + )?; + root.save(&state)?; + let outcome = guard_command(&request.process, &root, &mut state, &mut log); + if outcome.is_err() && state.command_pid.is_some() { + // Keep ownership while confirming internal-error cleanup. If even the + // census fails, retain the guard and lock rather than silently orphan. + let _ = os::session_signal(pid, libc::SIGTERM); + thread::sleep(GRACE); + loop { + let _ = os::session_signal(pid, libc::SIGKILL); + let _ = os::reap(); + if matches!(os::live_group(pid, Some(pid)), Ok(false)) { + break; + } + thread::sleep(TICK); + } + } + outcome +} + +fn guard_command( + process: &protocol::Process, + root: &Root, + state: &mut State, + log: &mut File, +) -> Result<()> { + let mut command = Command::new(&process.argv[0]); + command + .args(&process.argv[1..]) + .current_dir(&process.cwd) + .env_clear() + .envs(&process.env) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + // SAFETY: command_setup uses only async-signal-safe operations after fork. + unsafe { + command.pre_exec(os::command_setup); + } + let mut child = match command.spawn() { + Ok(child) => child, + Err(_) => { + state.phase = "failed-start".into(); + root.save(state)?; + protocol::write(&mut std::io::stdout(), state)?; + return Err(refused()); + } + }; + state.command_pid = Some(child.id() as i32); + state.command_start_time = os::inspect(child.id() as i32)?.map(|identity| identity.start); + state.phase = "running".into(); + root.save(state)?; + let mut stdout = child.stdout.take().ok_or_else(refused)?; + let mut stderr = child.stderr.take().ok_or_else(refused)?; + os::nonblocking(&stdout)?; + os::nonblocking(&stderr)?; + protocol::write(&mut std::io::stdout(), state)?; + detach_stdio()?; + let mut out = Lines::default(); + let mut err = Lines::default(); + let mut stopping = None; + let mut killed = false; + let mut direct_exited = false; + let mut drained_after_exit = None; + let mut last_census = Instant::now() - IDLE_TICK; + let mut live = true; + let mut stdout_open = true; + let mut stderr_open = true; + loop { + // Alternate bounded reads so stdout bursts cannot starve stderr/stop. + if stdout_open { + stdout_open = out.drain(&mut stdout, log, &state.name, "stdout")?; + } + if stderr_open { + stderr_open = err.drain(&mut stderr, log, &state.name, "stderr")?; + } + let was_exited = direct_exited; + if !direct_exited && let Some(status) = child.try_wait()? { + direct_exited = true; + state.exit_code = status.code(); + state.signal = status.signal(); + } + // Linux's subreaper may adopt exited grandchildren while the direct + // command remains alive. Reap continuously, retaining the direct child's + // wait status if it exits between try_wait and this raw waitpid census. + for (pid, status) in os::reap()? { + if pid == child.id() as i32 { + direct_exited = true; + state.exit_code = status.code(); + state.signal = status.signal(); + } + } + // A live direct child already proves work remains. Census is needed + // only after that child exits (descendants may remain), or during stop. + if (direct_exited && (last_census.elapsed() >= IDLE_TICK || !was_exited)) + || stopping.is_some() + { + live = os::live_group(state.process_group_id, Some(state.guard_pid))?; + last_census = Instant::now(); + } + if os::STOP.swap(false, Ordering::Relaxed) && stopping.is_none() { + state.phase = "stopping".into(); + root.save(state)?; + stopping = Some(Instant::now()); + os::session_signal(state.guard_pid, libc::SIGTERM)?; + } + if !live && direct_exited { + let since = *drained_after_exit.get_or_insert_with(Instant::now); + // Never wait on EOF: foreign fork/exec activity can transiently + // inherit a pipe on Darwin. Drain a bounded final interval instead. + if since.elapsed() >= Duration::from_millis(100) { + if !out.pending.is_empty() { + out.flush(log, &state.name, "stdout", true)?; + } + if !err.pending.is_empty() { + err.flush(log, &state.name, "stderr", true)?; + } + state.phase = if killed { "killed" } else { "exited" }.into(); + root.save(state)?; + return Ok(()); + } + } + if stopping.is_some_and(|start| start.elapsed() >= GRACE) { + state.phase = "kill-pending".into(); + if !killed { + root.save(state)?; + killed = true; + } + os::session_signal(state.guard_pid, libc::SIGKILL)?; + } + let fds: Vec<_> = [ + (stdout_open, stdout.as_raw_fd()), + (stderr_open, stderr.as_raw_fd()), + ] + .into_iter() + .filter_map(|(open, fd)| open.then_some(fd)) + .collect(); + os::wait_activity( + &fds, + if stopping.is_some() || drained_after_exit.is_some() { + TICK + } else { + IDLE_TICK + }, + )?; + } +} + +// A live guard owns shutdown even when a complete recovery census is unavailable +// (for example, a different-UID descendant). Orphans require strict capture. +fn stop_capture( + guard_live: bool, + capture: impl FnOnce() -> Result, +) -> Result<(Option, Option)> { + if guard_live { + Ok((None, capture().ok())) + } else { + Ok((Some(capture()?), None)) + } +} + +fn guard_lost(state: &State) -> Result { + Ok(os::inspect(state.guard_pid)?.is_none_or(|p| p.zombie)) +} + +// None transfers to orphan recovery. A census error cannot bypass that transfer +// when the guard disappears during the census or its witness recheck. +fn guarded_live(state: &State) -> Result> { + if guard_lost(state)? { + return Ok(None); + } + match os::recorded_live(state, false) { + Ok(live) => Ok(Some(live)), + Err(_) if guard_lost(state)? => Ok(None), + Err(error) => Err(error), + } +} + +fn stop(states: &[State]) -> Result<()> { + struct Target { + state: State, + recovery: Option, + admission: Option, + orphan_term: Option, + finished: bool, + } + impl Target { + fn recover(&mut self) -> Result<()> { + // Prefer a fresh census, but retain already-proved identities if + // TERM ended the command witness before the guard was lost. + let recovery = os::Recovery::capture(&self.state) + .or_else(|error| self.admission.take().ok_or(error))?; + recovery.signal(libc::SIGTERM)?; + self.orphan_term = Some(Instant::now()); + self.recovery = Some(recovery); + Ok(()) + } + + fn advance(&mut self, start: Instant) -> Result { + if self.recovery.is_none() { + match guarded_live(&self.state)? { + None => self.recover()?, + Some(false) => return Ok(false), + Some(true) => { + if start.elapsed() >= GRACE + CONFIRM_BOUND { + return Err(refused()); + } + return Ok(true); + } + } + } + let recovery = self.recovery.as_ref().ok_or_else(refused)?; + if !recovery.live()? { + return Ok(false); + } + let elapsed = self.orphan_term.ok_or_else(refused)?.elapsed(); + if elapsed >= GRACE + CONFIRM_BOUND { + return Err(refused()); + } + if elapsed >= GRACE { + recovery.signal(libc::SIGKILL)?; + } + Ok(true) + } + } + let mut active = Vec::new(); + // Validate admission before effects. Capture while a live guard can still + // prove membership, so loss during the later liveness check retains authority. + for state in states { + let guard_live = match os::inspect(state.guard_pid)? { + Some(identity) if !identity.zombie => { + if let Err(error) = os::same(state, &identity) { + if !os::recorded_live(state, false)? { + continue; + } + return Err(error); + } + true + } + _ => false, + }; + let (recovery, admission) = stop_capture(guard_live, || os::Recovery::capture(state))?; + match os::recorded_live(state, false) { + Ok(false) => continue, + Ok(true) => (), + Err(_) if guard_lost(state)? && (recovery.is_some() || admission.is_some()) => (), + Err(error) => return Err(error), + } + active.push(Target { + state: state.clone(), + recovery, + admission, + orphan_term: None, + finished: false, + }); + } + let mut failure = None; + for target in &mut active { + let signalled = if let Some(recovery) = &target.recovery { + target.orphan_term = Some(Instant::now()); + recovery.signal(libc::SIGTERM) + } else { + match os::signal_guard(&target.state) { + Ok(()) => Ok(()), + Err(error) => match guarded_live(&target.state) { + Ok(None) => target.recover(), + Ok(Some(false)) => Ok(()), + Ok(Some(true)) => Err(error), + Err(error) => Err(error), + }, + } + }; + if let Err(error) = signalled { + target.finished = true; + failure.get_or_insert(error); + } + } + let start = Instant::now(); + loop { + let mut live = false; + for target in active.iter_mut().filter(|target| !target.finished) { + match target.advance(start) { + Ok(true) => live = true, + Ok(false) => target.finished = true, + Err(error) => { + target.finished = true; + failure.get_or_insert(error); + } + } + } + if !live { + return failure.map_or(Ok(()), Err); + } + thread::sleep(TICK); + } +} + +fn detach_stdio() -> Result<()> { + let null = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/null")?; + for fd in 0..=2 { + // SAFETY: dup2 replaces only this process's standard descriptors with + // the borrowed live /dev/null descriptor; no pointer is retained. + if unsafe { libc::dup2(null.as_raw_fd(), fd) } < 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) +} + +#[derive(Default)] +struct Lines { + pending: Vec, +} + +impl Lines { + fn drain( + &mut self, + input: &mut impl Read, + log: &mut File, + name: &str, + stream: &str, + ) -> Result { + let mut buffer = [0u8; 8192]; + for _ in 0..32 { + match input.read(&mut buffer) { + Ok(0) => return Ok(false), + Ok(count) => { + for byte in &buffer[..count] { + if *byte == b'\n' { + self.flush(log, name, stream, true)?; + } else { + self.pending.push(*byte); + if self.pending.len() == LINE_LIMIT { + let tail_length = incomplete_utf8_tail(&self.pending); + let tail = self.pending.split_off(self.pending.len() - tail_length); + self.flush(log, name, stream, false)?; + self.pending = tail; + } + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + } + } + Ok(true) + } + + fn flush(&mut self, log: &mut File, name: &str, stream: &str, trim_cr: bool) -> Result<()> { + let bytes = if trim_cr { + self.pending.strip_suffix(b"\r").unwrap_or(&self.pending) + } else { + &self.pending + }; + let message = String::from_utf8_lossy(bytes).replace('\t', " "); + let record = format!("{}\t{name}\t{stream}\t{message}\n", os::timestamp()?); + // One write per complete record on an O_APPEND descriptor, with no + // lock: the kernel positions and writes each append atomically, so + // whole records from sibling guards do not interleave on a local file + // system. A guard never waits on another guard to log, so a stopped or + // hung sibling cannot stall this guard's loop (STOP, exit, census). + // Only a short write (for example a full disk) splits a record. + log.write_all(record.as_bytes())?; + self.pending.clear(); + Ok(()) + } +} + +fn incomplete_utf8_tail(bytes: &[u8]) -> usize { + for index in (bytes.len().saturating_sub(4)..bytes.len()).rev() { + let byte = bytes[index]; + if byte & 0xc0 == 0x80 { + continue; + } + let expected = match byte { + 0xc2..=0xdf => 2, + 0xe0..=0xef => 3, + 0xf0..=0xf4 => 4, + _ => 1, + }; + let available = bytes.len() - index; + return if expected > available { available } else { 0 }; + } + 0 +} + +#[cfg(test)] +mod tests { + use super::incomplete_utf8_tail; + + #[test] + fn live_guard_admission_tolerates_unavailable_recovery_census() { + let denied = || Err::<(), _>(std::io::Error::from_raw_os_error(libc::EPERM)); + let (recovery, admission) = super::stop_capture(true, denied).unwrap(); + assert!(recovery.is_none() && admission.is_none()); + assert!(super::stop_capture(false, denied).is_err()); + let (recovery, admission) = super::stop_capture(true, || Ok(42)).unwrap(); + assert!(recovery.is_none()); + assert_eq!(admission, Some(42)); + } + + #[test] + fn chunk_boundary_retains_only_incomplete_codepoints() { + for text in ["a", "é", "€", "🦀"] { + let bytes = text.as_bytes(); + assert_eq!(incomplete_utf8_tail(bytes), 0); + for length in 1..bytes.len() { + assert_eq!(incomplete_utf8_tail(&bytes[..length]), length); + } + } + assert_eq!(incomplete_utf8_tail(b"abc\xff"), 0); + } +} diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs new file mode 100644 index 000000000..a37a5d649 --- /dev/null +++ b/packages/lifecycle-supervisor/tests/fake_caller.rs @@ -0,0 +1,2205 @@ +//! Real-process qualification through the framed public protocol. No shell, +//! network, daemon, user HOME or registry is involved. +use serde_json::{Value, json}; +use std::{ + cell::RefCell, + fs, + io::{Read, Write}, + os::{fd::AsRawFd, unix::fs::MetadataExt}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::atomic::{AtomicU64, Ordering}, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +static SEQUENCE: AtomicU64 = AtomicU64::new(0); +const TOKEN: &str = "synthetic-owner-token"; +const TIMEOUT: Duration = Duration::from_secs(15); + +struct Fixture { + root: PathBuf, + owned: RefCell>, +} + +impl Fixture { + fn new() -> Self { + let parent = std::env::var_os("TMPDIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = parent.join(format!( + "supervisor-test-{}-{nonce}-{}", + std::process::id(), + SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + fs::DirBuilder::new().mode(0o700).create(&root).unwrap(); + Self { + root, + owned: RefCell::new(Vec::new()), + } + } + + fn process(&self, name: &str, mode: &str) -> Value { + json!({"name":name,"argv":[std::env::current_exe().unwrap(),"--exact","fixture_process","--nocapture"],"cwd":self.root,"env":{"HOME":self.root,"PATH":"/usr/bin:/bin","HACK_DAEMON_AUTOSTART_DISABLE":"1","SUPERVISOR_FIXTURE":mode}}) + } + + fn launch(&self, processes: Vec) -> Value { + json!({"version":1,"root":self.root,"ownerToken":TOKEN,"processes":processes}) + } + + fn control(&self, token: &str) -> Value { + json!({"version":1,"root":self.root,"ownerToken":token}) + } + + fn request(&self, action: &str, request: &Value) -> (Child, Value) { + let mut child = command(&self.root, action).spawn().unwrap(); + send(child.stdin.as_mut().unwrap(), request); + // Deliberately retain stdin: frame receipt must not depend on EOF. + let reply = receive(child.stdout.as_mut().unwrap()); + self.remember(&reply); + (child, reply) + } + + fn remember(&self, reply: &Value) { + for state in reply["processes"].as_array().into_iter().flatten() { + for field in ["guardPid", "commandPid"] { + if let Some(id) = state[field].as_i64().and_then(|id| identity(id as i32)) { + self.owned.borrow_mut().push(id); + } + } + } + } + + fn start(&self, processes: Vec) -> (Child, Value) { + self.request("run", &self.launch(processes)) + } + + fn start_generation(&self, processes: Vec, token: &str) -> (Child, Value) { + let mut plan = self.launch(processes); + plan["ownerToken"] = json!(token); + self.request("run", &plan) + } + + fn stop_generation(&self, token: &str) { + let (mut stop, _) = self.request("stop", &self.control(token)); + assert!(wait(&mut stop, TIMEOUT).success()); + } + + fn stop(&self) { + let mut child = command(&self.root, "stop").spawn().unwrap(); + send(child.stdin.as_mut().unwrap(), &self.control(TOKEN)); + drop(child.stdin.take()); + let status = wait(&mut child, TIMEOUT); + if !status.success() { + // Receipts contain only identity/phase metadata, never argv/env. + // Read after the same exit deadline so diagnostic collection cannot + // change stop admission or hide a resumed-guard timing failure. + let receipt = receive(child.stdout.as_mut().unwrap()); + panic!("owned stop failed: {receipt}"); + } + } + + /// Take the fixture's stop lock once its current holder releases it. A + /// launch writes its ready receipt just before it drops this lock, and a + /// guard keeps its startup lease until publication, so the lock can still + /// be held briefly after `start` returns. + #[track_caller] + fn take_stop_lock(&self) -> fs::File { + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(self.root.join(".stop.lock")) + .unwrap(); + eventually(|| { + // SAFETY: flock only borrows this live descriptor. + (unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0) + .then_some(()) + }); + lock + } + + fn state(&self, name: &str) -> Value { + eventually(|| { + fs::read(self.root.join(format!("{name}.json"))) + .ok() + .and_then(|b| serde_json::from_slice(&b).ok()) + }) + } + + fn log(&self) -> String { + fs::read_to_string(self.root.join("lifecycle.log")).unwrap_or_default() + } + + #[track_caller] + fn await_log(&self, needle: &str) { + eventually(|| self.log().contains(needle).then_some(())); + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + // Ask the normal owner first, including after an assertion panic. Stop + // has a bounded wait and all inputs refer only to this fixture root. + if let Ok(mut stop) = command(&self.root, "stop").spawn() { + if let Some(mut stdin) = stop.stdin.take() { + let bytes = serde_json::to_vec(&self.control(TOKEN)).unwrap_or_default(); + let _ = stdin.write_all(&(bytes.len() as u32).to_be_bytes()); + let _ = stdin.write_all(&bytes); + } + let deadline = Instant::now() + Duration::from_secs(8); + while stop.try_wait().ok().flatten().is_none() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(20)); + } + let _ = stop.kill(); + let _ = stop.wait(); + } + // Receipt copies are captured before tests deliberately corrupt state. + // A still-matching guard or command birth proves the original session + // still exists. Recheck each enumerated PID's birth before its signal. + for anchor in self.owned.borrow().iter() { + if identity(anchor.pid).as_ref() != Some(anchor) { + continue; + } + let mut members: Vec<_> = census() + .into_iter() + .filter(|p| p.session == anchor.session) + .collect(); + members.sort_by_key(|p| p.pid == anchor.session); + for member in members { + if identity(member.pid).as_ref() == Some(&member) { + // SAFETY: matched kernel birth and session from a live + // captured launch identity; no user-owned state is used. + unsafe { + libc::kill(member.pid, libc::SIGKILL); + } + } + } + } + let _ = fs::remove_dir_all(&self.root); + } +} + +#[derive(Clone, PartialEq, Eq)] +struct Identity { + pid: i32, + session: i32, + birth: u64, +} + +#[cfg(target_os = "linux")] +fn identity(pid: i32) -> Option { + let text = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let fields: Vec<_> = text.rsplit_once(") ")?.1.split_whitespace().collect(); + Some(Identity { + pid, + session: fields.get(3)?.parse().ok()?, + birth: fields.get(19)?.parse().ok()?, + }) +} + +#[cfg(target_os = "macos")] +fn identity(pid: i32) -> Option { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: kernel writes at most the supplied live buffer size. + let size = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + std::mem::size_of::() as i32, + ) + }; + if size as usize != std::mem::size_of::() { + return None; + } + let info = unsafe { info.assume_init() }; + let session = unsafe { libc::getsid(pid) }; + (session > 1).then_some(Identity { + pid, + session, + birth: info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec, + }) +} + +#[cfg(target_os = "linux")] +fn census() -> Vec { + fs::read_dir("/proc") + .into_iter() + .flatten() + .flatten() + .filter_map(|e| e.file_name().to_str()?.parse().ok()) + .filter_map(identity) + .collect() +} + +#[cfg(target_os = "macos")] +fn census() -> Vec { + let mut pids = vec![0i32; 65536]; + // SAFETY: initialized vector provides the supplied writable byte count. + let count = + unsafe { libc::proc_listpids(1, 0, pids.as_mut_ptr().cast(), (pids.len() * 4) as i32) }; + if count <= 0 { + return Vec::new(); + } + pids.into_iter() + .take(count as usize / 4) + .filter(|p| *p > 1) + .filter_map(identity) + .collect() +} + +fn parent_pid(pid: i32) -> Option { + #[cfg(target_os = "linux")] + { + let text = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + text.rsplit_once(") ")? + .1 + .split_whitespace() + .nth(1)? + .parse() + .ok() + } + #[cfg(target_os = "macos")] + { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: live writable buffer with the exact native struct size. + let size = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + std::mem::size_of::() as i32, + ) + }; + (size as usize == std::mem::size_of::()) + .then(|| unsafe { info.assume_init() }.pbi_ppid as i32) + } +} + +fn command(root: &Path, action: &str) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_hack-lifecycle-supervisor")); + command + .arg(action) + .env_clear() + .env("HOME", root) + .env("PATH", "/usr/bin:/bin") + .env("HACK_DAEMON_AUTOSTART_DISABLE", "1") + .env("SUPERVISOR_PARENT_ONLY", "synthetic-parent-only") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + command +} + +fn send(writer: &mut impl Write, value: &Value) { + let bytes = serde_json::to_vec(value).unwrap(); + writer + .write_all(&(bytes.len() as u32).to_be_bytes()) + .unwrap(); + writer.write_all(&bytes).unwrap(); + writer.flush().unwrap(); +} + +#[track_caller] +fn exact(reader: &mut (impl Read + AsRawFd), bytes: &mut [u8]) { + let end = Instant::now() + TIMEOUT; + let mut offset = 0; + while offset < bytes.len() { + assert!(Instant::now() < end, "framed response timed out"); + let mut poll = libc::pollfd { + fd: reader.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: one initialized pollfd is writable and alive for this call. + let n = unsafe { libc::poll(&mut poll, 1, 100) }; + if n == 0 { + continue; + } + assert!(n > 0, "poll failed"); + let n = reader.read(&mut bytes[offset..]).unwrap(); + assert!(n > 0, "response closed before a complete frame"); + offset += n; + } +} + +#[track_caller] +fn receive(reader: &mut (impl Read + AsRawFd)) -> Value { + let mut header = [0; 4]; + exact(reader, &mut header); + let length = u32::from_be_bytes(header) as usize; + assert!((1..=1024 * 1024).contains(&length)); + let mut bytes = vec![0; length]; + exact(reader, &mut bytes); + serde_json::from_slice(&bytes).unwrap() +} + +#[track_caller] +fn wait(child: &mut Child, timeout: Duration) -> std::process::ExitStatus { + let end = Instant::now() + timeout; + loop { + if let Some(status) = child.try_wait().unwrap() { + return status; + } + if Instant::now() >= end { + let _ = child.kill(); + let _ = child.wait(); + panic!("owned subprocess exceeded deadline"); + } + thread::sleep(Duration::from_millis(20)); + } +} + +#[track_caller] +fn eventually(mut predicate: impl FnMut() -> Option) -> T { + let end = Instant::now() + TIMEOUT; + loop { + if let Some(value) = predicate() { + return value; + } + assert!( + Instant::now() < end, + "expected fixture observation did not arrive" + ); + thread::sleep(Duration::from_millis(20)); + } +} + +fn pid(state: &Value, field: &str) -> i32 { + state[field].as_i64().unwrap() as i32 +} + +struct ForeignChild(Child); +impl ForeignChild { + fn spawn() -> Self { + use std::os::unix::process::CommandExt; + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", "fixture_process", "--nocapture"]) + .env_clear() + .env("SUPERVISOR_FIXTURE", "hold") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + // SAFETY: changes only the new child, with no allocated state. + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + Self(command.spawn().unwrap()) + } +} +impl Drop for ForeignChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn refused(fixture: &Fixture, action: &str, request: Value) { + let mut child = command(&fixture.root, action).spawn().unwrap(); + send(child.stdin.as_mut().unwrap(), &request); + drop(child.stdin.take()); + assert!( + !wait(&mut child, TIMEOUT).success(), + "unsafe operation was accepted" + ); +} + +#[test] +fn starts_and_records_identity() { + let f = Fixture::new(); + let (mut caller, receipt) = f.start(vec![f.process("worker", "hold")]); + assert_eq!(receipt["version"], 1); + f.await_log("fixture-ready"); + let state = f.state("worker"); + assert_eq!(state["ownerToken"], TOKEN); + assert_eq!(state["guardPid"], state["processGroupId"]); + assert!(state["guardStartTime"].as_u64().unwrap() > 0); + assert!(state["bootId"].as_str().is_some_and(|v| !v.is_empty())); + assert!(pid(&state, "commandPid") > 1); + assert_eq!( + fs::metadata(f.root.join("worker.json")).unwrap().mode() & 0o777, + 0o600 + ); + assert_eq!( + fs::metadata(f.root.join("lifecycle.log")).unwrap().mode() & 0o777, + 0o600 + ); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn stop_after_caller_closed() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + drop(caller.stdin.take()); + drop(caller.stdout.take()); + f.await_log("fixture-ready"); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn supervisor_kill_then_recover_stop() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + caller.kill().unwrap(); + caller.wait().unwrap(); + refused(&f, "run", f.launch(vec![f.process("worker", "hold")])); + f.stop(); +} + +#[test] +fn term_ignoring_command_is_killed_after_grace() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]); + f.await_log("fixture-ready"); + let start = Instant::now(); + f.stop(); + assert!( + start.elapsed() >= Duration::from_secs(5), + "TERM grace was shortened" + ); + assert!( + start.elapsed() < Duration::from_secs(7), + "stop exceeded its confirmation budget" + ); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn foreign_owner_cannot_stop() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + refused(&f, "stop", f.control("different-owner")); + let state = f.state("worker"); + // SAFETY: signal zero observes this fixture's process without delivering a signal. + assert_eq!(unsafe { libc::kill(pid(&state, "commandPid"), 0) }, 0); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn failed_start_runs_no_later_command() { + let f = Fixture::new(); + let broken = + json!({"name":"broken","argv":["/nonexistent/synthetic-program"],"cwd":f.root,"env":{}}); + refused( + &f, + "run", + f.launch(vec![broken, f.process("later", "hold")]), + ); + assert!( + !f.root.join("later.json").exists(), + "later command started after failure" + ); + assert!(!f.log().contains("fixture-ready")); +} + +#[test] +fn output_bursts_and_long_lines_include_partial_tail() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "burst")]); + assert!(wait(&mut caller, TIMEOUT).success()); + let text = f.log(); + assert!(text.contains("\tstderr\tfixture-error")); + assert!(text.contains("partial-tail")); + assert_eq!(text.matches("burst-record-").count(), 2000); + assert_eq!(text.bytes().filter(|v| *v == b'X').count(), 300_000); + for line in text.lines() { + let fields: Vec<_> = line.splitn(4, '\t').collect(); + assert_eq!(fields.len(), 4); + assert_eq!(fields[1], "worker"); + assert!(fields[0].ends_with('Z')); + } +} + +/// Without a log lock, each record is one O_APPEND write. Two guards bursting +/// at once, including 64 KiB records far above PIPE_BUF, must still leave only +/// whole records: every line parses, belongs to one guard and is not mixed. +#[test] +fn concurrent_guards_append_whole_records_without_a_lock() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![ + f.process("alpha", "burst"), + f.process("beta", "burst"), + ]); + assert!(wait(&mut caller, TIMEOUT).success()); + let text = f.log(); + for name in ["alpha", "beta"] { + let own: Vec<_> = text + .lines() + .filter(|line| line.split('\t').nth(1) == Some(name)) + .collect(); + assert_eq!( + own.iter().filter(|l| l.contains("\tburst-record-")).count(), + 2000 + ); + let xs: usize = own + .iter() + .filter_map(|line| line.splitn(4, '\t').nth(3)) + .filter(|message| message.starts_with('X')) + .map(|message| { + assert!(message.bytes().all(|b| b == b'X'), "mixed record"); + message.len() + }) + .sum(); + assert_eq!(xs, 300_000); + } + for line in text.lines() { + let fields: Vec<_> = line.splitn(4, '\t').collect(); + assert_eq!(fields.len(), 4, "split record: {line:.80}"); + assert!(matches!(fields[1], "alpha" | "beta"), "{line:.80}"); + assert!(fields[0].ends_with('Z')); + } +} + +#[test] +fn zombies_do_not_hold_liveness() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "zombie")]); + assert!( + wait(&mut caller, TIMEOUT).success(), + "zombie-only session remained alive" + ); + assert!(f.log().contains("unreaped-zombie-confirmed")); +} + +#[test] +fn sentinel_environment_never_enters_argv_or_state() { + let f = Fixture::new(); + let mut process = f.process("worker", "sentinel"); + let sentinel = format!( + "synthetic-credential-{}-{}", + std::process::id(), + SEQUENCE.fetch_add(1, Ordering::Relaxed) + ); + process["env"]["SYNTHETIC_PRIVATE_VALUE"] = json!(sentinel); + let (mut caller, _) = f.start(vec![process]); + f.await_log("sentinel-received"); + for entry in fs::read_dir(&f.root).unwrap().flatten() { + if entry.file_type().unwrap().is_file() { + let bytes = fs::read(entry.path()).unwrap(); + assert!( + !bytes + .windows(sentinel.len()) + .any(|v| v == sentinel.as_bytes()), + "private value persisted" + ); + } + } + let state = f.state("worker"); + #[cfg(target_os = "linux")] + for process_id in [ + caller.id() as i32, + pid(&state, "guardPid"), + pid(&state, "commandPid"), + ] { + let bytes = fs::read(format!("/proc/{process_id}/cmdline")).unwrap(); + assert!( + !bytes + .windows(sentinel.len()) + .any(|v| v == sentinel.as_bytes()), + "private value in argv" + ); + } + #[cfg(target_os = "linux")] + { + let bytes = fs::read(format!("/proc/{}/environ", pid(&state, "guardPid"))).unwrap(); + assert!(bytes.is_empty(), "guard inherited caller environment"); + } + #[cfg(target_os = "macos")] + { + let output = Command::new("/bin/ps") + .args([ + "-o", + "command=", + "-p", + &format!( + "{},{},{}", + caller.id(), + pid(&state, "guardPid"), + pid(&state, "commandPid") + ), + ]) + .output() + .unwrap(); + assert!( + !output + .stdout + .windows(sentinel.len()) + .any(|v| v == sentinel.as_bytes()), + "private value in argv" + ); + } + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn child_descriptor_allowlist() { + use std::os::fd::{FromRawFd, OwnedFd}; + use std::os::unix::fs::OpenOptionsExt; + let f = Fixture::new(); + let file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(f.root.join("unexpected-inheritance")) + .unwrap(); + // Deliberately emulate a caller's inheritable descriptor. The launcher's + // startup descriptor sweep and each exec boundary must exclude it. + let raw = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_DUPFD, 200) }; + assert!(raw >= 200); + let _inheritable = unsafe { OwnedFd::from_raw_fd(raw) }; + let (mut caller, _) = f.start(vec![f.process("worker", "descriptors")]); + assert!(wait(&mut caller, TIMEOUT).success()); + assert!(f.log().contains("descriptor-allowlist-ok")); + assert!(!f.log().contains("descriptor-leak")); +} + +#[test] +fn repeated_immediate_restart_waits_for_instance_lock_release() { + let f = Fixture::new(); + let mut previous = None; + for generation in 0..8 { + let token = format!("synthetic-generation-{generation}"); + let (caller, _) = f.start_generation(vec![f.process("worker", "hold")], &token); + if let Some(mut old) = previous.take() { + assert!(wait(&mut old, TIMEOUT).success()); + } + f.stop_generation(&token); + previous = Some(caller); + } + assert!(wait(previous.as_mut().unwrap(), TIMEOUT).success()); +} + +#[test] +fn raised_descriptor_limit_does_not_delay_startup() { + // Exercise the largest supported soft limit without changing this host's + // hard limit. The exact million-fd qualification remains separate below. + let mut limit: libc::rlimit = unsafe { std::mem::zeroed() }; + assert_eq!( + unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, + 0 + ); + descriptor_limit_startup(limit.rlim_max.min(1_048_576)); +} + +#[test] +#[ignore = "qualification requires a hard RLIMIT_NOFILE of at least 1048576"] +fn million_descriptor_limit_qualification() { + descriptor_limit_startup(1_048_576); +} + +fn descriptor_limit_startup(descriptors: libc::rlim_t) { + use std::os::unix::process::CommandExt; + let f = Fixture::new(); + let mut cmd = command(&f.root, "run"); + // SAFETY: adjusts only this freshly forked fixture child's descriptor limit. + unsafe { + cmd.pre_exec(move || { + let limit = libc::rlimit { + rlim_cur: descriptors, + rlim_max: descriptors, + }; + if libc::setrlimit(libc::RLIMIT_NOFILE, &limit) != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut caller = cmd.spawn().unwrap(); + let start = Instant::now(); + send( + caller.stdin.as_mut().unwrap(), + &f.launch(vec![f.process("worker", "descriptors")]), + ); + let receipt = receive(caller.stdout.as_mut().unwrap()); + f.remember(&receipt); + assert_eq!(receipt["outcome"], "ready"); + assert!(wait(&mut caller, TIMEOUT).success()); + assert!(start.elapsed() < Duration::from_secs(3)); + assert!(f.log().contains("descriptor-allowlist-ok")); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "bounded idle CPU/context-switch measurement; run alone with --nocapture"] +fn idle_guard_cpu_and_wakeup_measurement() { + fn sample(pid: i32) -> (u64, u64) { + let stat = fs::read_to_string(format!("/proc/{pid}/stat")).unwrap(); + let fields: Vec<_> = stat + .rsplit_once(") ") + .unwrap() + .1 + .split_whitespace() + .collect(); + let ticks = fields[11].parse::().unwrap() + fields[12].parse::().unwrap(); + let status = fs::read_to_string(format!("/proc/{pid}/status")).unwrap(); + let switches = status + .lines() + .find_map(|line| { + line.strip_prefix("voluntary_ctxt_switches:") + .map(|v| v.trim().parse::().unwrap()) + }) + .unwrap(); + (ticks, switches) + } + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let guard = pid(&f.state("worker"), "guardPid"); + let hz = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as f64; + for trial in 1..=3 { + let before = sample(guard); + let before_parent = sample(caller.id() as i32); + let start = Instant::now(); + thread::sleep(Duration::from_secs(5)); + let wall = start.elapsed().as_secs_f64(); + let after = sample(guard); + let after_parent = sample(caller.id() as i32); + let cpu = (after.0 - before.0) as f64 / hz; + let switches = after.1 - before.1; + eprintln!( + "trial={trial} wall_seconds={wall:.3} guard_cpu_seconds={cpu:.3} guard_voluntary_switches={switches} supervisor_cpu_seconds={:.3} supervisor_voluntary_switches={}", + (after_parent.0 - before_parent.0) as f64 / hz, + after_parent.1 - before_parent.1 + ); + assert!(cpu / wall < 0.01, "idle guard exceeded 1% of one CPU"); + assert!( + (switches as f64) / wall < 5.0, + "idle guard exceeded 5 voluntary switches/s" + ); + } + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn natural_exit_during_stop_admission_is_successful_absence() { + let f = Fixture::new(); + for generation in 0..8 { + let token = format!("synthetic-generation-{generation}"); + let (mut caller, _) = f.start_generation(vec![f.process("worker", "unicode")], &token); + let (mut stop, receipt) = f.request("stop", &f.control(&token)); + assert!(wait(&mut stop, TIMEOUT).success()); + assert_eq!(receipt["outcome"], "stopped"); + assert!(wait(&mut caller, TIMEOUT).success()); + } +} + +#[test] +fn status_reports_kernel_liveness_separately_from_persisted_phase() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + let (mut status, receipt) = f.request("status", &f.control(TOKEN)); + assert!(wait(&mut status, TIMEOUT).success()); + assert_eq!(receipt["liveProcessNames"], json!(["worker"])); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); + let mut state = f.state("worker"); + state["phase"] = json!("running"); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + let (mut status, receipt) = f.request("status", &f.control(TOKEN)); + assert!(wait(&mut status, TIMEOUT).success()); + assert_eq!(receipt["processes"][0]["phase"], "running"); + assert_eq!(receipt["liveProcessNames"], json!([])); + assert_eq!(receipt["unknownProcessNames"], json!([])); +} + +#[test] +fn concurrent_stop_requests_settle() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let mut first = command(&f.root, "stop").spawn().unwrap(); + let mut second = command(&f.root, "stop").spawn().unwrap(); + send(first.stdin.as_mut().unwrap(), &f.control(TOKEN)); + send(second.stdin.as_mut().unwrap(), &f.control(TOKEN)); + drop(first.stdin.take()); + drop(second.stdin.take()); + let first_ok = wait(&mut first, TIMEOUT).success(); + let second_ok = wait(&mut second, TIMEOUT).success(); + assert!(first_ok || second_ok, "neither stop request settled"); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn changed_process_group_remains_in_owned_session() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "change-group")]); + f.await_log("changed-group-ready"); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "60-second native idle CPU and kernel wakeup measurement; run alone"] +fn idle_guard_cpu_and_wakeup_measurement() { + fn sample(pid: i32) -> libc::rusage_info_v2 { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: proc_pid_rusage's opaque output points at the complete V2 + // struct for the requested flavor, live and writable for this call. + assert_eq!( + unsafe { libc::proc_pid_rusage(pid, libc::RUSAGE_INFO_V2, info.as_mut_ptr().cast()) }, + 0 + ); + unsafe { info.assume_init() } + } + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let guard = pid(&f.state("worker"), "guardPid"); + let before = sample(guard); + let parent_before = sample(caller.id() as i32); + let start = Instant::now(); + thread::sleep(Duration::from_secs(60)); + let after = sample(guard); + let parent_after = sample(caller.id() as i32); + for (label, before, after) in [ + ("guard", before, after), + ("owner", parent_before, parent_after), + ] { + println!( + "{label}: wall_s={:.3} cpu_ns={} idle_wakeups={} interrupt_wakeups={}", + start.elapsed().as_secs_f64(), + (after.ri_user_time - before.ri_user_time) + + (after.ri_system_time - before.ri_system_time), + after.ri_pkg_idle_wkups - before.ri_pkg_idle_wkups, + after.ri_interrupt_wkups - before.ri_interrupt_wkups + ); + } + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn inherited_ignored_signal_is_reset() { + use std::os::unix::process::CommandExt; + let f = Fixture::new(); + let mut command = command(&f.root, "run"); + // SAFETY: signal changes only the freshly forked test child; it is an + // async-signal-safe operation and captures no allocated state. + unsafe { + command.pre_exec(|| { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + Ok(()) + }); + } + let mut caller = command.spawn().unwrap(); + send( + caller.stdin.as_mut().unwrap(), + &f.launch(vec![f.process("worker", "signal-default")]), + ); + let receipt = receive(caller.stdout.as_mut().unwrap()); + f.remember(&receipt); + f.await_log("signal-default-ok"); + let start = Instant::now(); + f.stop(); + assert!( + start.elapsed() < Duration::from_secs(4), + "command retained ignored TERM disposition" + ); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased() { + use std::os::unix::fs::OpenOptionsExt; + let f = Fixture::new(); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .open(f.root.join("worker.lock")) + .unwrap(); + assert_eq!(unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) }, 0); + let mut caller = command(&f.root, "run").spawn().unwrap(); + send( + caller.stdin.as_mut().unwrap(), + &f.launch(vec![ + f.process("first", "hold"), + f.process("worker", "hold"), + ]), + ); + f.await_log("fixture-ready"); + let first = f.state("first"); + f.remember(&json!({"processes":[first.clone()]})); + let guard = eventually(|| { + census() + .into_iter() + .find(|p| { + p.pid != pid(&first, "guardPid") + && p.session == p.pid + && parent_pid(p.pid) == Some(caller.id() as i32) + }) + .map(|p| p.pid) + }); + // Linux additionally observes the exact fd-2 inode. Both platforms prove + // the lease behavior by a blocked stop after owner death below. + #[cfg(target_os = "linux")] + { + eventually(|| { + fs::read_link(format!("/proc/{guard}/exe")) + .ok() + .filter(|p| { + p.file_name() + .is_some_and(|n| n == "hack-lifecycle-supervisor") + }) + }); + let lease_inode = fs::metadata(f.root.join(".stop.lock")).unwrap().ino(); + eventually(|| { + fs::metadata(format!("/proc/{guard}/fd/2")) + .ok() + .filter(|metadata| metadata.ino() == lease_inode) + }); + } + let guard_identity = identity(guard).unwrap(); + f.owned.borrow_mut().push(guard_identity.clone()); + caller.kill().unwrap(); + caller.wait().unwrap(); + let mut stop = command(&f.root, "stop").spawn().unwrap(); + send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN)); + thread::sleep(Duration::from_millis(150)); + assert!( + stop.try_wait().unwrap().is_none(), + "stop passed an unpublished guard" + ); + assert!(!f.root.join("worker.json").exists()); + // SAFETY: captured direct guard is still blocked on our fixture lock. + assert!(identity(guard).as_ref() == Some(&guard_identity)); + assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0); + assert!(wait(&mut stop, TIMEOUT).success()); + drop(lock); + let (mut next, _) = f.start_generation( + vec![f.process("worker", "hold")], + "synthetic-next-generation", + ); + f.stop_generation("synthetic-next-generation"); + assert!(wait(&mut next, TIMEOUT).success()); +} + +#[test] +fn supervisor_killed_while_ready_receipt_is_blocked_recovers() { + use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd}; + let f = Fixture::new(); + let (_reader, mut writer) = std::os::unix::net::UnixStream::pair().unwrap(); + writer.set_nonblocking(true).unwrap(); + // Saturate the very descriptor used for the caller's receipt. Authored + // launch can finish, but the supervisor cannot commit a ready frame. + let buffer = [0u8; 8192]; + loop { + match writer.write(&buffer) { + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) => panic!("fixture receipt saturation: {e}"), + } + } + // SAFETY: ownership moves from UnixStream into OwnedFd exactly once. + let fd = unsafe { OwnedFd::from_raw_fd(writer.into_raw_fd()) }; + let mut caller = command(&f.root, "run") + .stdout(Stdio::from(fd)) + .spawn() + .unwrap(); + send( + caller.stdin.as_mut().unwrap(), + &f.launch(vec![f.process("worker", "hold")]), + ); + f.await_log("fixture-ready"); + let state = f.state("worker"); + f.remember(&json!({"processes":[state]})); + assert!(caller.try_wait().unwrap().is_none()); + caller.kill().unwrap(); + caller.wait().unwrap(); + f.stop(); +} + +#[test] +fn guard_crash_before_complete_launch_frame_starts_nothing() { + use std::os::unix::process::CommandExt; + let f = Fixture::new(); + let mut cmd = command(&f.root, "guard"); + // SAFETY: changes only the fresh child into its own session. + unsafe { + cmd.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut guard = cmd.spawn().unwrap(); + // The frame header authorizes no launch; kill at the deterministic input + // admission boundary rather than relying on scheduler timing. + guard + .stdin + .as_mut() + .unwrap() + .write_all(&100u32.to_be_bytes()) + .unwrap(); + guard.kill().unwrap(); + guard.wait().unwrap(); + assert!(!f.root.join("worker.json").exists()); + assert!(!f.root.join("lifecycle.log").exists()); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn supervisor_killed_during_output_burst_is_recoverable() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "continuous-output")]); + f.await_log("continuous-output"); + caller.kill().unwrap(); + caller.wait().unwrap(); + f.stop(); +} + +#[test] +fn dead_guard_with_live_session_recovers_term_then_kill() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]); + f.await_log("fixture-ready"); + let state = f.state("worker"); + // SAFETY: this fixture owns the captured live guard incarnation. + unsafe { + libc::kill(pid(&state, "guardPid"), libc::SIGKILL); + } + assert!(wait(&mut caller, TIMEOUT).success()); + refused(&f, "run", f.launch(vec![f.process("worker", "hold")])); + let (mut status, observed) = f.request("status", &f.control(TOKEN)); + assert!(wait(&mut status, TIMEOUT).success()); + assert_eq!(observed["liveProcessNames"], json!([])); + assert_eq!(observed["unknownProcessNames"], json!(["worker"])); + let start = Instant::now(); + let (mut stop, receipt) = f.request("stop", &f.control(TOKEN)); + assert!(wait(&mut stop, TIMEOUT).success()); + assert_eq!(receipt["outcome"], "stopped"); + assert!(start.elapsed() >= Duration::from_secs(5)); + assert_eq!(receipt["liveProcessNames"], json!([])); +} + +#[test] +fn guard_lost_mid_stop_keeps_admission_snapshot() { + guard_lost_during_stop(Duration::ZERO, false); +} + +#[test] +fn guard_lost_late_in_stop_gets_full_recovery_deadline() { + guard_lost_during_stop(Duration::from_secs(3), false); +} + +#[test] +fn expired_target_does_not_abandon_other_recovery() { + guard_lost_during_stop(Duration::from_secs(3), true); +} + +fn guard_lost_during_stop(delay: Duration, blocked: bool) { + let f = Fixture::new(); + let mut processes = vec![f.process("worker", "wrapper-term-child")]; + if blocked { + processes.push(f.process("blocked", "hold")); + } + let (mut caller, _) = f.start(processes); + let blocked_guard = blocked.then(|| pid(&f.state("blocked"), "guardPid")); + // Freeze only after launch and guard startup leases released the stop lock. + drop(f.take_stop_lock()); + if let Some(guard) = blocked_guard { + assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0); + } + eventually(|| f.root.join("wrapper-child-ready").exists().then_some(())); + let state = f.state("worker"); + let child = eventually(|| { + f.log().lines().find_map(|line| { + line.split("wrapper-child ") + .nth(1) + .and_then(|pid| pid.parse::().ok()) + .and_then(identity) + }) + }); + f.owned.borrow_mut().push(child.clone()); + let mut stop = command(&f.root, "stop").spawn().unwrap(); + send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN)); + // The guard has acted on this stop once its wrapper is dead. Its five-second + // grace leaves time to remove the guard before it can KILL the child. + eventually(|| { + identity(pid(&state, "commandPid")) + .is_none_or(|_| process_is_zombie(pid(&state, "commandPid"))) + .then_some(()) + }); + thread::sleep(delay); + // Freeze the guard before removing it, then prove it has not begun its own + // escalation: it persists kill-pending before its first KILL. The child must + // still be live, so only the stop's admission snapshot can KILL it. + let guard = pid(&state, "guardPid"); + let guard_birth = identity(guard).expect("guard identity").birth; + assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0); + // SIGSTOP delivery is asynchronous: wait until the kernel reports this + // guard incarnation stopped before trusting the observations below. + eventually(|| { + (identity(guard).is_some_and(|current| current.birth == guard_birth) + && process_is_stopped(guard)) + .then_some(()) + }); + let frozen = f.state("worker"); + assert_eq!( + frozen["phase"], "stopping", + "guard escalated first: {frozen}" + ); + assert!( + identity(child.pid).is_some_and(|current| current.birth == child.birth) + && !process_is_zombie(child.pid), + "TERM-ignoring child exited before guard loss" + ); + let recovery_started = Instant::now(); + // SAFETY: the frozen guard belongs to this fixture. + assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0); + if !blocked { + assert!(wait(&mut caller, TIMEOUT).success()); + } + let receipt = receive(stop.stdout.as_mut().unwrap()); + let stopped = wait(&mut stop, TIMEOUT).success(); + assert_eq!(stopped, !blocked, "{receipt}"); + assert_eq!( + receipt["outcome"], + if blocked { "unknown" } else { "stopped" } + ); + // Recovery starts its own TERM grace after detecting guard loss, so KILL of + // the captured child cannot precede a fresh five-second interval, even when + // the original guard deadline had almost expired. + let recovered_after = recovery_started.elapsed(); + assert!( + recovered_after >= Duration::from_secs(5), + "recovery skipped its fresh grace: {recovered_after:?}" + ); + assert!(recovered_after < TIMEOUT); + assert!(identity(child.pid).is_none() || process_is_zombie(child.pid)); + if let Some(guard) = blocked_guard { + assert_eq!(unsafe { libc::kill(guard, libc::SIGCONT) }, 0); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); + } +} + +#[test] +fn orphan_recovery_keeps_observed_child_after_term_ends_witness() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "wrapper-term-child")]); + f.await_log("fixture-ready"); + eventually(|| f.root.join("wrapper-child-ready").exists().then_some(())); + let state = f.state("worker"); + let child = eventually(|| { + f.log().lines().find_map(|line| { + line.split("wrapper-child ") + .nth(1) + .and_then(|pid| pid.parse::().ok()) + .and_then(identity) + }) + }); + f.owned.borrow_mut().push(child.clone()); + assert!(child.birth > state["commandStartTime"].as_u64().unwrap()); + // SAFETY: captured guard belongs to this fixture; command/wrapper survives. + assert_eq!( + unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) }, + 0 + ); + assert!(wait(&mut caller, TIMEOUT).success()); + let start = Instant::now(); + let (mut stop, receipt) = f.request("stop", &f.control(TOKEN)); + assert!(wait(&mut stop, TIMEOUT).success()); + assert_eq!(receipt["outcome"], "stopped"); + assert!( + start.elapsed() >= Duration::from_secs(5), + "child did not get a full TERM grace" + ); + assert!(start.elapsed() < Duration::from_secs(7)); + // A zombie is harmless and may await the host's subreaper; no live member remains. + assert!( + census() + .into_iter() + .all(|p| p.session != pid(&state, "guardPid") + || identity(p.pid).is_none() + || process_is_zombie(p.pid)) + ); +} + +#[test] +fn stop_waiting_past_lock_bound_reports_unknown_without_acting() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let state = f.state("worker"); + let worker = identity(pid(&state, "commandPid")).expect("command identity"); + // Hold the fixture's stop lock as a stuck stop would. + let lock = f.take_stop_lock(); + let start = Instant::now(); + let mut stop = command(&f.root, "stop").spawn().unwrap(); + send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN)); + // The frame arrives only after the sixteen-second wait; the reader's own + // timeout is shorter, so start reading part of the way through. + thread::sleep(Duration::from_secs(10)); + let receipt = receive(stop.stdout.as_mut().unwrap()); + assert!(!wait(&mut stop, TIMEOUT).success(), "{receipt}"); + assert_eq!(receipt["outcome"], "unknown"); + assert_eq!(receipt["reason"], "stop-lock-wait-expired"); + assert!(start.elapsed() >= Duration::from_secs(16)); + assert!( + identity(worker.pid).as_ref() == Some(&worker), + "stop acted while waiting" + ); + assert_eq!(f.state("worker")["phase"], "running"); + drop(lock); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn invalid_control_requests_still_get_a_framed_refusal() { + let f = Fixture::new(); + for action in ["stop", "status"] { + let (mut child, receipt) = f.request( + action, + &json!({"version": 99, "root": f.root, "ownerToken": TOKEN}), + ); + assert!(!wait(&mut child, TIMEOUT).success(), "{receipt}"); + assert_eq!(receipt["outcome"], "refused"); + assert_eq!(receipt["reason"], "invalid-request"); + assert_eq!(receipt["processes"], json!([])); + } +} + +/// Linux shows a leader whose main thread called `pthread_exit` as `Z` while +/// its worker threads run. It must count as live. After guard loss, recovery +/// signals captured members one by one, so a member misread as a zombie is +/// never captured: stop then reported absence while its worker survived. +#[cfg(target_os = "linux")] +#[test] +fn main_thread_exit_with_live_worker_is_still_stopped() { + for lose_guard in [false, true] { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "pthread-main-exit")]); + f.await_log("fixture-ready"); + let state = f.state("worker"); + let child = eventually(|| { + f.log().lines().find_map(|line| { + line.split("pthread-child ") + .nth(1) + .and_then(|pid| pid.parse::().ok()) + }) + }); + f.owned + .borrow_mut() + .push(identity(child).expect("pthread child identity")); + eventually(|| (process_state(child) == Some(b'Z') && live_task(child)).then_some(())); + if lose_guard { + drop(f.take_stop_lock()); + // SAFETY: the captured guard belongs to this fixture. + assert_eq!( + unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) }, + 0 + ); + assert!(wait(&mut caller, TIMEOUT).success()); + } + let (mut stop, receipt) = f.request("stop", &f.control(TOKEN)); + assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}"); + assert_eq!(receipt["outcome"], "stopped"); + eventually(|| (!live_task(child)).then_some(())); + if !lose_guard { + assert!(wait(&mut caller, TIMEOUT).success()); + } + } +} + +#[cfg(target_os = "linux")] +fn live_task(pid: i32) -> bool { + fs::read_dir(format!("/proc/{pid}/task")) + .into_iter() + .flatten() + .flatten() + .any(|task| { + fs::read_to_string(task.path().join("stat")) + .ok() + .and_then(|stat| stat.rsplit_once(") ").map(|(_, rest)| rest.bytes().next())) + .flatten() + .is_some_and(|state| state != b'Z' && state != b'X') + }) +} + +/// A guard stopped while it held a lock on the shared log (SIGSTOP between +/// lock and unlock) must not stall sibling guards. The test holds an exclusive +/// flock on the log from outside for the whole run, as such a frozen guard +/// would: a guard's output must still reach the log and its project must +/// still stop with a framed `stopped` receipt within the normal bounds. +#[test] +fn held_log_lock_never_stalls_a_guard() { + use std::os::unix::fs::OpenOptionsExt; + let f = Fixture::new(); + let log = fs::OpenOptions::new() + .append(true) + .create(true) + .mode(0o600) + .open(f.root.join("lifecycle.log")) + .unwrap(); + // SAFETY: flock only borrows this live descriptor. + assert_eq!(unsafe { libc::flock(log.as_raw_fd(), libc::LOCK_EX) }, 0); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let start = Instant::now(); + let (mut stop, receipt) = f.request("stop", &f.control(TOKEN)); + assert!(wait(&mut stop, TIMEOUT).success(), "{receipt}"); + assert_eq!(receipt["outcome"], "stopped"); + assert!(start.elapsed() < Duration::from_secs(7)); + assert!(wait(&mut caller, TIMEOUT).success()); + drop(log); +} + +fn process_is_zombie(pid: i32) -> bool { + process_state(pid) == Some(b'Z') +} + +/// Job-control stop observed by the kernel, not merely a submitted SIGSTOP. +fn process_is_stopped(pid: i32) -> bool { + matches!(process_state(pid), Some(b'T' | b't')) +} + +/// Linux /proc state letter; Darwin maps SZOMB to Z and SSTOP to T. +fn process_state(pid: i32) -> Option { + #[cfg(target_os = "linux")] + { + fs::read_to_string(format!("/proc/{pid}/stat")) + .ok() + .and_then(|s| { + s.rsplit_once(") ") + .and_then(|(_, rest)| rest.bytes().next()) + }) + } + #[cfg(target_os = "macos")] + { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: native output buffer is writable and exactly sized. + let size = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 1, + info.as_mut_ptr().cast(), + std::mem::size_of::() as i32, + ) + }; + if size as usize != std::mem::size_of::() { + return None; + } + Some(match unsafe { info.assume_init() }.pbi_status { + libc::SZOMB => b'Z', + libc::SSTOP => b'T', + _ => b'R', + }) + } +} + +#[test] +fn unreaped_matching_guard_proves_legacy_session_ownership() { + let f = Fixture::new(); + let (caller, _) = f.start(vec![f.process("worker", "hold")]); + let mut caller = ForeignChild(caller); + f.await_log("fixture-ready"); + let mut state = f.state("worker"); + // Freeze the parent only after launch has released the stop lock; + // otherwise the frozen parent would hold it and stop could never run. + drop(f.take_stop_lock()); + // Keep the parent from reaping its guard, then exercise legacy birth absence. + assert_eq!( + unsafe { libc::kill(caller.0.id() as i32, libc::SIGSTOP) }, + 0 + ); + // SIGSTOP delivery is asynchronous. Observe our own child's stopped + // wait status before killing its guard, so it cannot win the reap race. + eventually(|| { + let mut status = 0; + // SAFETY: this is our unreaped child; status is writable for waitpid. + let observed = unsafe { + libc::waitpid( + caller.0.id() as i32, + &mut status, + libc::WNOHANG | libc::WUNTRACED, + ) + }; + (observed == caller.0.id() as i32 && libc::WIFSTOPPED(status)).then_some(()) + }); + assert_eq!( + unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) }, + 0 + ); + eventually(|| process_is_zombie(pid(&state, "guardPid")).then_some(())); + state["commandStartTime"] = Value::Null; + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + f.stop(); + assert_eq!( + unsafe { libc::kill(caller.0.id() as i32, libc::SIGCONT) }, + 0 + ); + assert!(wait(&mut caller.0, TIMEOUT).success()); +} + +#[test] +fn guard_dying_during_stop_gets_a_fresh_authored_term_grace() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]); + f.await_log("fixture-ready"); + let guard = pid(&f.state("worker"), "guardPid"); + let birth = identity(guard).unwrap(); + // Freeze only after launch and guard startup leases released the stop lock. + drop(f.take_stop_lock()); + // Suspend this owned guard so its pending TERM cannot reach the command. + assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0); + let mut stop = command(&f.root, "stop").spawn().unwrap(); + send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN)); + thread::sleep(Duration::from_millis(150)); + assert!(identity(guard).as_ref() == Some(&birth)); + let killed_at = Instant::now(); + assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0); + let receipt = receive(stop.stdout.as_mut().unwrap()); + assert!(wait(&mut stop, TIMEOUT).success()); + assert_eq!(receipt["outcome"], "stopped"); + assert!(killed_at.elapsed() >= Duration::from_secs(5)); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn stale_receipt_cannot_signal_another_process() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let mut state = f.state("worker"); + let original = state.clone(); + // A valid-looking receipt with an incorrect birth time must never become + // authority just because its PID currently exists. + state["guardStartTime"] = json!(state["guardStartTime"].as_u64().unwrap() + 1); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + let (mut stop, _) = f.request("stop", &f.control(TOKEN)); + wait(&mut stop, TIMEOUT); + assert_eq!(unsafe { libc::kill(pid(&original, "commandPid"), 0) }, 0); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&original).unwrap(), + ) + .unwrap(); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn receipt_naming_a_foreign_pid_cannot_stop_it() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let mut foreign = ForeignChild::spawn(); + let original = f.state("worker"); + let mut state = original.clone(); + state["guardPid"] = json!(foreign.0.id()); + state["processGroupId"] = json!(foreign.0.id()); + // Linux births have clock-tick precision: two distinct live PIDs can have + // equal births. Make this stale identity differ deterministically. + let foreign_identity = eventually(|| identity(foreign.0.id() as i32)); + state["guardStartTime"] = json!(foreign_identity.birth + 1); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + let (mut stop, _) = f.request("stop", &f.control(TOKEN)); + wait(&mut stop, TIMEOUT); + let foreign_running = foreign.0.try_wait().unwrap().is_none(); + drop(foreign); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&original).unwrap(), + ) + .unwrap(); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); + assert!(foreign_running, "stale receipt signalled another process"); +} + +#[test] +fn stale_reused_session_leader_does_not_block_fresh_launch() { + let f = Fixture::new(); + let (mut first, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + f.stop(); + assert!(wait(&mut first, TIMEOUT).success()); + let mut foreign = ForeignChild::spawn(); + let foreign_identity = eventually(|| identity(foreign.0.id() as i32)); + let mut state = f.state("worker"); + state["guardPid"] = json!(foreign.0.id()); + state["processGroupId"] = json!(foreign.0.id()); + state["guardStartTime"] = json!(foreign_identity.birth + 1); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&state).unwrap(), + ) + .unwrap(); + let (mut next, _) = f.start_generation( + vec![f.process("worker", "hold")], + "synthetic-next-generation", + ); + f.stop_generation("synthetic-next-generation"); + assert!(wait(&mut next, TIMEOUT).success()); + assert!( + foreign.0.try_wait().unwrap().is_none(), + "foreign reused session was signalled" + ); +} + +#[test] +fn dead_reused_session_cannot_authorize_signals_from_old_receipt() { + use std::os::unix::process::CommandExt; + let f = Fixture::new(); + let (mut old, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + f.stop(); + assert!(wait(&mut old, TIMEOUT).success()); + let terminal = f.state("worker"); + // Use a real dead session leader and real surviving child. Relocate the + // old receipt's SID to that session rather than waiting for host PID wrap. + thread::sleep(Duration::from_millis(30)); + let mut launch = Command::new(std::env::current_exe().unwrap()); + launch + .args(["--exact", "fixture_process", "--nocapture"]) + .env_clear() + .env("SUPERVISOR_FIXTURE", "leave-session-child") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + unsafe { + launch.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut leader = launch.spawn().unwrap(); + let sid = leader.id() as i32; + let mut output = String::new(); + leader + .stdout + .take() + .unwrap() + .read_to_string(&mut output) + .unwrap(); + assert!(wait(&mut leader, TIMEOUT).success()); + let child_pid: i32 = output + .lines() + .find_map(|line| { + line.strip_prefix("session-child ") + .and_then(|pid| pid.parse().ok()) + }) + .unwrap(); + let child = identity(child_pid).unwrap(); + f.owned.borrow_mut().push(child.clone()); + assert_eq!(child.session, sid); + assert!(identity(sid).is_none(), "replacement leader must be gone"); + let mut relocated = terminal.clone(); + relocated["guardPid"] = json!(sid); + relocated["processGroupId"] = json!(sid); + for phase in ["running", "starting", "exited", "killed", "failed-start"] { + relocated["phase"] = json!(phase); + fs::write( + f.root.join("worker.json"), + serde_json::to_vec(&relocated).unwrap(), + ) + .unwrap(); + let (mut status, observed) = f.request("status", &f.control(TOKEN)); + let uncertain = matches!(phase, "running" | "starting"); + assert_eq!(wait(&mut status, TIMEOUT).success(), !uncertain); + assert_eq!( + observed["unknownProcessNames"].as_array().unwrap().len(), + usize::from(uncertain) + ); + let (mut stop, receipt) = f.request("stop", &f.control(TOKEN)); + assert_eq!( + receipt["outcome"], + if uncertain { "unknown" } else { "stopped" } + ); + assert_eq!(wait(&mut stop, TIMEOUT).success(), !uncertain); + assert!( + identity(child_pid).as_ref() == Some(&child), + "unrelated session was signalled" + ); + } + // A terminal old generation must not fence an unrelated generation. + let (mut next, _) = + f.start_generation(vec![f.process("worker", "hold")], "synthetic-new-owner"); + f.stop_generation("synthetic-new-owner"); + assert!(wait(&mut next, TIMEOUT).success()); + assert!(identity(child_pid).as_ref() == Some(&child)); +} + +#[test] +fn preexisting_broad_log_is_refused_before_launch() { + use std::os::unix::fs::PermissionsExt; + let f = Fixture::new(); + fs::write(f.root.join("lifecycle.log"), b"sentinel").unwrap(); + fs::set_permissions( + f.root.join("lifecycle.log"), + fs::Permissions::from_mode(0o644), + ) + .unwrap(); + refused(&f, "run", f.launch(vec![f.process("worker", "hold")])); + assert!(!f.root.join("worker.json").exists()); + assert_eq!(fs::read(f.root.join("lifecycle.log")).unwrap(), b"sentinel"); +} + +#[test] +fn incomplete_frame_has_one_whole_handoff_deadline() { + let f = Fixture::new(); + let mut caller = command(&f.root, "run").spawn().unwrap(); + let start = Instant::now(); + caller.stdin.as_mut().unwrap().write_all(&[0]).unwrap(); + thread::sleep(Duration::from_secs(6)); + // Complete the header late, leaving the announced body absent. A reset + // deadline per read would let this request survive sixteen seconds. + caller + .stdin + .as_mut() + .unwrap() + .write_all(&[0, 0, 16]) + .unwrap(); + assert!(!wait(&mut caller, Duration::from_secs(6)).success()); + assert!(start.elapsed() < Duration::from_secs(12)); + assert!(!f.root.join("lifecycle.log").exists()); +} + +#[test] +fn failure_teardown_stops_separate_command_group() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let state = f.state("worker"); + let command_pid = pid(&state, "commandPid"); + let guard_pid = pid(&state, "guardPid"); + assert_ne!(command_pid, guard_pid); + drop(f); + assert!(wait(&mut caller, TIMEOUT).success()); + assert!(identity(command_pid).is_none(), "fixture command leaked"); + assert!(identity(guard_pid).is_none(), "fixture guard leaked"); +} + +#[test] +fn unresponsive_guard_stop_remains_unknown() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "hold")]); + f.await_log("fixture-ready"); + let state = f.state("worker"); + let guard = pid(&state, "guardPid"); + // Freeze only after launch and guard startup leases released the stop lock. + drop(f.take_stop_lock()); + // SAFETY: the guard belongs to this fixture and its receipt is immutable. + assert_eq!(unsafe { libc::kill(guard, libc::SIGSTOP) }, 0); + let (mut stop, receipt) = f.request("stop", &f.control(TOKEN)); + assert_eq!(receipt["outcome"], "unknown"); + assert!(!wait(&mut stop, TIMEOUT).success()); + assert_eq!(unsafe { libc::kill(pid(&state, "commandPid"), 0) }, 0); + refused(&f, "run", f.launch(vec![f.process("worker", "hold")])); + assert_eq!(unsafe { libc::kill(guard, libc::SIGCONT) }, 0); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn new_generation_ignores_retained_terminal_receipts() { + let f = Fixture::new(); + let (mut first, _) = f.start(vec![f.process("old-name", "hold")]); + f.await_log("fixture-ready"); + f.stop(); + assert!(wait(&mut first, TIMEOUT).success()); + let mut launch = f.launch(vec![f.process("renamed", "hold")]); + launch["ownerToken"] = json!("synthetic-next-generation"); + let (mut next, _) = f.request("run", &launch); + let (mut stop, receipt) = f.request("stop", &f.control("synthetic-next-generation")); + assert!(wait(&mut stop, TIMEOUT).success()); + assert!(wait(&mut next, TIMEOUT).success()); + assert_eq!(receipt["processes"].as_array().unwrap().len(), 1); + assert!( + f.root.join("old-name.json").exists(), + "terminal receipt was discarded" + ); +} + +#[test] +fn case_only_rename_accepts_terminal_receipt_with_previous_spelling() { + let f = Fixture::new(); + let (mut first, _) = f.start(vec![f.process("Worker", "hold")]); + f.await_log("fixture-ready"); + f.stop(); + assert!(wait(&mut first, TIMEOUT).success()); + // Also emulate the case-insensitive lookup on Linux: the payload retains + // its original spelling while the lookup uses the new command name. + fs::rename(f.root.join("Worker.json"), f.root.join("worker.json")).unwrap(); + let (mut next, _) = f.start_generation( + vec![f.process("worker", "hold")], + "synthetic-next-generation", + ); + f.stop_generation("synthetic-next-generation"); + assert!(wait(&mut next, TIMEOUT).success()); +} + +#[test] +fn command_joining_guard_group_still_receives_kill() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "join-guard-group")]); + f.await_log("joined-guard-group"); + let start = Instant::now(); + f.stop(); + assert!(start.elapsed() >= Duration::from_secs(5)); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn process_group_leader_caller_refuses_without_launch() { + use std::os::unix::process::CommandExt; + let f = Fixture::new(); + let mut command = command(&f.root, "run"); + // SAFETY: changes only the newly forked child, using no allocated state. + unsafe { + command.pre_exec(|| { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut caller = command.spawn().unwrap(); + send( + caller.stdin.as_mut().unwrap(), + &f.launch(vec![f.process("worker", "hold")]), + ); + drop(caller.stdin.take()); + assert!(!wait(&mut caller, TIMEOUT).success()); + assert!(!f.root.join("worker.json").exists()); + assert!(!f.root.join("lifecycle.log").exists()); +} + +#[test] +fn stop_serializes_with_multiple_process_launches() { + let f = Fixture::new(); + let processes: Vec<_> = (0..16) + .map(|i| f.process(&format!("worker-{i}"), "hold")) + .collect(); + let mut caller = command(&f.root, "run").spawn().unwrap(); + send(caller.stdin.as_mut().unwrap(), &f.launch(processes)); + f.state("worker-0"); + let mut stop = command(&f.root, "stop").spawn().unwrap(); + send(stop.stdin.as_mut().unwrap(), &f.control(TOKEN)); + let ready = receive(caller.stdout.as_mut().unwrap()); + f.remember(&ready); + let receipt = receive(stop.stdout.as_mut().unwrap()); + assert_eq!(receipt["outcome"], "stopped"); + assert!(wait(&mut stop, TIMEOUT).success()); + assert!(wait(&mut caller, TIMEOUT).success()); + for state in ready["processes"].as_array().unwrap() { + assert!( + identity(pid(state, "commandPid")).is_none(), + "a launch escaped concurrent stop" + ); + } +} + +#[test] +fn unicode_codepoint_crossing_chunk_boundary_is_preserved() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "unicode")]); + assert!(wait(&mut caller, TIMEOUT).success()); + let log = f.log(); + assert_eq!(log.matches('🦀').count(), 1); + assert!( + !log.contains('\u{fffd}'), + "UTF-8 chunk boundary corrupted a codepoint" + ); +} + +#[test] +fn case_insensitive_process_name_collision_refuses_all_launches() { + let f = Fixture::new(); + refused( + &f, + "run", + f.launch(vec![ + f.process("Worker", "hold"), + f.process("worker", "hold"), + ]), + ); + assert!(!f.root.join("Worker.json").exists()); + assert!(!f.root.join("worker.json").exists()); + assert!(!f.root.join("lifecycle.log").exists()); +} + +#[test] +fn status_observes_without_creating_files() { + let f = Fixture::new(); + let (mut status, receipt) = f.request("status", &f.control(TOKEN)); + assert!(wait(&mut status, TIMEOUT).success()); + assert_eq!(receipt["outcome"], "observed"); + assert!(receipt["processes"].as_array().unwrap().is_empty()); + assert_eq!(fs::read_dir(&f.root).unwrap().count(), 0); +} + +#[cfg(target_os = "linux")] +#[test] +fn adopted_orphan_is_reaped_while_direct_command_lives() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "orphan-exits")]); + f.await_log("orphan-ready"); + let orphan = eventually(|| { + f.log().lines().find_map(|line| { + line.split("orphan-ready ") + .nth(1) + .and_then(|value| value.parse::().ok()) + }) + }); + eventually(|| identity(orphan).is_none().then_some(())); + let state = f.state("worker"); + assert!(identity(pid(&state, "commandPid")).is_some()); + f.stop(); + assert!(wait(&mut caller, TIMEOUT).success()); +} + +#[test] +fn fixture_process() { + let Ok(mode) = std::env::var("SUPERVISOR_FIXTURE") else { + return; + }; + assert!( + std::env::var_os("SUPERVISOR_PARENT_ONLY").is_none(), + "command inherited caller environment" + ); + match mode.as_str() { + "burst" => { + let mut out = std::io::stdout().lock(); + for i in 0..2000 { + writeln!(out, "burst-record-{i}").unwrap(); + } + out.write_all(&vec![b'X'; 300_000]).unwrap(); + out.write_all(b"\npartial-tail").unwrap(); + out.flush().unwrap(); + eprintln!("fixture-error"); + std::process::exit(0); + } + "descriptors" => { + assert_eq!(unsafe { libc::isatty(0) }, 0); + let mut byte = [0u8]; + assert_eq!( + std::io::stdin().read(&mut byte).unwrap(), + 0, + "command inherited private protocol input" + ); + let mut leaked = false; + for fd in 3..1024 { + // SAFETY: F_GETFD observes descriptor state without mutation. + if unsafe { libc::fcntl(fd, libc::F_GETFD) } >= 0 { + leaked = true; + } + } + println!( + "{}", + if leaked { + "descriptor-leak" + } else { + "descriptor-allowlist-ok" + } + ); + std::process::exit(if leaked { 1 } else { 0 }); + } + "zombie" => { + // SAFETY: fixture immediately exits in the fork child; it invokes + // no allocator or Rust destructors after fork in that branch. + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + libc::_exit(0); + } + } + let mut event = std::mem::MaybeUninit::::zeroed(); + // SAFETY: observe only this fixture's direct child into a writable + // buffer. WNOWAIT confirms exit while deliberately retaining it. + assert_eq!( + unsafe { + libc::waitid( + libc::P_PID, + child as libc::id_t, + event.as_mut_ptr(), + libc::WEXITED | libc::WNOWAIT, + ) + }, + 0 + ); + println!("unreaped-zombie-confirmed {child}"); + std::io::stdout().flush().unwrap(); + std::process::exit(0); + } + #[cfg(target_os = "linux")] + "orphan-exits" => { + let (mut input, output) = std::os::unix::net::UnixStream::pair().unwrap(); + // SAFETY: both fork descendants use only async-signal-safe libc + // calls and immediately exit; no Rust allocator state is reused. + let intermediary = unsafe { libc::fork() }; + assert!(intermediary >= 0); + if intermediary == 0 { + unsafe { + let orphan = libc::fork(); + if orphan == 0 { + libc::usleep(300_000); + libc::_exit(0); + } + let bytes = orphan.to_be_bytes(); + libc::write(output.as_raw_fd(), bytes.as_ptr().cast(), bytes.len()); + libc::_exit(if orphan < 0 { 98 } else { 0 }); + } + } + let mut status = 0; + assert_eq!( + unsafe { libc::waitpid(intermediary, &mut status, 0) }, + intermediary + ); + let mut bytes = [0; 4]; + input.read_exact(&mut bytes).unwrap(); + let orphan = i32::from_be_bytes(bytes); + assert!(orphan > 1); + println!("orphan-ready {orphan}"); + } + "change-group" => { + // SAFETY: fork child uses only async-signal-safe calls and never + // returns to Rust, so the libtest thread state is not reused. + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + if libc::setpgid(0, 0) != 0 { + libc::_exit(98); + } + libc::write(1, c"changed-group-ready\n".as_ptr().cast(), 20); + loop { + libc::pause(); + } + } + } + } + "join-guard-group" => { + // SAFETY: fixture joins its existing session leader's group; only + // this authored test process's group/disposition are changed. + unsafe { + let session = libc::getsid(0); + assert!(session > 1); + assert_eq!(libc::setpgid(0, session), 0); + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + println!("joined-guard-group"); + } + "unicode" => { + let mut out = std::io::stdout().lock(); + out.write_all(&vec![b'a'; 65535]).unwrap(); + out.write_all("🦀\n".as_bytes()).unwrap(); + out.flush().unwrap(); + std::process::exit(0); + } + #[cfg(target_os = "linux")] + "non-dumpable" => { + // SAFETY: affects only this owned fixture's dumpability, not its UID. + assert_eq!(unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0) }, 0); + println!("non-dumpable-ready"); + } + "signal-default" => { + let mut disposition: libc::sigaction = unsafe { std::mem::zeroed() }; + assert_eq!( + unsafe { libc::sigaction(libc::SIGTERM, std::ptr::null(), &mut disposition) }, + 0 + ); + assert_eq!(disposition.sa_sigaction, libc::SIG_DFL); + println!("signal-default-ok"); + } + "continuous-output" => loop { + println!("continuous-output {}", "x".repeat(4096)); + thread::sleep(Duration::from_millis(1)); + }, + "sentinel" => { + assert!(std::env::var_os("SYNTHETIC_PRIVATE_VALUE").is_some()); + println!("sentinel-received"); + } + "ignore-term" => { + // SAFETY: SIG_IGN is a valid process-local disposition. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + } + "leave-session-child" => { + // SAFETY: fork child uses only async-signal-safe operations and + // never returns into libtest's multithreaded runtime. + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + libc::close(1); + loop { + libc::pause(); + } + } + } + println!("session-child {child}"); + std::io::stdout().flush().unwrap(); + std::process::exit(0); + } + "wrapper-term-child" => { + // Make the child newer than the command even with Linux clock-tick + // resolution, so it cannot accidentally satisfy the old birth bound. + thread::sleep(Duration::from_millis(100)); + // SAFETY: fork child invokes only libc operations and never returns + // into Rust/libtest's inherited multithreaded allocator state. + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + let fd = libc::open( + c"wrapper-child-ready".as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL, + 0o600, + ); + if fd < 0 { + libc::_exit(98); + } + libc::close(fd); + loop { + libc::pause(); + } + } + } + println!("wrapper-child {child}"); + } + #[cfg(target_os = "linux")] + "pthread-main-exit" => { + extern "C" fn worker(_: *mut libc::c_void) -> *mut libc::c_void { + loop { + // SAFETY: pause has no preconditions. + unsafe { + libc::pause(); + } + } + } + // The fork child is single-threaded, so its main thread is the + // thread-group leader. It starts one worker and ends only its main + // thread (as `pthread_exit` does); Linux then shows the leader as + // `Z` while the worker runs. + // SAFETY: the child calls only libc thread and exit functions and + // never returns into Rust/libtest state. + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + let mut thread: libc::pthread_t = std::mem::zeroed(); + if libc::pthread_create( + &mut thread, + std::ptr::null(), + worker, + std::ptr::null_mut(), + ) != 0 + { + libc::_exit(96); + } + // End only the main thread, as pthread_exit would, without + // its unwinder: the raw exit syscall is thread-scoped. + libc::syscall(libc::SYS_exit, 0); + libc::_exit(95); + } + } + println!("pthread-child {child}"); + } + "hold" => (), + _ => std::process::exit(99), + } + println!("fixture-ready"); + std::io::stdout().flush().unwrap(); + loop { + thread::sleep(Duration::from_secs(1)); + } +} + +use std::os::unix::fs::DirBuilderExt; + +#[test] +fn completed_stop_fences_a_run_whose_frame_arrives_after_retirement() { + let f = Fixture::new(); + let mut delayed = command(&f.root, "run").spawn().unwrap(); + f.stop(); + send( + delayed.stdin.as_mut().unwrap(), + &f.launch(vec![f.process("late", "hold")]), + ); + assert!(!wait(&mut delayed, TIMEOUT).success()); + assert!(!f.root.join("late.json").exists()); +} + +#[cfg(target_os = "linux")] +#[test] +fn non_dumpable_command_keeps_its_process_uid_for_orphan_recovery() { + let f = Fixture::new(); + let (mut caller, _) = f.start(vec![f.process("worker", "non-dumpable")]); + f.await_log("non-dumpable-ready"); + let state = f.state("worker"); + let command = pid(&state, "commandPid"); + // On an unprivileged runner the inode owner changes to root, while the + // process UID stays ours. The recovery path must use the latter. + if unsafe { libc::geteuid() } != 0 { + assert_eq!( + fs::metadata(format!("/proc/{command}/stat")).unwrap().uid(), + 0 + ); + } + // SAFETY: this fixture owns the still-running guard recorded at launch. + assert_eq!( + unsafe { libc::kill(pid(&state, "guardPid"), libc::SIGKILL) }, + 0 + ); + f.stop(); + let _ = wait(&mut caller, TIMEOUT); + let live = fs::read_to_string(format!("/proc/{command}/stat")) + .ok() + .is_some_and(|stat| { + stat.rsplit_once(") ") + .is_some_and(|(_, fields)| !fields.starts_with("Z ") && !fields.starts_with("X ")) + }); + assert!(!live); +} diff --git a/scripts/check-native-env-plan-cli.ts b/scripts/check-native-env-plan-cli.ts index 7aa23b708..5fe694d7f 100644 --- a/scripts/check-native-env-plan-cli.ts +++ b/scripts/check-native-env-plan-cli.ts @@ -318,6 +318,7 @@ async function invoke( PATH: "/usr/bin:/bin", HOME: home, HACK_LOGGER: "console", + HACK_DAEMON_AUTOSTART_DISABLE: "1", AWS_SECRET_ACCESS_KEY: sentinel, ...extraEnv, }, diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index a5e161cf0..f83baf12f 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -141,6 +141,34 @@ const restartPolicyStaleUses = [ { engine: "exited", bound: 3 }, ] as const; const contracts: readonly ModelContract[] = [ + { + name: "lifecycle-supervisor", + module: "LifecycleSupervisor", + states: 426, + invariant: "NoDuplicate", + action: "Retry", + fields: ["duplicate = TRUE", "live = TRUE", "durable = TRUE"], + additionalControls: [ + { + name: "sid-reuse", + negative: true, + invariant: "NoForeignSignal", + action: "RecoverOrphan", + fields: [ + 'identity = "sid-reused"', + "unsafeSignal = TRUE", + "guard = FALSE", + ], + }, + { + name: "witness-loss", + negative: true, + invariant: "RecoveryAuthorityRetained", + action: "WitnessExit", + fields: ["snapshot = FALSE", "witness = FALSE", "live = TRUE"], + }, + ], + }, { name: "env-migration", module: "Migration", diff --git a/tests/models/tla/lifecycle-supervisor/COVERAGE.md b/tests/models/tla/lifecycle-supervisor/COVERAGE.md new file mode 100644 index 000000000..400a0eebf --- /dev/null +++ b/tests/models/tla/lifecycle-supervisor/COVERAGE.md @@ -0,0 +1,106 @@ +# Lifecycle supervisor ownership model + +`LifecycleSupervisor.tla` models one configured command name and at most two +start attempts. This is a finite control-flow abstraction, not a model of kernel observations or lock admission. The first artifact is a +standalone Rust binary exercised by a fake caller; the CLI remains unchanged. + +| Action/property | Source transition | Regression boundary | +| --- | --- | --- | +| `Reserve`, `Publish`, `Spawn`; `DurableBeforeLaunch` | Per-command flock, guard session creation, synced identity before argv spawn | `starts_and_records_identity`; refused publication/spawn cannot run a later command | +| `SupervisorDeath`; `GuardRetainsLock` | Resident guard retains flock and stdout/stderr readers after project supervisor loss | `supervisor_kill_then_recover_stop`; output continues after supervisor SIGKILL | +| `GuardDeath`, `Retry`; `NoDuplicate` | Retained session census refuses a retry even if separately killed guard released its flock | Guard-loss retry and identity refusal regression | +| `Stop`, `Kill`; `NoForeignSignal` | Model assumes identity remains valid across signaling; source performs native rechecks | Wrong-owner/stale-identity refusal; TERM-ignore escalation | +| `ProcessExit`, `Reap` | Session census ignores zombies; reap direct/adopted children; release lock after no live members | Zombie liveness; child changes process group within session | +| `StopUnknown` | Bounded confirmation can report uncertainty while live members remain | Honest unknown recovery outcome, never successful absence | + +Positive TLC exploration: 426 distinct states, queue exhausted, depth 14. Negative +control sets `SkipLiveFence = TRUE`; `Retry` then violates `NoDuplicate` with live, +durable and duplicate all true in the same state. Both guard-loss and project-loss +traces can witness this violation; the registry requires the named action and +same-state fields without depending on worker scheduling. + +Checker: checksum-pinned TLA+ 1.7.4 artifact (TLC reports version 2.19), Java 17, +two workers and a bounded heap. The maintained runner pins the positive count and +validates the negative invariant/action/witness. Apalache is not qualified here. + +The guard-owned Stop and Kill checks still assume matching identity. The added +orphan signal admission is exercised by a separate SID-reuse negative control. +The orphan path includes an environment SID-reuse action and requires an exact recorded incarnation +witness, represented by matching identity. The sid-reuse negative control removes +that witness guard and must violate NoForeignSignal at RecoverOrphan with identity +sid-reused. Native clock units, enumeration and identity-to-effect races are omitted. `GuardRetainsLock` similarly checks the abstract action +assignments, not descriptor inheritance or actual flock ownership. `NoDuplicate` +checks the selected live fence and its negative control in this finite abstraction; +it does not prove lock/census correctness. The positive count now includes ReuseSessionID and RecoverOrphan. Terminal phases +have no recovery signal action. + +Assumptions: flock serialization, atomic/synchronized publication, reliable native +identity and session census, and rechecking identity before effects. Signal delivery +does not imply process exit: `KillPending` can retain live processes, and +`StopUnknown` is distinct from `Exited`. No fairness or eventual exit is assumed. +An identity change blocks signaling. A separately killed guard leaves an orphaned +session; it never grants new launch authority simply because its lock was released. + +Omissions: actual filesystem durability, command contents, multi-command rollback, +pipe buffers, descriptor creation races, scheduler latency, resource use, power loss, +PID reuse between observation and signaling, hostile same-user mutation, and +commands escaping their session with `setsid`. Those remain native implementation +or explicitly unsupported boundaries. This finite model does not qualify a shipped +artifact, macOS behavior, CLI integration or the default runner. + +Native recovery added after this abstraction is deliberately not claimed as model +coverage: dead-guard TERM/grace/KILL recovery, supervisor death before guard +publication, inherited launch-admission lease, stop/exit inspection races, census +errors and restricted views, high descriptor limits, EINTR retries, and idle +poll scheduling. RecoverOrphan and OrphanKill model capture, witness loss and KILL admission; +wall-clock grace timing and native confirmation remain implementation tests. +ReuseSessionID represents a later survivor after its replacement leader exits. Native regressions include +`dead_guard_with_live_session_recovers_term_then_kill`, +`supervisor_and_guard_prepublication_crashes_keep_stop_admission_leased`, +`supervisor_killed_while_ready_receipt_is_blocked_recovers`, and +`guard_crash_before_complete_launch_frame_starts_nothing`. +They qualify only their exercised boundaries on the host running them. +No observation/lock/census refinement or liveness proof is asserted here; the +shared registry and model-coverage map remain owned by the integration task. + +Durable owner-generation retirement is an additional implementation fence, not a +transition checked by this finite model. Stop records cancellation even before a +late launch frame arrives; run checks it while holding admission. New generations +use fresh tokens. The regression +`completed_stop_fences_a_run_whose_frame_arrives_after_retirement` proves the +late frame creates no command receipt. Linux executable replacement is qualified +by the incremental integration test in the opt-in slice, not by this model. + +The real-process dead_reused_session_cannot_authorize_signals_from_old_receipt +regression creates a dead session leader and surviving child, then relocates +a stale receipt SID to reproduce reuse without forcing the host PID allocator +to wrap. It checks terminal absence, non-terminal unknown without signals, and +fresh-owner launch after a terminal receipt. + +Orphan recovery now models WitnessExit after RecoverOrphan captured the old +incarnations. OrphanKill remains enabled after witness loss; the implementation +retains those exact identities through its grace/KILL loop. RecoveryAuthorityRetained +and OrphanKillEnabled check that availability boundary. The witness-loss negative +control deliberately clears the captured snapshot on WitnessExit and must violate +RecoveryAuthorityRetained with live work and no snapshot. SID reuse may follow +an emptied session even during stopping; the old snapshot never authorizes a +new incarnation. The native wrapper/TERM-ignoring-child regression checks this +sequence with real processes. New descendants not in the snapshot remain an +uncovered recovery/unknown boundary; no eventual-exit claim is made. + +Stop now captures members while a live guard proves ownership as well. GuardDeath +retains the stopping phase, and WitnessExit may happen before or after guard loss. +The native guard_lost_mid_stop_keeps_admission_snapshot regression kills the guard +after TERM ended its wrapper and checks KILL and confirmed absence for its child. + +Recovery deadlines are not modeled: each recovered target receives a fresh five-second +TERM grace plus two seconds for confirmation, independently of the original guard +wait. guard_lost_late_in_stop_gets_full_recovery_deadline kills the guard three +seconds after wrapper death and checks successful KILL confirmation and child absence; the early-loss variant checks the same boundary without delay. +Live-guard admission snapshots are best-effort; a denied census does not prevent +the guard-owned shutdown path. Orphan capture stays strict (unit-tested admission seam). + +The census-to-guard-loss handoff and multi-target timeout scheduling are native +implementation boundaries, not modeled. guarded_live rechecks guard loss after +a failed census. expired_target_does_not_abandon_other_recovery checks that one +paused guard timing out does not prevent another target from completing KILL. diff --git a/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla new file mode 100644 index 000000000..e68d1b885 --- /dev/null +++ b/tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla @@ -0,0 +1,118 @@ +------------------------ MODULE LifecycleSupervisor ------------------------ +EXTENDS Naturals +CONSTANT SkipLiveFence, SkipOrphanFence, LoseRecoverySnapshot +VARIABLES phase, durable, guard, locked, live, supervisor, attempts, + identity, unsafeSignal, duplicate, witness, snapshot, orphanKilled +vars == <> + +Init == /\ phase = "Absent" + /\ durable = FALSE /\ guard = FALSE /\ locked = FALSE + /\ live = FALSE /\ supervisor = FALSE /\ attempts = 0 + /\ identity = "match" /\ unsafeSignal = FALSE /\ duplicate = FALSE + /\ witness = FALSE /\ snapshot = FALSE /\ orphanKilled = FALSE + +Reserve == /\ phase \in {"Absent", "Exited"} /\ ~live /\ attempts < 2 + /\ phase' = "Starting" /\ guard' = TRUE /\ locked' = TRUE + /\ supervisor' = TRUE /\ durable' = FALSE + /\ attempts' = attempts + 1 + /\ /\ witness' = FALSE /\ snapshot' = FALSE /\ orphanKilled' = FALSE + /\ UNCHANGED <> +Publish == /\ phase = "Starting" /\ ~durable + /\ durable' = TRUE + /\ UNCHANGED <> +Spawn == /\ phase = "Starting" /\ durable /\ guard /\ locked + /\ phase' = "Running" /\ live' = TRUE + /\ /\ witness' = TRUE + /\ UNCHANGED <> +StartFailure == /\ phase = "Starting" + /\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE + /\ supervisor' = FALSE + /\ UNCHANGED <> +SupervisorDeath == /\ supervisor /\ phase = "Running" + /\ supervisor' = FALSE /\ phase' = "Recoverable" + /\ UNCHANGED <> +GuardDeath == /\ guard /\ live + /\ guard' = FALSE /\ locked' = FALSE + /\ phase' = IF phase = "Stopping" THEN "Stopping" ELSE "Orphaned" + /\ supervisor' = FALSE + /\ UNCHANGED <> +Retry == /\ ~supervisor /\ live /\ attempts < 2 + /\ SkipLiveFence + /\ duplicate' = TRUE /\ attempts' = attempts + 1 + /\ UNCHANGED <> +ReplaceIdentity == /\ identity = "match" /\ durable + /\ identity' \in {"foreign", "unknown"} + /\ UNCHANGED <> +Stop == /\ phase \in {"Running", "Recoverable"} + /\ guard /\ durable /\ identity = "match" + /\ phase' = "Stopping" /\ snapshot' = TRUE + /\ unsafeSignal' = (unsafeSignal \/ identity # "match") + /\ UNCHANGED <> +Kill == /\ phase = "Stopping" /\ guard /\ identity = "match" + /\ phase' = "KillPending" + /\ unsafeSignal' = (unsafeSignal \/ identity # "match") + /\ UNCHANGED <> +StopUnknown == /\ phase \in {"Stopping", "KillPending"} /\ live + /\ phase' = "StopUnknown" + /\ UNCHANGED <> +ProcessExit == /\ live /\ live' = FALSE + /\ UNCHANGED <> +Reap == /\ ~live /\ phase \notin {"Absent", "Starting", "Exited"} + /\ phase' = "Exited" /\ guard' = FALSE /\ locked' = FALSE + /\ supervisor' = FALSE + /\ UNCHANGED <> +\* Environment: after the old session empties, a new leader can take its +\* numeric SID, fork, and exit. The survivor has no old birth witness. +ReuseSessionID == /\ durable /\ ~guard /\ ~live + /\ identity = "match" + /\ phase \in {"Orphaned", "Exited", "Stopping", "KillPending"} + /\ identity' = "sid-reused" /\ live' = TRUE + /\ UNCHANGED <> +RecoverOrphan == /\ phase = "Orphaned" /\ ~guard /\ live /\ durable + /\ (identity = "match" \/ (SkipOrphanFence /\ identity = "sid-reused")) + /\ unsafeSignal' = (unsafeSignal \/ identity # "match") + /\ phase' = "Stopping" + /\ /\ (witness \/ SkipOrphanFence) /\ snapshot' = TRUE + /\ UNCHANGED <> +WitnessExit == /\ phase = "Stopping" /\ live /\ witness + /\ witness' = FALSE + /\ snapshot' = IF LoseRecoverySnapshot THEN FALSE ELSE snapshot + /\ UNCHANGED <> +OrphanKill == /\ phase = "Stopping" /\ ~guard /\ live /\ snapshot + /\ identity = "match" + /\ phase' = "KillPending" /\ orphanKilled' = TRUE + /\ UNCHANGED <> +Next == WitnessExit \/ OrphanKill \/ ReuseSessionID \/ RecoverOrphan \/ Reserve \/ Publish \/ Spawn \/ StartFailure \/ SupervisorDeath + \/ GuardDeath \/ Retry \/ ReplaceIdentity \/ Stop \/ Kill + \/ StopUnknown \/ ProcessExit \/ Reap +Spec == Init /\ [][Next]_vars +TypeOK == /\ phase \in {"Absent", "Starting", "Running", "Recoverable", + "Orphaned", "Stopping", "KillPending", "StopUnknown", "Exited"} + /\ attempts \in 0..2 /\ identity \in {"match", "foreign", "unknown", "sid-reused"} + /\ \A b \in {durable, guard, locked, live, supervisor, + unsafeSignal, duplicate, witness, snapshot, orphanKilled} : b \in BOOLEAN +DurableBeforeLaunch == live => durable +GuardRetainsLock == guard => locked +NoForeignSignal == ~unsafeSignal +NoDuplicate == ~duplicate +RecoveryAuthorityRetained == (phase \in {"Stopping", "KillPending"} /\ ~guard + /\ live /\ identity = "match") => snapshot +OrphanKillEnabled == (phase = "Stopping" /\ ~guard /\ live /\ ~witness + /\ identity = "match" /\ snapshot) => ENABLED OrphanKill +============================================================================= diff --git a/tests/models/tla/lifecycle-supervisor/negative.cfg b/tests/models/tla/lifecycle-supervisor/negative.cfg new file mode 100644 index 000000000..7659fcd6c --- /dev/null +++ b/tests/models/tla/lifecycle-supervisor/negative.cfg @@ -0,0 +1,6 @@ +SPECIFICATION Spec +CONSTANT LoseRecoverySnapshot = FALSE +CONSTANT SkipOrphanFence = FALSE +CONSTANT SkipLiveFence = TRUE +INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate RecoveryAuthorityRetained OrphanKillEnabled +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/lifecycle-supervisor/positive.cfg b/tests/models/tla/lifecycle-supervisor/positive.cfg new file mode 100644 index 000000000..2383ec14c --- /dev/null +++ b/tests/models/tla/lifecycle-supervisor/positive.cfg @@ -0,0 +1,6 @@ +SPECIFICATION Spec +CONSTANT LoseRecoverySnapshot = FALSE +CONSTANT SkipOrphanFence = FALSE +CONSTANT SkipLiveFence = FALSE +INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate RecoveryAuthorityRetained OrphanKillEnabled +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg b/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg new file mode 100644 index 000000000..02e0a9b41 --- /dev/null +++ b/tests/models/tla/lifecycle-supervisor/sid-reuse.cfg @@ -0,0 +1,6 @@ +SPECIFICATION Spec +CONSTANT SkipLiveFence = FALSE +CONSTANT LoseRecoverySnapshot = FALSE +CONSTANT SkipOrphanFence = TRUE +INVARIANTS TypeOK DurableBeforeLaunch GuardRetainsLock NoForeignSignal NoDuplicate RecoveryAuthorityRetained OrphanKillEnabled +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/lifecycle-supervisor/witness-loss.cfg b/tests/models/tla/lifecycle-supervisor/witness-loss.cfg new file mode 100644 index 000000000..fb7e98a26 --- /dev/null +++ b/tests/models/tla/lifecycle-supervisor/witness-loss.cfg @@ -0,0 +1,6 @@ +SPECIFICATION Spec +CONSTANT SkipLiveFence = FALSE +CONSTANT SkipOrphanFence = FALSE +CONSTANT LoseRecoverySnapshot = TRUE +INVARIANTS TypeOK NoForeignSignal RecoveryAuthorityRetained OrphanKillEnabled +CHECK_DEADLOCK FALSE