diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 389e85a48..a4e6ded93 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -744,6 +744,22 @@ jobs:
HACK_E2E_COMPOSE_PLUGIN_SHA256="$(sha256sum "$HACK_E2E_COMPOSE_PLUGIN_PATH" | cut -d ' ' -f1)"
export HACK_E2E_DOCKER_HOST HACK_E2E_NETWORK_SOURCE_REVISION HACK_E2E_NETWORK_CLI_SHA256 HACK_E2E_NETWORK_COMPILER_SHA256 HACK_E2E_COMPOSE_PLUGIN_PATH HACK_E2E_COMPOSE_PLUGIN_SHA256
HACK_E2E_REQUIRE_DOCKER=1 HACK_E2E_REQUIRE_TMUX=1 bun run test:e2e:local:docker --shard="$HACK_E2E_SHARD"
+ - name: Setup supervisor toolchain
+ if: matrix.shard == 4
+ uses: dtolnay/rust-toolchain@1.97.1
+ - name: Qualify lifecycle scenarios with the supervisor runner
+ if: matrix.shard == 4
+ env:
+ HACK_DAEMON_AUTOSTART_DISABLE: '1'
+ HACK_LIFECYCLE_RUNNER: supervisor
+ HACK_LIFECYCLE_SUPERVISOR_BINARY: ${{ github.workspace }}/dist/hack-lifecycle-supervisor
+ HACK_E2E_REQUIRE_DOCKER: '1'
+ HACK_E2E_DOCKER: '1'
+ HACK_E2E_CLI_BIN: ${{ github.workspace }}/dist/hack
+ run: |
+ cargo +1.97.1 build --locked --release --manifest-path packages/lifecycle-supervisor/Cargo.toml --target-dir .hack-local/target/lifecycle-supervisor
+ cp .hack-local/target/lifecycle-supervisor/release/hack-lifecycle-supervisor dist/hack-lifecycle-supervisor
+ bun tests/e2e/run.ts --only=lifecycle-host-process,lifecycle-session-recovery
- name: Preserve process-policy capsule and startup diagnostic
# Closed fixed-field JSON only (counts, enums, booleans); the gh log viewer truncates it inline.
if: failure()
diff --git a/.github/workflows/lifecycle-supervisor.yml b/.github/workflows/lifecycle-supervisor.yml
index 54bd02643..d576c87b9 100644
--- a/.github/workflows/lifecycle-supervisor.yml
+++ b/.github/workflows/lifecycle-supervisor.yml
@@ -4,11 +4,47 @@ on:
pull_request:
paths:
- packages/lifecycle-supervisor/**
+ - src/lib/lifecycle-supervisor.ts
+ - src/lib/project-lifecycle-supervisor.ts
+ - src/lib/project-lifecycle-controller.ts
+ - src/lib/lifecycle-runtime.ts
+ - tests/lifecycle-supervisor-controller.test.ts
+ - tests/lifecycle-supervisor-observers.test.ts
+ - tests/lifecycle-state-publication.test.ts
+ - tests/lifecycle-runner-config.test.ts
+ - tests/e2e/scenarios/lifecycle-*.ts
+ - src/lib/project-lifecycle-hygiene.ts
+ - src/lib/lifecycle-runner.ts
+ - src/lib/runtime-projects.ts
+ - src/lib/private-file.ts
+ - src/lib/projects-registry-lock.ts
+ - src/lib/child-environment.ts
+ - src/commands/host-ps.ts
+ - src/commands/project.ts
+ - src/commands/doctor.ts
- .github/workflows/lifecycle-supervisor.yml
push:
branches: [next, main]
paths:
- packages/lifecycle-supervisor/**
+ - src/lib/lifecycle-supervisor.ts
+ - src/lib/project-lifecycle-supervisor.ts
+ - src/lib/project-lifecycle-controller.ts
+ - src/lib/lifecycle-runtime.ts
+ - tests/lifecycle-supervisor-controller.test.ts
+ - tests/lifecycle-supervisor-observers.test.ts
+ - tests/lifecycle-state-publication.test.ts
+ - tests/lifecycle-runner-config.test.ts
+ - tests/e2e/scenarios/lifecycle-*.ts
+ - src/lib/project-lifecycle-hygiene.ts
+ - src/lib/lifecycle-runner.ts
+ - src/lib/runtime-projects.ts
+ - src/lib/private-file.ts
+ - src/lib/projects-registry-lock.ts
+ - src/lib/child-environment.ts
+ - src/commands/host-ps.ts
+ - src/commands/project.ts
+ - src/commands/doctor.ts
- .github/workflows/lifecycle-supervisor.yml
workflow_dispatch:
@@ -42,5 +78,17 @@ jobs:
run: cargo clippy --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --all-targets -- -D warnings
- name: Test standalone supervisor with private fake callers
run: cargo test --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml -- --test-threads=4
+ - name: Test deterministic guard crash windows (qualification build only)
+ run: cargo test --locked --manifest-path packages/lifecycle-supervisor/Cargo.toml --features test-hooks guard_killed_after_ -- --test-threads=1
- name: Build standalone release executable
run: cargo build --locked --release --manifest-path packages/lifecycle-supervisor/Cargo.toml
+ - uses: oven-sh/setup-bun@v1
+ with:
+ bun-version: "1.4.4"
+ - name: Install locked CLI test dependencies
+ run: bun install --frozen-lockfile
+ - name: Test current-source CLI controller against native supervisor
+ env:
+ HACK_TEST_REQUIRE_SUPERVISOR: '1'
+ HACK_TEST_SUPERVISOR_BINARY: ${{ env.CARGO_TARGET_DIR }}/release/hack-lifecycle-supervisor
+ run: bun test tests/lifecycle-supervisor-controller.test.ts tests/lifecycle-runner-config.test.ts tests/lifecycle-state-publication.test.ts tests/lifecycle-supervisor-observers.test.ts
diff --git a/docs/diagrams/lifecycle-controller.mmd b/docs/diagrams/lifecycle-controller.mmd
new file mode 100644
index 000000000..c072b5006
--- /dev/null
+++ b/docs/diagrams/lifecycle-controller.mmd
@@ -0,0 +1,31 @@
+---
+title: Lifecycle controller · launch receipts and adoption
+---
+stateDiagram-v2
+ direction TB
+ [*] --> Idle
+ Idle --> Starting : Begin
+ Starting --> AwaitReply : Launch [pending published]
+ AwaitReply --> Ready : Accept
+ AwaitReply --> InspectLate : LoseReply
+ InspectLate --> LatePublished : PublishLate [native live]
+ LatePublished --> Failed : FailUnknown
+ Ready --> Adopted : Adopt [native live]
+ Failed --> Adopted : Adopt [native live]
+ Adopted --> AwaitReply : Add [not stopped]
+ Ready --> Aborted : Abort [epoch matches]
+ Failed --> Aborted : Abort [epoch matches]
+ Ready --> Stopped : Stop [absence confirmed]
+ Adopted --> Stopped : Stop [absence confirmed]
+ Failed --> Stopped : Stop [absence confirmed]
+ note right of Starting
+ INV ownership precedes launch
+ end note
+ note right of Adopted
+ INV old cleanup cannot stop
+ Token stable, epoch changes
+ end note
+ note right of Failed
+ Unknown is not absence
+ Verified late effect retained
+ end note
diff --git a/docs/diagrams/lifecycle-controller.svg b/docs/diagrams/lifecycle-controller.svg
new file mode 100644
index 000000000..ad958fdee
--- /dev/null
+++ b/docs/diagrams/lifecycle-controller.svg
@@ -0,0 +1,66 @@
+
\ No newline at end of file
diff --git a/docs/lifecycle-supervisor.md b/docs/lifecycle-supervisor.md
index d3920c32b..4d1938f9f 100644
--- a/docs/lifecycle-supervisor.md
+++ b/docs/lifecycle-supervisor.md
@@ -1,9 +1,11 @@
-# Lifecycle supervisor: standalone contract
+# Lifecycle supervisor contract
-This first slice builds a standalone Rust supervisor with a fake caller. It does
-not change `hack up`, configuration, or the default runner. CLI integration follows
-lifecycle retirement changes; packaging is a separate change. The current default
-continues to use the lifecycle mux session described in [lifecycle](./lifecycle.md).
+The built-in Rust supervisor is an experimental opt-in for host lifecycle
+processes. `lifecycle.runner: "supervisor"` selects it; omission keeps the mux
+runner described in [lifecycle](./lifecycle.md). Its standalone process contracts
+and CLI integration are tested separately. Distribution packaging is a separate
+change: opt-in requires a matching supervisor executable alongside Hack or an
+explicit test executable path.
## Artifact choice
@@ -19,7 +21,7 @@ The supported build targets are `aarch64-apple-darwin`, `x86_64-apple-darwin`, a
## Protocol and private state
-The binary accepts `run`, `stop`, and `status`. Input and output are one four-byte
+The binary accepts `run`, `serve`, `add`, `stop`, and `status`. Input and output are one four-byte
big-endian length followed by that many JSON bytes. Frames are bounded to 1 MiB;
each frame read or write has one 10-second deadline. A complete frame, rather than descriptor close,
commits delivery. Environment values travel only in the input frame and process
@@ -33,6 +35,11 @@ absolute `cwd`, and `env`. Names are safe file identifiers and must also be uniq
without regard to ASCII case, including on case-sensitive filesystems. Commands execute argv
directly. A shell-authored command can explicitly use `sh -c`; the supervisor adds
no shell wrapper. The full environment is explicit, with no ambient inheritance.
+An optional `displayName` retains the user's service label while `name` stays a
+safe native file key. CLI callers use stable hashed keys so labels containing
+spaces or separators cannot change the native directory layout. Optional
+`logPath` selects the existing private lifecycle log; its parent and file must
+pass the same owner, mode and no-alias checks as the native root.
`stop` and `status` accept `{version: 1, root, ownerToken}`. Read-only `status`
returns saved phases plus kernel-verified `liveProcessNames` and
@@ -82,8 +89,9 @@ timers, which is why an expired wait still returns its own receipt.
the supervisor calls `setsid` itself. A caller using `detached: true` or
`setpgid(0, 0)` is refused before authored code starts.
-The root contains per-command state and lock files, an instance lock and
-`lifecycle.log`. Directories are mode 0700; files are mode 0600. Symlinks,
+The root contains per-command state and lock files, an instance lock and,
+for incremental launches, `control.sock`. The default log is `lifecycle.log`.
+Directories are mode 0700; files and the control socket are mode 0600. Symlinks,
hard-linked files, foreign owners and broad permissions are refused. State contains
name, ownership token, guard PID, process group, OS start identity, boot identity,
UID, command PID and start identity when known, phase, and the direct command's exit code or signal
@@ -116,12 +124,12 @@ The bounds below limit each wait, not the total duration of a request:
| Lock | Held by | Held across a blocking call? | Who waits, and the bound |
|---|---|---|---|
-| `.instance.lock` | The project supervisor (`run`), for its lifetime | Yes, its whole run | Another `run` for the root: retries 1 s, then refused. Guards, `stop` and `status` never take it. |
-| `.stop.lock` | `run` from admission until its ready receipt; each new guard (inherited on fd 2) until it publishes and detaches; `stop` for its duration; the project supervisor while it stops on a signal | Yes. A guard frozen before it publishes keeps the lease until it is continued or killed | `stop` waits up to 16 s, then answers `unknown` with reason `stop-lock-wait-expired` and acts on nothing. A launch (`run`) waits up to 16 s, then exits nonzero without a ready receipt and starts nothing. `run` waits up to 10 s for each guard's receipt, then rolls back. |
+| `.instance.lock` | The project supervisor (`run` or `serve`), for its lifetime | Yes, its whole run | Another `run` or `serve` for the root: up to 10 s, then refused. Guards, `add`, `stop` and `status` never take it. |
+| `.stop.lock` | `run` or `serve` from admission until its ready receipt; `add` while it admits one command; each new guard (inherited on fd 2) until it publishes and detaches; `stop` for its duration; the project supervisor while it stops on a signal | Yes. A guard frozen before it publishes keeps the lease until it is continued or killed | `stop` waits up to 16 s, then answers `unknown` with reason `stop-lock-wait-expired` and acts on nothing. `status` waits up to 16 s only when the lock file exists (it never creates it), with the same framed answer on expiry. `add` waits up to 16 s; its forwarder reads the reply for up to 26 s. A launch (`run`) waits up to 16 s, then exits nonzero without a ready receipt and starts nothing. `run` waits up to 10 s for each guard's receipt, then rolls back. |
| `.lock` | The guard of that command, for its lifetime | Yes | Only a new guard for the same command name: up to 10 s, then refused. Never a sibling, `stop` or `status`. |
| Receipts, retirement marker, `lifecycle.log` | No lock | No | Receipts are replaced atomically; a reader that races a replacement reopens the name. Log records are single appends (above). |
-`status` takes no lock. `stop` handles each target separately: a frozen live guard costs
+`status` takes only the bounded stop-lock wait above. `stop` handles each target separately: a frozen live guard costs
its own target the five-second grace plus the two-second confirmation bound (and the
same again if the guard is lost), then that target reports `unknown` while the others
complete.
@@ -212,7 +220,7 @@ Neither feature is part of this unit; interactive workspaces keep tmux by defaul
## Delivery gates
-The CLI opt-in is planned as `lifecycle.runner = "mux" | "supervisor"`, default
+The CLI opt-in is `lifecycle.runner = "mux" | "supervisor"`, default
`mux`, plus a test environment override. Both runners must pass existing lifecycle
unit and e2e scenarios before that integration is ready. Packaging must include the
binary wherever the Compose CLI installs: stable archives, prerelease bundles,
@@ -242,6 +250,97 @@ POSIX `sh -c` intentionally. Interactive tmux environment delivery stays separat
Zellij removal, its breaking-change notice, and the workspace-only `MuxBackend`
contract belong to the third unit.
+## Opt-in CLI integration
+
+
+
+The [controller model coverage](../tests/models/tla/lifecycle-controller/COVERAGE.md)
+maps the finite ownership model to implementation tests, separately from the
+physical-process model above.
+
+The integration slice adds `lifecycle.runner: "supervisor"`; omission remains
+`"mux"`. `HACK_LIFECYCLE_RUNNER=mux|supervisor` overrides the project setting for
+verification. Invalid values fail before hooks or runtime effects. The executable
+must be installed beside Hack, or selected by an absolute
+`HACK_LIFECYCLE_SUPERVISOR_BINARY` test path. An unavailable executable is a refusal,
+not a fallback to another runner. Packaging remains a separate delivery gate.
+
+Persistent before hooks start immediately, in their configured order between
+blocking hooks. The native `serve` operation starts the first command and listens
+on a private Unix socket. Finite `add` clients forward later framed launch requests
+to that same project owner. Environment stays in the frame and process memory.
+The socket uses a relative address under an opened private directory, avoiding
+macOS pathname-length limits. Every handoff has an explicit deadline.
+
+The common lifecycle state records the native owner token and command identities;
+logs remain at the existing per-instance log path, so `hack logs -f ` watches
+them. Processes do not appear in tmux. Interactive `hack session` behavior is
+unchanged. Stop uses the persisted backend even if configuration changes later.
+Both runners use a five-second TERM grace, KILL escalation, and a bounded
+confirmation window; zombies do not count as running. Unknown outcomes keep
+ownership and shared helper holders for another `hack down` attempt.
+
+A healthy matching definition can be adopted. A definition whose recorded command
+has exited is replaced as a whole, including live siblings, matching the mux
+controller's healthy-window requirement. Adoption records a new cleanup generation
+without changing the native token, so an earlier start's rollback cannot stop
+adopted work. CLI publication and native effects share an instance lock; the common
+state file has a separate project-wide writer lock and atomic private publication.
+
+A failed incremental handoff is reconciled with native status. If the requested
+command became live after the reply was lost, its verified receipt is published
+and the failure remains explicit. It is never launched a second time. A new owner
+is retried only after native status proves the earlier commands absent. A missing
+native root with no possible state is absent; unreadable, malformed or unverifiable
+state remains unknown. Native status reports live and unknown names using kernel
+identity, not a persisted `running` string. Doctor and host status use those fields;
+missing tools and unknown identities cannot authorize repair or state removal.
+
+Sending TERM to the project owner invokes its owned rollback path; this is not a
+finite protocol request and has no response frame. Use the finite `stop` operation
+when a confirmation receipt is needed. Logs currently have no automatic rotation.
+
+The CLI keeps native receipts, owner capabilities and `control.sock` in the
+host's Hack home under `lifecycle-supervisors/`, outside the
+checkout. The shared log stays in the existing project log location. Do not bind
+mount the host Hack home into containers. A Hack home inside the checkout is
+refused for this runner. Linux incremental launch peers must have the same UID
+and PID, user and mount namespaces; macOS peers must have the same UID (Compose
+runs in a separate VM). Same-user host code is trusted. Earlier experimental
+checkout-local owners must be stopped with their original CLI before upgrading;
+leftover native roots are never treated as proof of absence.
+
+Doctor holds the same instance controller lock as `up` throughout final
+inspection, generation validation, native stop and conditional state removal.
+Adoption cannot interleave between validation and the stop effect.
+
+Before creating ownership or running hooks, Hack checks the sibling executable's
+permissions and exact version/capability handshake. Compiled installations ignore
+`HACK_LIFECYCLE_SUPERVISOR_BINARY`; that override is for source development and
+tests only. `hack doctor` reports the resolved executable and its version.
+A missing or incompatible executable fails before publication. An entry durably
+marked as never having attempted launch, with no recorded PID or process, can be
+retired by `down` or doctor without that executable. An interrupted launch with
+an uncertain result retains ownership.
+
+The native root is keyed on the checkout's real path and its absolute location
+is recorded with ownership. Changing HOME or HACK_HOME does not relocate a
+running instance. Losing that recorded directory after a launch attempt is an
+unconfirmed outcome, never proof of stopped processes; down and doctor retain
+ownership. Restore access to the recorded host state before retrying retirement.
+
+Run `hack down` before downgrading or switching CLI builds. Older CLIs cannot
+preserve supervisor entries when rewriting shared project state and may start
+mux duplicates. After the project owner is killed, existing guards keep working
+and `down` can recover them; adding a new persistent hook may refuse until `down`
+retires the old owner. Stop and retry rather than removing its socket or state.
+
+Adoption requires a verified live command in the `running` phase. Commands in
+`stopping` or `kill-pending` finish retirement before replacement; they are never
+adopted as healthy. Native status waits for an existing launch/stop admission
+lock before reading receipts, so an in-flight stop cannot expose a pre-TERM
+snapshot as adoptable work. This observation creates no lock file in an unused
+root and times out conservatively if admission cannot settle.
## Resource and qualification boundaries
A live direct child proves its guard is needed without a full-system census.
@@ -267,19 +366,26 @@ startup descriptor leases or signal errors. Some safety predicates hold by
construction. Native regressions, rather than that model, are the evidence for
those boundaries; see the model coverage document for the reduced claims.
-Stop durably retires its owner generation before confirming teardown, even if no
-command receipt exists. A delayed run checks that fence under the admission lock
-before spawning. New generations must use fresh owner tokens; markers persist in
-the private root and contain no command environment. Retired-but-live work remains
-unconfirmed. Linux guard spawning uses `/proc/self/exe`, preserving an executable
-reference even after atomic helper replacement.
+Stop durably retires the owner generation before confirming teardown, even when
+no command receipt exists yet. A delayed `run`, `serve` or `add` checks that fence
+under the same admission lock before spawning. A new run after a completed stop
+must use a fresh owner token; adoption keeps its existing token. Retirement
+markers persist in the private native root and contain no command environment.
+Status of retired-but-live work is unconfirmed, so it cannot be adopted.
+
+On Linux, guard spawning uses the kernel's executable reference under
+`/proc/self/exe`, which remains usable when an installer atomically replaces the
+running helper. A rebuild between the first command and a later add therefore
+does not depend on the deleted executable pathname.
Linux ownership checks read the process's effective UID from `/proc//status`.
They do not infer UID from proc-file ownership, which changes when a command
disables dumpability. Such a command remains recoverable after guard loss.
-Restart policy and restart backoff belong to the CLI integration in #811, not
-this binary's command ownership loop. This crate does not restart commands.
+This crate does not restart commands, and the CLI opt-in does not add a restart
+policy either: a persistent command that exits stays down under both runners
+until `hack restart` or `hack up`. Automatic restart with crash-loop backoff is
+deferred to a later slice and would belong to the CLI, not this ownership loop.
Logs are append-only and have no automatic rotation. Stop all generations and
confirm native absence before archiving the private root/log and starting with
@@ -289,11 +395,14 @@ arbitrarily delayed frames. Retire the entire quiescent root only after all old
callers and guards have exited. These files have no automatic size/count bound;
long-running installations must arrange a maintenance window and offline rotation.
-Native crash tests currently cover prepublication command-lock blocking, incomplete
-guard launch frames, and blocked ready delivery. Deterministic tests at the two
-smaller instruction windows (starting-state save to spawn, and spawn to running-state
-save) remain unqualified; polling a receipt cannot reliably stop a process in either
-window. They require a dedicated synchronization seam, not a timing-only test.
+Native crash tests cover prepublication command-lock blocking, incomplete launch
+frames, and blocked ready delivery. The opt-in integration also adds deterministic
+SIGKILL tests at both smaller instruction windows: after saving starting but before
+spawn, and after spawn but before saving running. Only a build with the test-hooks
+feature contains these checkpoints; normal and release builds omit them. Before
+spawn the session is absent; after an unrecorded spawn it is unknown and stop sends
+no signal without a birth witness. CI runs the two checkpoint tests separately from
+the default suite and builds the release executable without this feature.
## Manual recovery after an unknown stop
diff --git a/docs/lifecycle.md b/docs/lifecycle.md
index 769b34f44..842412dfd 100644
--- a/docs/lifecycle.md
+++ b/docs/lifecycle.md
@@ -4,9 +4,8 @@ Many projects need host-side setup around `hack up` (auth, local proxies, tunnel
Lifecycle is configured in `.hack/hack.config.json` under `lifecycle`.
-The [standalone Rust supervisor contract](./lifecycle-supervisor.md) describes
-the separately tested supervisor work. It does not change the current mux runner
-or expose a new configuration setting in this slice.
+The [Rust supervisor contract](./lifecycle-supervisor.md) describes the experimental
+`lifecycle.runner: "supervisor"` opt-in. Omission keeps the current mux runner.
With the Compose backend, the first Ctrl-C during foreground `hack up` asks
Compose to stop the project containers. Hack waits for Compose to finish stopping
@@ -688,3 +687,46 @@ Notes:
`--no-interactive` convention used across the CLI (documented defaults or `E_INTERACTIVE_REQUIRED`).
It does not change hook `stdin` behavior — a hook that itself blocks on interactive auth still
blocks under `--no-interactive`.
+
+## Experimental built-in runner
+
+Set `lifecycle.runner` to `"supervisor"` to use the built-in Rust host-process
+owner; the default remains `"mux"`. The supervisor executable must be available
+beside Hack. The [supervisor contract](./lifecycle-supervisor.md) records delivery
+and platform qualification. This opt-in does not provide a PTY: stdin is null,
+stdout/stderr are pipes, and commands requiring a terminal must use a
+noninteractive mode or an interactive workspace.
+
+Supervisor lifecycle processes do not appear as tmux sessions. Watch their output
+with `hack logs -f `. Stop and restart use the recorded owner even after the
+runner setting changes. Neither runner restarts a persistent command that exits or
+crashes: it stays down, its exit is recorded, and status reports it as not live.
+Run `hack restart` (or `hack up`) to start it again. Automatic restart with backoff
+is not part of this opt-in. Unknown shutdown retains the ownership record and any
+shared helper holders, and fails so `hack down` can be retried. Shared helper
+commands themselves keep their existing ownership contract in this unit.
+
+### Refused ownership state
+
+For projects that have never published supervisor ownership, an empty or
+unparsable legacy `.hack/.internal/lifecycle/state.json` is moved to a private
+`state.json.legacy--` sibling, with one explanation, and treated as
+empty. This recovers interrupted non-atomic writes from older mux versions.
+After a project has used the supervisor, the durable `supervisor-state` marker
+disables that fallback: losing ownership may hide running native commands. With
+that marker present, a missing `state.json` is also refused rather than read as
+"nothing running", so `hack down`, status and runner changes cannot skip native
+retirement. Removing entries rewrites the file; Hack never deletes it.
+Unsafe files (including symlinks, hard links and foreign ownership) and
+semantically invalid records still refuse under both runners. State mutations
+now serialize on `state.lock` and publish atomically with mode 0600.
+Global project listings continue and mark the affected project's
+lifecycle as `state_unreadable`; this is unknown, not stopped.
+
+To recover, inspect that project's state file and the processes or sessions it
+records. Confirm identity before stopping any leftover work; a PID alone may have
+been reused. Preserve the file for diagnosis. Once the recorded work is confirmed
+stopped, seek an ownership-state repair; do not hand-edit or discard ownership
+records while commands may still run. A
+symlink, foreign owner, or unreadable directory needs its owner to repair the path
+before Hack can safely proceed. `hack doctor --fix` does not bypass these checks.
diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md
index 33423b881..88f5ec35e 100644
--- a/docs/reference/v5-model-coverage.md
+++ b/docs/reference/v5-model-coverage.md
@@ -37,6 +37,7 @@ must be attached to the eventual exact integration revision.
| Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification |
| --- | --- | --- | --- |
| Standalone lifecycle supervisor: durable pre-launch identity, retained guard lock, project-supervisor loss, separately lost guard, verified stop and unknown stop | [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/COVERAGE.md) | `packages/lifecycle-supervisor` and its fake-caller contract tests; no CLI integration or default change in this slice | One command name, at most two starts, matching/foreign/unknown/reused-SID identities, arbitrary process exit and supervisor/guard loss; 426 states. Negative control removes the live-session retry fence and violates `NoDuplicate`. Native PID identity, filesystem synchronization, descriptors, output and macOS effects require implementation tests. |
+| Opt-in lifecycle controller: pending ownership before launch, stable native owner across adoption, cleanup epochs, publication after a lost reply and serialized add/stop | [lifecycle-controller](../../tests/models/tla/lifecycle-controller/COVERAGE.md) | `project-lifecycle-supervisor.ts`, `lifecycle-supervisor.ts`, common lifecycle state publication and controller/incremental regressions | One instance, one creator plus one adopter, one initial launch plus one incremental add; 21 states. Negative controls remove the cleanup epoch fence (`NoAdoptedStop`) and pending publication fence (`DurableBeforeLaunch`). Atomic lock regions and trustworthy native readback are assumptions; process identity, sockets, cancellation latency and OS effects remain implementation tests. |
| Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 74 states, two steps per phase, deferred Compose repair, separate publication and receipt, safe pre-removal cleanup refusal; unsafe controls fail `NoSameRunCleanup` and `FreshRunProof` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, admits later explicit Compose repair without republishing env layers, and requires a new successful project run plus fresh snapshot/delivery/live-input proofs for cleanup. Real-file tests check preview parity, sentinels, backup damage, consent, tighter modes, refused cleanup retry and rollback; owned child processes receive SIGKILL at initial and later journal/publication boundaries | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. |
| Interactive exec write-ahead admission, crash, EOF, replacement boot and retirement | interactive-session (459 states); no-replay, EOF-retirement, failed-inspect and wrong-boot negative controls | `interactive_session.rs` atomic intent/exec records, exact stopped-boot proof, known-undispatched withdrawal, restore-before-busy-retirement and run-scoped cleanup-report tests, and lock-free observation/revalidation tests; Engine effect fences; authored owner start and SCM_RIGHTS handoff | One session, two boot identities, two possible starts. Atomic durable publication and exact identity/absence proofs are assumptions; real fsync/power loss, real owner/Engine handoff and VM loss need Mac acceptance; receipt lock timing and terminal restoration are regression-covered, not modeled. Validated exact exec exit permits retirement; it makes no descendant cleanup claim. |
| Private interactive envelope, echo state and bytes still in flight on failure | interactive-envelope (15 states); restore-on-error negative control | HKEE2 launcher and host gate, Linux PTY seven-case matrix, bounded marker/ACK and coalesced-output tests | Two abstract private bytes, one handshake. PTY fixtures prove local termios ordering; Docker detach scanning, real guest wiring, same-user access and VM effects remain separate. Source/compiled Bun PTY and native-process fixtures cover lifetime/terminal boundaries, not model liveness. |
@@ -85,6 +86,7 @@ liveness/reachability controls have their own documented assumptions.
| Model | Positive distinct states | Configurations |
| --- | ---: | ---: |
| [lifecycle-supervisor](../../tests/models/tla/lifecycle-supervisor/LifecycleSupervisor.tla) | 426 | 4 |
+| [lifecycle-controller](../../tests/models/tla/lifecycle-controller/LifecycleController.tla) | 21 | 2 |
| [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 |
| [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 |
| [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 |
diff --git a/docs/versions/v5/getting-started.md b/docs/versions/v5/getting-started.md
index 131d16b4a..94a3a928e 100644
--- a/docs/versions/v5/getting-started.md
+++ b/docs/versions/v5/getting-started.md
@@ -118,6 +118,13 @@ routing services. See [Run and inspect](guides/run-project.md) for service selec
log history and restarts. When this Compose run works, try native only on a
[disposable project](native/start.md).
+Lifecycle host processes can opt into `lifecycle.runner: "supervisor"` when the
+Rust supervisor executable is installed alongside the CLI. The default stays
+`"mux"`. This mode has no terminal: use noninteractive commands and
+`hack logs -f ` to watch output. See the
+[lifecycle runner implementation contract](https://github.com/hack-dance/hack/blob/next/docs/lifecycle-supervisor.md) for prerequisites and
+recovery behavior.
+
Project JSON rows and hackd project snapshots share `backend`, `state`,
`started_at` and per-service readiness, completion, health and exit fields.
Unavailable evidence stays unknown/null. A zero exit is a completed job only
diff --git a/packages/lifecycle-supervisor/Cargo.toml b/packages/lifecycle-supervisor/Cargo.toml
index 1fdb83721..fcd872576 100644
--- a/packages/lifecycle-supervisor/Cargo.toml
+++ b/packages/lifecycle-supervisor/Cargo.toml
@@ -5,6 +5,10 @@ edition = "2024"
rust-version = "1.97.1"
publish = false
+[features]
+# Deterministic crash checkpoints for native qualification; never enable in shipped builds.
+test-hooks = []
+
[dependencies]
libc = "0.2"
serde = { version = "1.0", features = ["derive"] }
diff --git a/packages/lifecycle-supervisor/README.md b/packages/lifecycle-supervisor/README.md
index fce258562..9d3e5bcf0 100644
--- a/packages/lifecycle-supervisor/README.md
+++ b/packages/lifecycle-supervisor/README.md
@@ -5,8 +5,10 @@ owner for Linux and macOS. It has no dependency on runtime-core, a terminal
multiplexer, Python, or a shell wrapper. Authored shell commands can still
explicitly select a shell in their argv.
-The first slice is exercised by a fake caller. It does not select a new CLI runner
-or install the executable. See [the protocol, state model and delivery gates](../../docs/lifecycle-supervisor.md).
+The CLI can select this owner with `lifecycle.runner: "supervisor"`; the default
+remains `"mux"`. Fake-caller contracts and the real CLI integration have separate
+regressions. Distribution packaging remains a separate change. See
+[the protocol, state model and delivery gates](../../docs/lifecycle-supervisor.md).
Build and verify from the repository root with the pinned toolchain:
@@ -23,9 +25,11 @@ processes and cleans them up. CI runs the same crate on Linux x86_64, macOS arm6
and macOS x86_64. A cross-target `cargo check` checks compilation only; it does not
qualify platform signal, process identity, output or recovery behavior.
-`run` must be spawned as a fresh child without the caller making it a process-group
+`run` and `serve` must be spawned as fresh children without the caller making them process-group
leader. The supervisor calls `setsid` itself and detaches its standard descriptors
-after the framed ready receipt. `stop` and `status` are finite framed requests.
+after the framed ready receipt. `serve` accepts later `add` requests through a
+private local socket, preserving persistent-hook ordering. `add`, `stop`, and
+`status` are finite framed requests.
Frames, not pipe closure, delimit delivery. The per-command guard publishes kernel
identity before executing authored argv and retains ownership after the project
supervisor dies. A killed guard with surviving session members prevents a fresh
@@ -89,7 +93,11 @@ never ordered across processes. A reused numeric SID alone never authorizes a si
Terminal receipts prove the old guard completed cleanup; uncertain legacy or
newer-member-only sessions remain unknown without effects. See
[resource and qualification boundaries](../../docs/lifecycle-supervisor.md#resource-and-qualification-boundaries)
-for offline rotation, Darwin different-UID limits and remaining crash-test gaps.
+for offline rotation, Darwin different-UID limits and crash-window qualification.
The ignored idle measurement has a native Darwin variant using proc_pid_rusage
over 60 seconds; its kernel idle/interrupt wakeups differ from Linux context switches.
Descriptor-limit and functional prepublication lease tests run on both platforms.
+
+Deterministic crash checkpoints are available only with the test-hooks feature.
+Run cargo test --features test-hooks guard_killed_after_ to exercise both save/spawn
+windows. Never enable this feature in shipped/default/release artifacts.
diff --git a/packages/lifecycle-supervisor/src/incremental.rs b/packages/lifecycle-supervisor/src/incremental.rs
new file mode 100644
index 000000000..20d800718
--- /dev/null
+++ b/packages/lifecycle-supervisor/src/incremental.rs
@@ -0,0 +1,374 @@
+//! Incremental launch transport. Only fresh internal processes change cwd, using
+//! the opened private root as the Unix socket namespace. Frames and deadlines,
+//! rather than descriptor closure, delimit every request and reply.
+use crate::{
+ Result, os,
+ protocol::{self, Launch},
+ refused,
+ storage::Root,
+ supervisor::{self, Receipt},
+};
+use std::{
+ fs,
+ os::fd::{AsRawFd, FromRawFd, OwnedFd},
+ os::unix::{
+ fs::{FileTypeExt, MetadataExt, PermissionsExt},
+ net::{UnixListener, UnixStream},
+ },
+ process::Child,
+ sync::atomic::Ordering,
+ thread,
+ time::Duration,
+};
+
+const SOCKET: &str = "control.sock";
+
+/// Socket permissions alone do not distinguish a same-UID container peer.
+#[cfg(target_os = "linux")]
+fn verify_peer(stream: &UnixStream) -> Result<()> {
+ use std::os::unix::fs::MetadataExt;
+ // SAFETY: getsockopt writes at most the supplied live ucred buffer.
+ let credentials = unsafe {
+ let mut credentials: libc::ucred = std::mem::zeroed();
+ let mut length = std::mem::size_of_val(&credentials) as libc::socklen_t;
+ if libc::getsockopt(
+ stream.as_raw_fd(),
+ libc::SOL_SOCKET,
+ libc::SO_PEERCRED,
+ (&mut credentials as *mut libc::ucred).cast(),
+ &mut length,
+ ) != 0
+ || length as usize != std::mem::size_of_val(&credentials)
+ {
+ return Err(refused());
+ }
+ credentials
+ };
+ // SAFETY: geteuid has no preconditions.
+ if credentials.uid != unsafe { libc::geteuid() } || credentials.pid <= 1 {
+ return Err(refused());
+ }
+ for namespace in ["pid", "user", "mnt"] {
+ let own = fs::metadata(format!("/proc/self/ns/{namespace}"))?;
+ let peer = fs::metadata(format!("/proc/{}/ns/{namespace}", credentials.pid))?;
+ if own.dev() != peer.dev() || own.ino() != peer.ino() {
+ return Err(refused());
+ }
+ }
+ Ok(())
+}
+
+#[cfg(target_os = "macos")]
+fn verify_peer(stream: &UnixStream) -> Result<()> {
+ let mut uid = 0;
+ let mut gid = 0;
+ // SAFETY: getpeereid writes two live scalar buffers; the descriptor is live.
+ if unsafe { libc::getpeereid(stream.as_raw_fd(), &mut uid, &mut gid) } != 0
+ || uid != unsafe { libc::geteuid() }
+ {
+ return Err(refused());
+ }
+ Ok(())
+}
+
+struct SocketOwner {
+ device: u64,
+ inode: u64,
+}
+
+impl Drop for SocketOwner {
+ fn drop(&mut self) {
+ if let Ok(metadata) = checked_socket()
+ && metadata.dev() == self.device
+ && metadata.ino() == self.inode
+ {
+ let _ = fs::remove_file(SOCKET);
+ }
+ }
+}
+
+fn checked_socket() -> Result {
+ let metadata = fs::symlink_metadata(SOCKET)?;
+ // SAFETY: geteuid only observes this process's effective owner.
+ if !metadata.file_type().is_socket()
+ || metadata.uid() != unsafe { libc::geteuid() }
+ || metadata.mode() & 0o777 != 0o600
+ {
+ return Err(refused());
+ }
+ Ok(metadata)
+}
+
+fn bind() -> Result<(UnixListener, SocketOwner)> {
+ match checked_socket() {
+ Ok(_) => fs::remove_file(SOCKET)?,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
+ Err(error) => return Err(error),
+ }
+ let listener = UnixListener::bind(SOCKET)?;
+ fs::set_permissions(SOCKET, fs::Permissions::from_mode(0o600))?;
+ let metadata = checked_socket()?;
+ listener.set_nonblocking(true)?;
+ Ok((
+ listener,
+ SocketOwner {
+ device: metadata.dev(),
+ inode: metadata.ino(),
+ },
+ ))
+}
+
+pub(crate) fn add(request: Launch) -> Result<()> {
+ supervisor::validate_launch(&request)?;
+ if request.processes.len() != 1 {
+ return Err(refused());
+ }
+ let root = Root::open(&request.root)?;
+ root.enter()?;
+ let metadata = checked_socket()?;
+ let mut stream = connect()?;
+ let current = checked_socket()?;
+ if current.dev() != metadata.dev() || current.ino() != metadata.ino() {
+ return Err(refused());
+ }
+ protocol::write(&mut stream, &request)?;
+ // The server may wait the full stop-lock bound before it launches, then
+ // needs one ordinary frame deadline for its own work. Giving up sooner
+ // would report failure for an add the server still completes.
+ let receipt: serde_json::Value = protocol::read_within(
+ &mut stream,
+ crate::supervisor::STOP_LOCK_WAIT + protocol::DEADLINE,
+ )?;
+ if receipt["version"] != 1 || !receipt["processes"].is_array() {
+ return Err(refused());
+ }
+ protocol::write(&mut std::io::stdout(), &receipt)?;
+ if receipt["outcome"] != "ready" {
+ return Err(refused());
+ }
+ Ok(())
+}
+
+fn connect() -> Result {
+ // Darwin lacks SOCK_CLOEXEC. Set both flags immediately after socket and
+ // use an explicit frame/deadline; no liveness relies on descriptor closure.
+ // SAFETY: socket returns a fresh owned descriptor with no retained pointer.
+ let raw = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) };
+ if raw < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ // SAFETY: the new descriptor has exactly one Rust owner.
+ let fd = unsafe { OwnedFd::from_raw_fd(raw) };
+ // SAFETY: integer descriptor flags are set on the live owned socket.
+ if unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) } < 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ os::nonblocking(&fd)?;
+ // SAFETY: zeroed sockaddr_un is initialized below and passed at exact size.
+ let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() };
+ address.sun_family = libc::AF_UNIX as libc::sa_family_t;
+ for (target, byte) in address.sun_path.iter_mut().zip(SOCKET.bytes()) {
+ *target = byte as libc::c_char;
+ }
+ #[cfg(target_os = "macos")]
+ {
+ address.sun_len = std::mem::size_of_val(&address) as u8;
+ }
+ // SAFETY: address is initialized, NUL-terminated and live for the supplied size.
+ if unsafe {
+ libc::connect(
+ fd.as_raw_fd(),
+ (&address as *const libc::sockaddr_un).cast(),
+ std::mem::size_of_val(&address) as libc::socklen_t,
+ )
+ } != 0
+ {
+ let error = std::io::Error::last_os_error();
+ if !matches!(
+ error.raw_os_error(),
+ Some(libc::EINPROGRESS) | Some(libc::EAGAIN)
+ ) {
+ return Err(error);
+ }
+ let mut poll = libc::pollfd {
+ fd: fd.as_raw_fd(),
+ events: libc::POLLOUT,
+ revents: 0,
+ };
+ // SAFETY: one initialized descriptor record is borrowed for the bounded wait.
+ if unsafe { libc::poll(&mut poll, 1, protocol::DEADLINE.as_millis() as i32) } <= 0 {
+ return Err(refused());
+ }
+ let mut result: libc::c_int = 0;
+ let mut length = std::mem::size_of_val(&result) as libc::socklen_t;
+ // SAFETY: getsockopt writes one live bounded integer and length.
+ if unsafe {
+ libc::getsockopt(
+ fd.as_raw_fd(),
+ libc::SOL_SOCKET,
+ libc::SO_ERROR,
+ (&mut result as *mut libc::c_int).cast(),
+ &mut length,
+ )
+ } < 0
+ {
+ return Err(std::io::Error::last_os_error());
+ }
+ if result != 0 {
+ return Err(std::io::Error::from_raw_os_error(result));
+ }
+ }
+ Ok(UnixStream::from(fd))
+}
+
+pub(crate) fn serve(request: Launch) -> Result<()> {
+ supervisor::validate_launch(&request)?;
+ let root = Root::open(&request.root)?;
+ let _instance_lock = root.lock_wait(".instance.lock", crate::supervisor::INSTANCE_LOCK_WAIT)?;
+ os::census_preflight()?;
+ let launch_lock = root.lock_wait(".stop.lock", crate::supervisor::STOP_LOCK_WAIT)?;
+ root.assert_not_retired(&request.owner_token)?;
+ for state in root.states()? {
+ if os::recorded_live(&state, false)? {
+ return Err(refused());
+ }
+ }
+ os::session()?;
+ os::signals()?;
+ root.enter()?;
+ let (listener, _socket_owner) = bind()?;
+ let mut children = Vec::new();
+ let outcome = (|| {
+ for process in request.processes {
+ supervisor::start_guard(
+ &request.root,
+ &request.owner_token,
+ request.log_path.as_deref(),
+ &launch_lock,
+ process,
+ &mut children,
+ )?;
+ }
+ protocol::write(
+ &mut std::io::stdout(),
+ &Receipt::new(
+ std::process::id(),
+ "ready",
+ root.states()?
+ .into_iter()
+ .filter(|s| s.owner_token == request.owner_token)
+ .collect(),
+ ),
+ )?;
+ drop(launch_lock);
+ supervisor::detach_stdio()?;
+ loop {
+ if os::STOP.load(Ordering::Relaxed) {
+ return Err(refused());
+ }
+ match listener.accept() {
+ Ok((mut stream, _)) => {
+ stream.set_nonblocking(true)?;
+ let mut response_started = false;
+ let added = handle_add(
+ &root,
+ &request.owner_token,
+ request.log_path.as_deref(),
+ &mut stream,
+ &mut children,
+ &mut response_started,
+ );
+ if added.is_err() && !response_started {
+ let _ = protocol::write(
+ &mut stream,
+ &Receipt::new(std::process::id(), "refused", Vec::new()),
+ );
+ }
+ }
+ Err(error)
+ if matches!(
+ error.kind(),
+ std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
+ ) => {}
+ Err(error) => return Err(error),
+ }
+ let mut live = false;
+ for child in &mut children {
+ if child.try_wait()?.is_none() {
+ live = true;
+ }
+ }
+ if !live {
+ return Ok(());
+ }
+ thread::sleep(Duration::from_millis(20));
+ }
+ })();
+ if outcome.is_err() {
+ supervisor::rollback(&mut children);
+ }
+ outcome
+}
+
+fn handle_add(
+ root: &Root,
+ owner: &str,
+ log_path: Option<&std::path::Path>,
+ stream: &mut UnixStream,
+ children: &mut Vec,
+ response_started: &mut bool,
+) -> Result<()> {
+ verify_peer(stream)?;
+ let request: Launch = protocol::read(stream)?;
+ supervisor::validate_launch(&request)?;
+ if request.root != root.path
+ || request.owner_token != owner
+ || request.processes.len() != 1
+ || request.log_path.as_deref() != log_path
+ {
+ return Err(refused());
+ }
+ let _stop_lock = root.lock_wait(".stop.lock", crate::supervisor::STOP_LOCK_WAIT)?;
+ root.assert_not_retired(owner)?;
+ let process = request.processes.into_iter().next().ok_or_else(refused)?;
+ let states = root.states()?;
+ let mut live = 0;
+ for state in &states {
+ let present = os::recorded_live(state, false)?;
+ if present {
+ if state.owner_token != owner
+ || matches!(state.phase.as_str(), "stopping" | "kill-pending")
+ || matches!(os::observe_state(state), os::Observation::Unknown)
+ {
+ return Err(refused());
+ }
+ live += 1;
+ }
+ if state.name.eq_ignore_ascii_case(&process.name) && (present || state.name != process.name)
+ {
+ return Err(refused());
+ }
+ }
+ // A stop that completed while add waited closes this owner's admission.
+ // A later hook may start a fresh owner once its instance lock is released.
+ if live == 0 || live >= 128 {
+ return Err(refused());
+ }
+ let before = children.len();
+ let started =
+ supervisor::start_guard(&root.path, owner, log_path, &_stop_lock, process, children);
+ if started.is_err() {
+ supervisor::rollback(&mut children[before..]);
+ return Err(refused());
+ }
+ let receipt = Receipt::new(
+ std::process::id(),
+ "ready",
+ root.states()?
+ .into_iter()
+ .filter(|state| state.owner_token == owner)
+ .collect(),
+ );
+ *response_started = true;
+ protocol::write(stream, &receipt)
+}
diff --git a/packages/lifecycle-supervisor/src/main.rs b/packages/lifecycle-supervisor/src/main.rs
index 8799f6ed9..247021eb4 100644
--- a/packages/lifecycle-supervisor/src/main.rs
+++ b/packages/lifecycle-supervisor/src/main.rs
@@ -22,12 +22,17 @@
//! leader. Killing that guard separately makes health unknown; stop can recover
//! the recorded session only after boot, identity and member-incarnation checks.
+mod incremental;
mod os;
mod protocol;
mod storage;
mod supervisor;
fn main() {
+ if std::env::args().nth(1).as_deref() == Some("--version") {
+ println!("hack-lifecycle-supervisor 0.1.0 protocol=1 incremental=1");
+ return;
+ }
if supervisor::main().is_err() {
// Inputs can contain credentials. Never include parser or OS error text.
eprintln!("Lifecycle supervisor refused or failed; values omitted.");
diff --git a/packages/lifecycle-supervisor/src/os.rs b/packages/lifecycle-supervisor/src/os.rs
index 2d1c3f26d..4fc71b969 100644
--- a/packages/lifecycle-supervisor/src/os.rs
+++ b/packages/lifecycle-supervisor/src/os.rs
@@ -478,13 +478,18 @@ pub fn session_signal(session: i32, signal: i32) -> Result<()> {
.filter(|p| p.group != session)
.map(|p| p.group)
.collect();
+ let mut first_error = None;
for member in members.iter().filter(|p| p.group == session) {
- signal_member(member, signal)?;
+ if let Err(error) = signal_member(member, signal) {
+ first_error.get_or_insert(error);
+ }
}
for group in groups {
- group_signal(group, signal)?;
+ if let Err(error) = group_signal(group, signal) {
+ first_error.get_or_insert(error);
+ }
}
- Ok(())
+ first_error.map_or(Ok(()), Err)
}
/// Open a pidfd for `pid`, or `None` when no process with that thread-group
@@ -1003,6 +1008,7 @@ mod tests {
let state = crate::storage::State {
version: 1,
name: "worker".into(),
+ display_name: None,
owner_token: "synthetic".into(),
guard_pid: 44,
process_group_id: 44,
diff --git a/packages/lifecycle-supervisor/src/protocol.rs b/packages/lifecycle-supervisor/src/protocol.rs
index f12c63fbd..c76a92d43 100644
--- a/packages/lifecycle-supervisor/src/protocol.rs
+++ b/packages/lifecycle-supervisor/src/protocol.rs
@@ -10,6 +10,11 @@ use std::{
pub const LIMIT: usize = 1024 * 1024;
pub const DEADLINE: Duration = Duration::from_secs(10);
+static STDOUT_STARTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
+pub fn stdout_started() -> bool {
+ STDOUT_STARTED.load(std::sync::atomic::Ordering::Relaxed)
+}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@@ -18,6 +23,8 @@ pub struct Launch {
pub root: PathBuf,
pub owner_token: String,
pub processes: Vec,
+ #[serde(default)]
+ pub log_path: Option,
}
#[derive(Deserialize, Serialize)]
@@ -27,6 +34,12 @@ pub struct Process {
pub argv: Vec,
pub cwd: PathBuf,
pub env: BTreeMap,
+ #[serde(
+ rename = "displayName",
+ default,
+ skip_serializing_if = "Option::is_none"
+ )]
+ pub display_name: Option,
}
#[derive(Deserialize, Serialize)]
@@ -44,6 +57,8 @@ pub struct GuardLaunch {
pub root: PathBuf,
pub owner_token: String,
pub process: Process,
+ #[serde(default)]
+ pub log_path: Option,
}
pub fn valid_name(name: &str) -> bool {
@@ -71,6 +86,13 @@ pub fn validate_control(version: u32, root: &std::path::Path, token: &str) -> Re
impl Process {
pub fn validate(&self) -> Result<()> {
+ if self
+ .display_name
+ .as_ref()
+ .is_some_and(|name| name.trim().is_empty() || name.len() > 4096 || name.contains('\0'))
+ {
+ return Err(refused());
+ }
if !valid_name(&self.name)
|| !self.cwd.is_absolute()
|| self.argv.is_empty()
@@ -89,7 +111,16 @@ impl Process {
/// Length and deadline, never descriptor closure, delimit a handoff. Polling
/// handles Darwin's brief inheritable-descriptor window without waiting on EOF.
pub fn read(input: &mut (impl Read + AsRawFd)) -> Result {
- let deadline = Instant::now() + DEADLINE;
+ read_within(input, DEADLINE)
+}
+
+/// `read` with an explicit deadline, for a reply that follows a bounded wait
+/// on the other side (for example a stop-lock wait before an incremental add).
+pub fn read_within(
+ input: &mut (impl Read + AsRawFd),
+ within: Duration,
+) -> Result {
+ let deadline = Instant::now() + within;
let mut header = [0; 4];
exact(input, &mut header, deadline)?;
let length = u32::from_be_bytes(header) as usize;
@@ -161,6 +192,9 @@ pub fn write(output: &mut (impl Write + AsRawFd), value: &impl Serialize) -> Res
return Err(refused());
}
let fd = output.as_raw_fd();
+ if fd == libc::STDOUT_FILENO {
+ STDOUT_STARTED.store(true, std::sync::atomic::Ordering::Relaxed);
+ }
// SAFETY: fcntl borrows this live, exclusively used protocol descriptor.
let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
if flags < 0 {
diff --git a/packages/lifecycle-supervisor/src/storage.rs b/packages/lifecycle-supervisor/src/storage.rs
index 3108dfa1b..f49364c4d 100644
--- a/packages/lifecycle-supervisor/src/storage.rs
+++ b/packages/lifecycle-supervisor/src/storage.rs
@@ -15,6 +15,8 @@ use std::{
pub struct State {
pub version: u32,
pub name: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub display_name: Option,
pub owner_token: String,
pub guard_pid: i32,
pub process_group_id: i32,
@@ -40,7 +42,8 @@ pub struct Root {
impl Root {
fn retirement_key(token: &str) -> String {
- // Stable non-authentication index. A collision can only refuse launch.
+ // Stable non-authentication index. A hash collision only refuses a
+ // launch; it can never permit a retired generation to run again.
let hash = token.bytes().fold(0xcbf29ce484222325_u64, |hash, byte| {
(hash ^ u64::from(byte)).wrapping_mul(0x100000001b3)
});
@@ -60,6 +63,15 @@ impl Root {
}
}
+ /// Fresh internal binaries may use a descriptor-relative cwd to keep Unix
+ /// socket addresses below Darwin's 104-byte limit. Caller cwd is untouched.
+ pub fn enter(&self) -> Result<()> {
+ // SAFETY: the owned descriptor refers to the admitted private directory.
+ if unsafe { libc::fchdir(self.directory.as_raw_fd()) } != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ Ok(())
+ }
pub fn open(path: &Path) -> Result {
let directory = OpenOptions::new()
.read(true)
@@ -129,6 +141,23 @@ impl Root {
pub fn lock_wait(&self, name: &str, wait: std::time::Duration) -> Result {
let file = self.file(name, libc::O_RDWR | libc::O_CREAT)?;
+ Self::wait_for_lock(file, wait)
+ }
+
+ pub fn existing_lock_wait(
+ &self,
+ name: &str,
+ wait: std::time::Duration,
+ ) -> Result> {
+ let file = match self.file(name, libc::O_RDWR) {
+ Ok(file) => file,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+ Err(error) => return Err(error),
+ };
+ Self::wait_for_lock(file, wait).map(Some)
+ }
+
+ fn wait_for_lock(file: File, wait: std::time::Duration) -> Result {
let deadline = std::time::Instant::now() + wait;
loop {
// SAFETY: flock only borrows the live owned descriptor.
@@ -153,6 +182,12 @@ impl Root {
}
pub fn save(&self, state: &State) -> Result<()> {
+ let bytes = serde_json::to_vec(state).map_err(|_| refused())?;
+ // Match the bounded read contract before publishing or spawning. Human
+ // labels can expand when JSON escapes control characters.
+ if bytes.len() > 8192 {
+ return Err(refused());
+ }
let final_name = format!("{}.json", state.name);
// Refuse unsafe pre-existing targets before atomically replacing an owned
// regular receipt. Never truncate through a link.
@@ -184,7 +219,6 @@ impl Root {
Err(error) => return Err(error),
}
};
- let bytes = serde_json::to_vec(state).map_err(|_| refused())?;
temporary.write_all(&bytes)?;
temporary.sync_all()?;
let temporary_name = std::ffi::CString::new(temporary_name).map_err(|_| refused())?;
@@ -261,6 +295,7 @@ mod tests {
State {
version: 1,
name: "worker".into(),
+ display_name: None,
owner_token: "synthetic".into(),
guard_pid: 44,
process_group_id: 44,
diff --git a/packages/lifecycle-supervisor/src/supervisor.rs b/packages/lifecycle-supervisor/src/supervisor.rs
index 075979dac..56564fa8a 100644
--- a/packages/lifecycle-supervisor/src/supervisor.rs
+++ b/packages/lifecycle-supervisor/src/supervisor.rs
@@ -28,15 +28,17 @@ const CONFIRM_BOUND: Duration = Duration::from_secs(2);
/// waiter normally gets its own outcome. Admission, census and signalling run
/// outside the timers, so it is not a hard bound: an expired wait returns an
/// `unknown` receipt with reason `stop-lock-wait-expired` and acts on nothing.
-const STOP_LOCK_WAIT: Duration =
+pub(crate) const STOP_LOCK_WAIT: Duration =
Duration::from_secs(2 * (GRACE.as_secs() + CONFIRM_BOUND.as_secs()) + 2);
/// Bounded wait for a previous command owner to release its lock.
const COMMAND_LOCK_WAIT: Duration = Duration::from_secs(10);
+/// Bounded wait for another project supervisor to release the instance lock.
+pub(crate) const INSTANCE_LOCK_WAIT: Duration = Duration::from_secs(10);
const LINE_LIMIT: usize = 64 * 1024;
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
-struct Receipt {
+pub(crate) struct Receipt {
version: u32,
supervisor_pid: u32,
outcome: &'static str,
@@ -49,14 +51,16 @@ struct Receipt {
}
impl Receipt {
- fn new(supervisor_pid: u32, outcome: &'static str, processes: Vec) -> Self {
+ pub(crate) fn new(supervisor_pid: u32, outcome: &'static str, processes: Vec) -> Self {
let mut live_process_names = Vec::new();
let mut unknown_process_names = Vec::new();
for state in &processes {
match os::observe_state(state) {
- os::Observation::Live => live_process_names.push(state.name.clone()),
+ os::Observation::Live => live_process_names
+ .push(state.display_name.as_ref().unwrap_or(&state.name).clone()),
+ os::Observation::Unknown => unknown_process_names
+ .push(state.display_name.as_ref().unwrap_or(&state.name).clone()),
os::Observation::Absent => (),
- os::Observation::Unknown => unknown_process_names.push(state.name.clone()),
}
}
Self {
@@ -143,7 +147,16 @@ fn control_effects(
}
}
} else {
- None
+ // Status waits for a stop in progress, but never creates the lock file.
+ match root.existing_lock_wait(".stop.lock", STOP_LOCK_WAIT) {
+ Ok(lock) => lock,
+ Err(error) => {
+ if error.kind() == std::io::ErrorKind::TimedOut {
+ *reason = "stop-lock-wait-expired";
+ }
+ return Err(error);
+ }
+ }
};
let mut states = Vec::new();
*reason = "receipt-read-failed";
@@ -178,6 +191,17 @@ fn control_effects(
}
pub fn main() -> Result<()> {
+ let result = dispatch();
+ if result.is_err() && !protocol::stdout_started() {
+ let _ = protocol::write(
+ &mut std::io::stdout(),
+ &Receipt::new(0, "refused", Vec::new()),
+ );
+ }
+ result
+}
+
+fn dispatch() -> Result<()> {
os::close_inherited()?;
let args: Vec<_> = std::env::args_os().skip(1).collect();
if args.len() != 1 {
@@ -185,14 +209,23 @@ pub fn main() -> Result<()> {
}
match args[0].to_str() {
Some("run") => run(protocol::read(&mut std::io::stdin())?),
+ Some("serve") => crate::incremental::serve(protocol::read(&mut std::io::stdin())?),
+ Some("add") => crate::incremental::add(protocol::read(&mut std::io::stdin())?),
Some("guard") => guard(protocol::read(&mut std::io::stdin())?),
Some(action @ ("stop" | "status")) => control(action),
_ => Err(refused()),
}
}
-fn run(request: Launch) -> Result<()> {
+pub(crate) fn validate_launch(request: &Launch) -> Result<()> {
protocol::validate_control(request.version, &request.root, &request.owner_token)?;
+ if request
+ .log_path
+ .as_ref()
+ .is_some_and(|path| !path.is_absolute() || path.file_name().is_none())
+ {
+ return Err(refused());
+ }
if request.processes.is_empty() || request.processes.len() > 128 {
return Err(refused());
}
@@ -203,8 +236,16 @@ fn run(request: Launch) -> Result<()> {
return Err(refused());
}
}
+ Ok(())
+}
+
+fn run(request: Launch) -> Result<()> {
+ validate_launch(&request)?;
let root = Root::open(&request.root)?;
- let _instance_lock = instance_lock(&root)?;
+ let _instance_lock = match root.lock(".instance.lock") {
+ Ok(lock) => lock,
+ Err(_) => root.lock_wait(".instance.lock", INSTANCE_LOCK_WAIT)?,
+ };
os::census_preflight()?;
let launch_lock = root.lock_wait(".stop.lock", STOP_LOCK_WAIT)?;
root.assert_not_retired(&request.owner_token)?;
@@ -224,42 +265,14 @@ fn run(request: Launch) -> Result<()> {
if os::STOP.load(Ordering::Relaxed) {
return Err(refused());
}
- #[cfg(target_os = "linux")]
- let executable = std::path::PathBuf::from("/proc/self/exe");
- #[cfg(not(target_os = "linux"))]
- let executable = std::env::current_exe()?;
- let mut command = Command::new(executable);
- command
- .arg("guard")
- .env_clear()
- .stdin(Stdio::piped())
- .stdout(Stdio::piped())
- // Keep stop admission leased even if this supervisor dies
- // before the guard publishes its durable identity. fd 2 is
- // reserved until guard receipt delivery/detach (or exit).
- .stderr(Stdio::from(launch_lock.try_clone()?));
- // SAFETY: setsid is async-signal-safe, and this closure allocates no
- // application state. It runs in the freshly forked child only.
- unsafe {
- command.pre_exec(os::session);
- }
- let child = command.spawn()?;
- children.push(child);
- let child = children.last_mut().ok_or_else(refused)?;
- let plan = GuardLaunch {
- version: 1,
- root: request.root.clone(),
- owner_token: request.owner_token.clone(),
+ let state = start_guard(
+ &request.root,
+ &request.owner_token,
+ request.log_path.as_deref(),
+ &launch_lock,
process,
- };
- protocol::write(&mut child.stdin.take().ok_or_else(refused)?, &plan)?;
- let state: State = protocol::read(&mut child.stdout.take().ok_or_else(refused)?)?;
- if state.guard_pid != child.id() as i32 || state.owner_token != request.owner_token {
- return Err(refused());
- }
- if state.phase == "failed-start" {
- return Err(refused());
- }
+ &mut children,
+ )?;
states.push(state);
}
protocol::write(
@@ -298,32 +311,53 @@ fn run(request: Launch) -> Result<()> {
outcome
}
-fn instance_lock(root: &Root) -> Result {
- // Preserve the uncontended fast path; a bounded retry below distinguishes
- // contention from persistent file/OS failures with the actual errno.
- if let Ok(file) = root.lock(".instance.lock") {
- return Ok(file);
- }
- let file = root.file(".instance.lock", libc::O_RDWR | libc::O_CREAT)?;
- let deadline = Instant::now() + Duration::from_secs(1);
- loop {
- // SAFETY: file remains owned throughout this bounded retry.
- if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
- return Ok(file);
- }
- let error = std::io::Error::last_os_error();
- if !matches!(
- error.kind(),
- std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
- ) || Instant::now() >= deadline
- {
- return Err(error);
- }
- thread::sleep(TICK);
+pub(crate) fn start_guard(
+ root: &std::path::Path,
+ owner_token: &str,
+ log_path: Option<&std::path::Path>,
+ launch_lock: &File,
+ process: protocol::Process,
+ children: &mut Vec,
+) -> Result {
+ if os::STOP.load(Ordering::Relaxed) {
+ return Err(refused());
+ }
+ #[cfg(target_os = "linux")]
+ let executable = std::path::PathBuf::from("/proc/self/exe");
+ #[cfg(not(target_os = "linux"))]
+ let executable = std::env::current_exe()?;
+ let mut command = Command::new(executable);
+ command
+ .arg("guard")
+ .env_clear()
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::from(launch_lock.try_clone()?));
+ // SAFETY: setsid only uses async-signal-safe operations in the fresh child.
+ unsafe {
+ command.pre_exec(os::session);
}
+ children.push(command.spawn()?);
+ let child = children.last_mut().ok_or_else(refused)?;
+ let plan = GuardLaunch {
+ version: 1,
+ root: root.into(),
+ owner_token: owner_token.into(),
+ process,
+ log_path: log_path.map(std::path::Path::to_path_buf),
+ };
+ protocol::write(&mut child.stdin.take().ok_or_else(refused)?, &plan)?;
+ let state: State = protocol::read(&mut child.stdout.take().ok_or_else(refused)?)?;
+ if state.guard_pid != child.id() as i32
+ || state.owner_token != owner_token
+ || state.phase == "failed-start"
+ {
+ return Err(refused());
+ }
+ Ok(state)
}
-fn rollback(children: &mut [Child]) {
+pub(crate) fn rollback(children: &mut [Child]) {
for child in children.iter_mut() {
if child.try_wait().ok().flatten().is_some() {
continue;
@@ -371,6 +405,7 @@ fn guard(request: GuardLaunch) -> Result<()> {
let mut state = State {
version: 1,
name: request.process.name.clone(),
+ display_name: request.process.display_name.clone(),
owner_token: request.owner_token,
guard_pid: pid,
process_group_id: pid,
@@ -383,10 +418,25 @@ fn guard(request: GuardLaunch) -> Result<()> {
exit_code: None,
signal: None,
};
- let mut log = root.file(
- "lifecycle.log",
- libc::O_WRONLY | libc::O_APPEND | libc::O_CREAT,
- )?;
+ let mut log = if let Some(path) = request.log_path {
+ if !path.is_absolute() {
+ return Err(refused());
+ }
+ let parent = Root::open(path.parent().ok_or_else(refused)?)?;
+ let name = path
+ .file_name()
+ .and_then(|v| v.to_str())
+ .ok_or_else(refused)?;
+ if name == "." || name == ".." || name.contains('/') {
+ return Err(refused());
+ }
+ parent.file(name, libc::O_WRONLY | libc::O_APPEND | libc::O_CREAT)?
+ } else {
+ root.file(
+ "lifecycle.log",
+ libc::O_WRONLY | libc::O_APPEND | libc::O_CREAT,
+ )?
+ };
root.save(&state)?;
let outcome = guard_command(&request.process, &root, &mut state, &mut log);
if outcome.is_err() && state.command_pid.is_some() {
@@ -406,12 +456,33 @@ fn guard(request: GuardLaunch) -> Result<()> {
outcome
}
+/// Only qualification builds can pause at these instruction boundaries. The
+/// requested checkpoint is read from the explicit test process plan, never the
+/// guard's ambient environment. Shipped/default builds contain no checkpoint.
+#[cfg(feature = "test-hooks")]
+fn test_pause(process: &protocol::Process, checkpoint: &str) {
+ if process
+ .env
+ .get("SUPERVISOR_TEST_CHECKPOINT")
+ .map(String::as_str)
+ == Some(checkpoint)
+ {
+ // SAFETY: affects only this guard, which the fixture owns and kills.
+ unsafe {
+ libc::raise(libc::SIGSTOP);
+ }
+ }
+}
+
fn guard_command(
process: &protocol::Process,
root: &Root,
state: &mut State,
log: &mut File,
) -> Result<()> {
+ let log_name = sanitize_service(state.display_name.as_deref().unwrap_or(&state.name));
+ #[cfg(feature = "test-hooks")]
+ test_pause(process, "before-spawn");
let mut command = Command::new(&process.argv[0]);
command
.args(&process.argv[1..])
@@ -434,6 +505,8 @@ fn guard_command(
return Err(refused());
}
};
+ #[cfg(feature = "test-hooks")]
+ test_pause(process, "after-spawn");
state.command_pid = Some(child.id() as i32);
state.command_start_time = os::inspect(child.id() as i32)?.map(|identity| identity.start);
state.phase = "running".into();
@@ -457,10 +530,10 @@ fn guard_command(
loop {
// Alternate bounded reads so stdout bursts cannot starve stderr/stop.
if stdout_open {
- stdout_open = out.drain(&mut stdout, log, &state.name, "stdout")?;
+ stdout_open = out.drain(&mut stdout, log, &log_name, "stdout")?;
}
if stderr_open {
- stderr_open = err.drain(&mut stderr, log, &state.name, "stderr")?;
+ stderr_open = err.drain(&mut stderr, log, &log_name, "stderr")?;
}
let was_exited = direct_exited;
if !direct_exited && let Some(status) = child.try_wait()? {
@@ -498,10 +571,10 @@ fn guard_command(
// inherit a pipe on Darwin. Drain a bounded final interval instead.
if since.elapsed() >= Duration::from_millis(100) {
if !out.pending.is_empty() {
- out.flush(log, &state.name, "stdout", true)?;
+ out.flush(log, &log_name, "stdout", true)?;
}
if !err.pending.is_empty() {
- err.flush(log, &state.name, "stderr", true)?;
+ err.flush(log, &log_name, "stderr", true)?;
}
state.phase = if killed { "killed" } else { "exited" }.into();
root.save(state)?;
@@ -683,7 +756,7 @@ fn stop(states: &[State]) -> Result<()> {
}
}
-fn detach_stdio() -> Result<()> {
+pub(crate) fn detach_stdio() -> Result<()> {
let null = std::fs::OpenOptions::new()
.read(true)
.write(true)
@@ -776,9 +849,21 @@ fn incomplete_utf8_tail(bytes: &[u8]) -> usize {
0
}
+fn sanitize_service(name: &str) -> String {
+ let compact = name.split_whitespace().collect::>().join(" ");
+ let mut units = 0;
+ compact
+ .chars()
+ .take_while(|ch| {
+ units += ch.len_utf16();
+ units <= 96
+ })
+ .collect()
+}
+
#[cfg(test)]
mod tests {
- use super::incomplete_utf8_tail;
+ use super::{incomplete_utf8_tail, sanitize_service};
#[test]
fn live_guard_admission_tolerates_unavailable_recovery_census() {
@@ -801,5 +886,8 @@ mod tests {
}
}
assert_eq!(incomplete_utf8_tail(b"abc\xff"), 0);
+ assert_eq!(sanitize_service(" API\t/\nworker "), "API / worker");
+ assert_eq!(sanitize_service(&"x".repeat(120)).len(), 96);
+ assert_eq!(sanitize_service(&format!("{}🦀", "x".repeat(95))).len(), 95);
}
}
diff --git a/packages/lifecycle-supervisor/tests/fake_caller.rs b/packages/lifecycle-supervisor/tests/fake_caller.rs
index a37a5d649..7c61b8311 100644
--- a/packages/lifecycle-supervisor/tests/fake_caller.rs
+++ b/packages/lifecycle-supervisor/tests/fake_caller.rs
@@ -1683,6 +1683,96 @@ fn preexisting_broad_log_is_refused_before_launch() {
assert_eq!(fs::read(f.root.join("lifecycle.log")).unwrap(), b"sentinel");
}
+#[cfg(feature = "test-hooks")]
+fn stopped(pid: i32) -> bool {
+ #[cfg(target_os = "linux")]
+ {
+ fs::read_to_string(format!("/proc/{pid}/stat"))
+ .ok()
+ .and_then(|s| s.rsplit_once(") ").map(|(_, rest)| rest.starts_with('T')))
+ .unwrap_or(false)
+ }
+ #[cfg(target_os = "macos")]
+ {
+ let mut info = std::mem::MaybeUninit::::zeroed();
+ // SAFETY: writable native buffer lives through this synchronous call.
+ let size = unsafe {
+ libc::proc_pidinfo(
+ pid,
+ libc::PROC_PIDTBSDINFO,
+ 0,
+ info.as_mut_ptr().cast(),
+ std::mem::size_of::() as i32,
+ )
+ };
+ size as usize == std::mem::size_of::()
+ && unsafe { info.assume_init() }.pbi_status == libc::SSTOP
+ }
+}
+
+#[cfg(feature = "test-hooks")]
+fn kill_guard_at_launch_checkpoint(checkpoint: &str) {
+ let f = Fixture::new();
+ let mut process = f.process("worker", "hold-checkpoint");
+ process["env"]["SUPERVISOR_TEST_CHECKPOINT"] = json!(checkpoint);
+ let mut caller = command(&f.root, "run").spawn().unwrap();
+ send(caller.stdin.as_mut().unwrap(), &f.launch(vec![process]));
+ let state = f.state("worker");
+ let guard = pid(&state, "guardPid");
+ eventually(|| stopped(guard).then_some(()));
+ f.remember(&json!({"processes":[state.clone()]}));
+ assert_eq!(f.state("worker")["phase"], "starting");
+ let children: Vec<_> = census()
+ .into_iter()
+ .filter(|p| parent_pid(p.pid) == Some(guard))
+ .collect();
+ assert_eq!(children.len(), usize::from(checkpoint == "after-spawn"));
+ for child in &children {
+ f.owned.borrow_mut().push(child.clone());
+ }
+ if checkpoint == "after-spawn" {
+ // The paused guard cannot drain output. Wait until the command has
+ // finished its initial writes so killing the reader cannot race SIGPIPE.
+ eventually(|| {
+ f.root
+ .join("checkpoint-command-ready")
+ .exists()
+ .then_some(())
+ });
+ assert!(identity(children[0].pid).as_ref() == Some(&children[0]));
+ }
+ // SAFETY: exact stopped guard and session belong to this fixture.
+ assert_eq!(unsafe { libc::kill(guard, libc::SIGKILL) }, 0);
+ let _ = wait(&mut caller, TIMEOUT);
+ let (mut stop, receipt) = f.request("stop", &f.control(TOKEN));
+ if checkpoint == "before-spawn" {
+ assert_eq!(receipt["outcome"], "stopped");
+ assert!(wait(&mut stop, TIMEOUT).success());
+ } else {
+ // No running receipt/birth witness was published. Safety requires
+ // uncertainty without signals, rather than guessing session ownership.
+ assert_eq!(receipt["outcome"], "unknown");
+ assert!(!wait(&mut stop, TIMEOUT).success());
+ assert!(identity(children[0].pid).as_ref() == Some(&children[0]));
+ let (mut status, observed) = f.request("status", &f.control(TOKEN));
+ assert!(!wait(&mut status, TIMEOUT).success());
+ assert_eq!(observed["unknownProcessNames"], json!(["worker"]));
+ refused(&f, "run", f.launch(vec![f.process("worker", "hold")]));
+ }
+}
+
+#[cfg(feature = "test-hooks")]
+#[test]
+fn guard_killed_after_starting_save_before_spawn_starts_nothing() {
+ kill_guard_at_launch_checkpoint("before-spawn");
+}
+
+#[cfg(feature = "test-hooks")]
+#[test]
+fn guard_killed_after_spawn_before_running_save_retains_unknown() {
+ kill_guard_at_launch_checkpoint("after-spawn");
+}
+
#[test]
fn incomplete_frame_has_one_whole_handoff_deadline() {
let f = Fixture::new();
@@ -2147,11 +2237,14 @@ fn fixture_process() {
}
println!("pthread-child {child}");
}
- "hold" => (),
+ "hold" | "hold-checkpoint" => (),
_ => std::process::exit(99),
}
println!("fixture-ready");
std::io::stdout().flush().unwrap();
+ if mode == "hold-checkpoint" {
+ fs::write("checkpoint-command-ready", b"ready").unwrap();
+ }
loop {
thread::sleep(Duration::from_secs(1));
}
@@ -2159,6 +2252,27 @@ fn fixture_process() {
use std::os::unix::fs::DirBuilderExt;
+#[test]
+fn status_waits_for_inflight_stop_before_reporting_adoptable_liveness() {
+ let f = Fixture::new();
+ let (mut caller, _) = f.start(vec![f.process("worker", "ignore-term")]);
+ f.await_log("fixture-ready");
+ let mut stopping = command(&f.root, "stop").spawn().unwrap();
+ send(stopping.stdin.as_mut().unwrap(), &f.control(TOKEN));
+ eventually(|| (f.state("worker")["phase"] == "stopping").then_some(()));
+ let (mut observer, receipt) = f.request("status", &f.control(TOKEN));
+ assert!(receipt["liveProcessNames"].as_array().unwrap().is_empty());
+ assert!(
+ receipt["unknownProcessNames"]
+ .as_array()
+ .unwrap()
+ .is_empty()
+ );
+ assert!(wait(&mut observer, TIMEOUT).success());
+ assert!(wait(&mut stopping, TIMEOUT).success());
+ assert!(wait(&mut caller, TIMEOUT).success());
+}
+
#[test]
fn completed_stop_fences_a_run_whose_frame_arrives_after_retirement() {
let f = Fixture::new();
diff --git a/packages/lifecycle-supervisor/tests/incremental.rs b/packages/lifecycle-supervisor/tests/incremental.rs
new file mode 100644
index 000000000..a9db8b5af
--- /dev/null
+++ b/packages/lifecycle-supervisor/tests/incremental.rs
@@ -0,0 +1,614 @@
+//! Real incremental callers use framed stdin, private fixtures and no daemon.
+use serde_json::{Value, json};
+use std::{
+ fs,
+ io::{Read, Write},
+ os::{
+ fd::AsRawFd,
+ unix::fs::{DirBuilderExt, MetadataExt},
+ },
+ path::PathBuf,
+ process::{Child, Command, Stdio},
+ sync::atomic::{AtomicU64, Ordering},
+ thread,
+ time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
+
+const TOKEN: &str = "synthetic-incremental-owner";
+static SERIAL: AtomicU64 = AtomicU64::new(0);
+
+struct Fixture {
+ base: PathBuf,
+ root: PathBuf,
+ owners: Vec,
+}
+
+impl Fixture {
+ fn new() -> Self {
+ let nonce = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ let base = std::env::temp_dir().join(format!(
+ "incremental-{}-{nonce}-{}",
+ std::process::id(),
+ SERIAL.fetch_add(1, Ordering::Relaxed)
+ ));
+ fs::DirBuilder::new().mode(0o700).create(&base).unwrap();
+ let root = base.join("private-instance-with-a-deliberately-long-directory-name-so-unix-socket-addresses-exceed-the-darwin-limit-without-relative-addressing");
+ fs::DirBuilder::new().mode(0o700).create(&root).unwrap();
+ Self {
+ base,
+ root,
+ owners: Vec::new(),
+ }
+ }
+ fn command(&self, operation: &str) -> Command {
+ self.command_binary(
+ operation,
+ std::path::Path::new(env!("CARGO_BIN_EXE_hack-lifecycle-supervisor")),
+ )
+ }
+ fn command_binary(&self, operation: &str, executable: &std::path::Path) -> Command {
+ let mut command = Command::new(executable);
+ command
+ .arg(operation)
+ .env_clear()
+ .env("HOME", &self.base)
+ .env("HACK_DAEMON_AUTOSTART_DISABLE", "1")
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null());
+ command
+ }
+ fn process(&self, name: &str, mode: &str) -> Value {
+ json!({"name":name,"argv":[std::env::current_exe().unwrap(),"--exact","incremental_fixture","--nocapture"],"cwd":self.root,"env":{"HOME":self.base,"HACK_DAEMON_AUTOSTART_DISABLE":"1","INCREMENTAL_MODE":mode,"INCREMENTAL_VALUE":"synthetic-stdin-only-value"}})
+ }
+ fn launch(&self, process: Value) -> Value {
+ json!({"version":1,"root":self.root,"ownerToken":TOKEN,"processes":[process]})
+ }
+ fn call(&self, operation: &str, request: &Value) -> (Child, Value) {
+ let mut child = self.command(operation).spawn().unwrap();
+ send(child.stdin.as_mut().unwrap(), request);
+ // Keep stdin open: neither side may wait on descriptor closure.
+ let receipt = receive(child.stdout.as_mut().unwrap());
+ (child, receipt)
+ }
+ fn serve(&mut self, process: Value) -> Value {
+ let (child, receipt) = self.call("serve", &self.launch(process));
+ self.owners.push(child);
+ assert_eq!(receipt["outcome"], "ready");
+ receipt
+ }
+ fn add(&self, process: Value) -> Value {
+ let (mut child, receipt) = self.call("add", &self.launch(process));
+ assert!(wait(&mut child).success());
+ assert_eq!(receipt["outcome"], "ready");
+ receipt
+ }
+ fn control(&self, operation: &str) -> Value {
+ let (mut child, receipt) = self.call(
+ operation,
+ &json!({"version":1,"root":self.root,"ownerToken":TOKEN}),
+ );
+ assert!(wait(&mut child).success());
+ receipt
+ }
+ fn state(&self, name: &str) -> Value {
+ serde_json::from_slice(&fs::read(self.root.join(format!("{name}.json"))).unwrap()).unwrap()
+ }
+}
+
+impl Drop for Fixture {
+ fn drop(&mut self) {
+ if let Ok(mut child) = self.command("stop").spawn() {
+ if let Some(mut input) = child.stdin.take() {
+ send(
+ &mut input,
+ &json!({"version":1,"root":self.root,"ownerToken":TOKEN}),
+ );
+ }
+ let _ = wait(&mut child);
+ }
+ for child in &mut self.owners {
+ let _ = child.kill();
+ let _ = child.wait();
+ }
+ let _ = fs::remove_dir_all(&self.base);
+ }
+}
+
+/// Spawn a binary this test has just written. On Linux another test thread
+/// can fork while the copy's write descriptor is still open; that child holds
+/// the descriptor until it execs, and exec of the new file then fails with
+/// ETXTBSY. The window is short, so retry only that error, briefly.
+fn spawn_fresh_executable(command: &mut Command) -> Child {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ match command.spawn() {
+ Ok(child) => return child,
+ Err(error)
+ if error.raw_os_error() == Some(libc::ETXTBSY) && Instant::now() < deadline =>
+ {
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ Err(error) => panic!("spawn fresh executable: {error}"),
+ }
+ }
+}
+
+fn send(output: &mut impl Write, value: &Value) {
+ let bytes = serde_json::to_vec(value).unwrap();
+ output
+ .write_all(&(bytes.len() as u32).to_be_bytes())
+ .unwrap();
+ output.write_all(&bytes).unwrap();
+ output.flush().unwrap();
+}
+fn exact(input: &mut (impl Read + AsRawFd), bytes: &mut [u8]) {
+ let deadline = Instant::now() + Duration::from_secs(20);
+ let mut offset = 0;
+ while offset < bytes.len() {
+ assert!(
+ Instant::now() < deadline,
+ "framed response deadline expired"
+ );
+ let mut fd = libc::pollfd {
+ fd: input.as_raw_fd(),
+ events: libc::POLLIN,
+ revents: 0,
+ };
+ // SAFETY: one live initialized poll record.
+ assert!(unsafe { libc::poll(&mut fd, 1, 1000) } >= 0);
+ if fd.revents == 0 {
+ continue;
+ }
+ let count = input.read(&mut bytes[offset..]).unwrap();
+ assert!(count > 0, "incomplete frame");
+ offset += count;
+ }
+}
+fn receive(input: &mut (impl Read + AsRawFd)) -> Value {
+ let mut header = [0; 4];
+ exact(input, &mut header);
+ let length = u32::from_be_bytes(header) as usize;
+ assert!(length <= 1024 * 1024);
+ let mut bytes = vec![0; length];
+ exact(input, &mut bytes);
+ serde_json::from_slice(&bytes).unwrap()
+}
+fn wait(child: &mut Child) -> std::process::ExitStatus {
+ let deadline = Instant::now() + Duration::from_secs(20);
+ loop {
+ if let Some(status) = child.try_wait().unwrap() {
+ return status;
+ }
+ if Instant::now() >= deadline {
+ let _ = child.kill();
+ let _ = child.wait();
+ panic!("owned child did not settle");
+ }
+ thread::sleep(Duration::from_millis(20));
+ }
+}
+fn eventually(mut condition: impl FnMut() -> bool) {
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while !condition() {
+ assert!(Instant::now() < deadline);
+ thread::sleep(Duration::from_millis(20));
+ }
+}
+
+#[test]
+fn incremental_fixture() {
+ let Ok(mode) = std::env::var("INCREMENTAL_MODE") else {
+ return;
+ };
+ assert_eq!(
+ std::env::var("INCREMENTAL_VALUE").unwrap(),
+ "synthetic-stdin-only-value"
+ );
+ if mode == "fd" {
+ // The command only inherits its three intended standard descriptors.
+ for fd in 3..256 {
+ // SAFETY: observation only; the command fixture owns no other fd.
+ assert_eq!(
+ unsafe { libc::fcntl(fd, libc::F_GETFD) },
+ -1,
+ "unexpected inherited descriptor {fd}"
+ );
+ }
+ }
+ println!("incremental-ready-{mode}");
+ if mode == "short" {
+ return;
+ }
+ loop {
+ thread::sleep(Duration::from_millis(50));
+ }
+}
+
+#[test]
+fn long_root_and_incremental_hook_order_keep_one_owner() {
+ let mut fixture = Fixture::new();
+ assert!(fixture.root.as_os_str().len() > 104);
+ let first = fixture.serve(fixture.process("before-one", "hold"));
+ let second = fixture.add(fixture.process("before-two", "fd"));
+ let third = fixture.add(fixture.process("service", "hold"));
+ assert_eq!(first["supervisorPid"], second["supervisorPid"]);
+ assert_eq!(first["supervisorPid"], third["supervisorPid"]);
+ let status = fixture.control("status");
+ assert_eq!(status["liveProcessNames"].as_array().unwrap().len(), 3);
+ assert_eq!(status["unknownProcessNames"], json!([]));
+ assert_eq!(
+ fs::metadata(fixture.root.join("control.sock"))
+ .unwrap()
+ .mode()
+ & 0o777,
+ 0o600
+ );
+ fixture.control("stop");
+ for owner in &mut fixture.owners {
+ assert!(wait(owner).success());
+ }
+ assert!(!fixture.root.join("control.sock").exists());
+}
+
+#[test]
+fn wrong_token_and_duplicate_add_refuse_without_harming_owner() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ for mut request in [
+ fixture.launch(fixture.process("helper", "hold")),
+ fixture.launch(fixture.process("HELPER", "hold")),
+ fixture.launch(fixture.process("other", "hold")),
+ ] {
+ if request["processes"][0]["name"] == "other" {
+ request["ownerToken"] = json!("wrong-owner");
+ }
+ let (mut child, response) = fixture.call("add", &request);
+ assert_eq!(response["outcome"], "refused");
+ assert!(!wait(&mut child).success());
+ }
+ assert_eq!(
+ fixture.control("status")["liveProcessNames"],
+ json!(["helper"])
+ );
+}
+
+#[test]
+fn owner_death_keeps_native_status_and_recovery_stop() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ fixture.owners[0].kill().unwrap();
+ fixture.owners[0].wait().unwrap();
+ let (mut duplicate, response) =
+ fixture.call("serve", &fixture.launch(fixture.process("helper", "hold")));
+ assert_eq!(response["outcome"], "refused");
+ assert!(!wait(&mut duplicate).success());
+ assert_eq!(
+ fixture.control("status")["liveProcessNames"],
+ json!(["helper"])
+ );
+ fixture.control("stop");
+ let state = fixture.state("helper");
+ assert!(matches!(
+ state["phase"].as_str(),
+ Some("exited") | Some("killed")
+ ));
+}
+
+/// Add waits up to the stop-lock bound (16 s) on the server side. The
+/// forwarder must keep waiting for that reply instead of reporting failure
+/// after the ordinary 10-second frame deadline while the server still launches.
+#[test]
+fn add_reply_outlasts_a_stop_lock_held_past_the_frame_deadline() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ let lock = fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(fixture.root.join(".stop.lock"))
+ .unwrap();
+ let deadline = Instant::now() + Duration::from_secs(15);
+ // SAFETY: flock only borrows this live descriptor.
+ while unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 {
+ assert!(
+ Instant::now() < deadline,
+ "stop lock stayed busy after serve"
+ );
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ let mut add = fixture.command("add").spawn().unwrap();
+ send(
+ add.stdin.as_mut().unwrap(),
+ &fixture.launch(fixture.process("later", "hold")),
+ );
+ std::thread::sleep(Duration::from_secs(12));
+ drop(lock);
+ let receipt = receive(add.stdout.as_mut().unwrap());
+ assert!(wait(&mut add).success(), "{receipt}");
+ assert_eq!(receipt["outcome"], "ready");
+ assert!(fixture.root.join("later.json").exists());
+}
+
+#[test]
+fn completed_stop_cannot_be_followed_by_add_to_old_owner() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ fixture.control("stop");
+ let (mut add, receipt) = fixture.call("add", &fixture.launch(fixture.process("later", "hold")));
+ assert_eq!(receipt["outcome"], "refused");
+ assert!(!wait(&mut add).success());
+ assert!(!fixture.root.join("later.json").exists());
+}
+
+#[test]
+fn external_private_log_is_shared_and_add_cannot_redirect_it() {
+ let mut fixture = Fixture::new();
+ let logs = fixture.base.join("logs");
+ fs::DirBuilder::new().mode(0o700).create(&logs).unwrap();
+ let log = logs.join("project.log");
+ let mut request = fixture.launch(fixture.process("helper", "hold"));
+ request["logPath"] = json!(log);
+ let (owner, reply) = fixture.call("serve", &request);
+ assert_eq!(reply["outcome"], "ready");
+ fixture.owners.push(owner);
+ let mut add = fixture.launch(fixture.process("service", "hold"));
+ add["logPath"] = json!(log);
+ let (mut caller, reply) = fixture.call("add", &add);
+ assert_eq!(reply["outcome"], "ready");
+ assert!(wait(&mut caller).success());
+ eventually(|| {
+ fs::read_to_string(&log).is_ok_and(|value| {
+ value.contains("helper\tstdout") && value.contains("service\tstdout")
+ })
+ });
+ assert_eq!(fs::metadata(&log).unwrap().mode() & 0o777, 0o600);
+ assert!(!fixture.root.join("lifecycle.log").exists());
+ add["processes"][0]["name"] = json!("redirect");
+ add["logPath"] = json!(logs.join("other.log"));
+ let (mut caller, reply) = fixture.call("add", &add);
+ assert_eq!(reply["outcome"], "refused");
+ assert!(!wait(&mut caller).success());
+ assert!(!logs.join("other.log").exists());
+}
+
+#[test]
+fn short_hook_owner_retires_then_later_hook_can_serve_again() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("first", "short"));
+ assert!(wait(&mut fixture.owners[0]).success());
+ fixture.serve(fixture.process("later", "hold"));
+ assert_eq!(
+ fixture.control("status")["liveProcessNames"],
+ json!(["later"])
+ );
+}
+
+#[test]
+fn framing_refusal_is_finite_without_stdin_eof() {
+ let fixture = Fixture::new();
+ let request = json!({"version":42,"root":fixture.root,"ownerToken":TOKEN,"processes":[]});
+ let (mut child, receipt) = fixture.call("serve", &request);
+ assert_eq!(receipt["outcome"], "refused");
+ assert_eq!(receipt["processes"], json!([]));
+ assert!(!wait(&mut child).success());
+}
+
+#[test]
+fn concurrent_add_and_stop_leave_no_later_command_running() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ let mut add = fixture.command("add").spawn().unwrap();
+ send(
+ add.stdin.as_mut().unwrap(),
+ &fixture.launch(fixture.process("later", "hold")),
+ );
+ let mut stop = fixture.command("stop").spawn().unwrap();
+ send(
+ stop.stdin.as_mut().unwrap(),
+ &json!({"version":1,"root":fixture.root,"ownerToken":TOKEN}),
+ );
+ let add_reply = receive(add.stdout.as_mut().unwrap());
+ let stop_reply = receive(stop.stdout.as_mut().unwrap());
+ let add_status = wait(&mut add);
+ assert_eq!(add_status.success(), add_reply["outcome"] == "ready");
+ assert_eq!(stop_reply["outcome"], "stopped");
+ assert!(wait(&mut stop).success());
+ for owner in &mut fixture.owners {
+ assert!(wait(owner).success());
+ }
+ assert_eq!(fixture.control("status")["liveProcessNames"], json!([]));
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn incremental_environment_is_absent_from_argv_and_persisted_files() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("first", "hold"));
+ let receipt = fixture.add(fixture.process("second", "hold"));
+ let sentinel = b"synthetic-stdin-only-value";
+ for state in receipt["processes"].as_array().unwrap() {
+ for key in ["guardPid", "commandPid"] {
+ let args = fs::read(format!("/proc/{}/cmdline", state[key].as_i64().unwrap())).unwrap();
+ assert!(!args.windows(sentinel.len()).any(|v| v == sentinel));
+ }
+ }
+ for entry in fs::read_dir(&fixture.root).unwrap() {
+ let entry = entry.unwrap();
+ let metadata = entry.metadata().unwrap();
+ assert_eq!(metadata.mode() & 0o077, 0);
+ if metadata.is_file() {
+ let bytes = fs::read(entry.path()).unwrap();
+ assert!(!bytes.windows(sentinel.len()).any(|v| v == sentinel));
+ }
+ }
+}
+
+#[test]
+#[cfg(target_os = "linux")]
+fn status_names_unknown_when_guard_lost_with_live_command() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ let state = fixture.state("helper");
+ let guard = state["guardPid"].as_i64().unwrap() as i32;
+ let command = state["commandPid"].as_i64().unwrap() as i32;
+ let original = fs::read_to_string(format!("/proc/{command}/stat")).unwrap();
+ let birth = original
+ .rsplit_once(") ")
+ .unwrap()
+ .1
+ .split_whitespace()
+ .nth(19)
+ .unwrap()
+ .to_owned();
+ // SAFETY: the guard identity was returned by this fixture's own launch.
+ unsafe {
+ libc::kill(guard, libc::SIGKILL);
+ }
+ eventually(|| fs::read_to_string(format!("/proc/{guard}/stat")).is_err());
+ let status = fixture.control("status");
+ assert_eq!(status["liveProcessNames"], json!([]));
+ assert_eq!(status["unknownProcessNames"], json!(["helper"]));
+ // The test deliberately removed the recovery guard. Stop this synthetic
+ // direct command only after rechecking its captured native birth.
+ let now = fs::read_to_string(format!("/proc/{command}/stat")).unwrap();
+ assert_eq!(
+ now.rsplit_once(") ").unwrap().1.split_whitespace().nth(19),
+ Some(birth.as_str())
+ );
+ // SAFETY: exact live birth matches this fixture's captured command identity.
+ unsafe {
+ libc::kill(command, libc::SIGKILL);
+ }
+}
+
+#[test]
+fn display_names_are_visible_labels_never_state_paths() {
+ let mut fixture = Fixture::new();
+ let display = "API worker / ready\ttask\nnext";
+ let mut process = fixture.process("service-0123456789abcdef", "hold");
+ process["displayName"] = json!(display);
+ let first = fixture.serve(process);
+ assert_eq!(first["processes"][0]["name"], "service-0123456789abcdef");
+ assert_eq!(first["processes"][0]["displayName"], display);
+ let mut second = fixture.process("service-fedcba9876543210", "hold");
+ second["displayName"] = json!("other / service");
+ fixture.add(second);
+ let status = fixture.control("status");
+ assert!(
+ status["liveProcessNames"]
+ .as_array()
+ .unwrap()
+ .contains(&json!(display))
+ );
+ assert!(
+ status["liveProcessNames"]
+ .as_array()
+ .unwrap()
+ .contains(&json!("other / service"))
+ );
+ eventually(|| {
+ fs::read_to_string(fixture.root.join("lifecycle.log"))
+ .is_ok_and(|log| log.contains("\tAPI worker / ready task next\tstdout\t"))
+ });
+ assert!(fixture.root.join("service-0123456789abcdef.json").exists());
+ assert!(fixture.root.join("service-fedcba9876543210.json").exists());
+ assert_eq!(
+ fs::read_dir(&fixture.root)
+ .unwrap()
+ .filter_map(Result::ok)
+ .filter(|entry| entry.file_type().unwrap().is_dir())
+ .count(),
+ 0
+ );
+}
+
+#[test]
+fn display_name_limit_refuses_without_launch() {
+ let fixture = Fixture::new();
+ let mut process = fixture.process("safe-key", "hold");
+ process["displayName"] = json!("x".repeat(4097));
+ let (mut owner, receipt) = fixture.call("serve", &fixture.launch(process));
+ assert_eq!(receipt["outcome"], "refused");
+ assert!(!wait(&mut owner).success());
+ assert!(!fixture.root.join("safe-key.json").exists());
+}
+
+#[test]
+fn lost_add_client_reply_retains_native_ownership_and_live_observation() {
+ let mut fixture = Fixture::new();
+ fixture.serve(fixture.process("helper", "hold"));
+ let mut add = fixture.command("add").spawn().unwrap();
+ drop(add.stdout.take());
+ send(
+ add.stdin.as_mut().unwrap(),
+ &fixture.launch(fixture.process("late", "hold")),
+ );
+ assert!(!wait(&mut add).success());
+ let status = fixture.control("status");
+ assert!(
+ status["liveProcessNames"]
+ .as_array()
+ .unwrap()
+ .contains(&json!("late"))
+ );
+ assert_eq!(fixture.state("late")["ownerToken"], TOKEN);
+ fixture.control("stop");
+}
+
+#[test]
+fn escaped_display_name_cannot_publish_unreadable_state() {
+ let fixture = Fixture::new();
+ let mut process = fixture.process("safe-key", "hold");
+ process["displayName"] = json!(format!("x{}", "\u{1}".repeat(4095)));
+ let (mut owner, receipt) = fixture.call("serve", &fixture.launch(process));
+ assert_eq!(receipt["outcome"], "refused");
+ assert!(!wait(&mut owner).success());
+ assert!(!fixture.root.join("safe-key.json").exists());
+}
+
+#[test]
+fn atomic_executable_replacement_between_serve_and_add_keeps_guard_spawn_usable() {
+ let mut fixture = Fixture::new();
+ let installed = fixture.base.join("installed-supervisor");
+ let replacement = fixture.base.join("replacement-supervisor");
+ fs::copy(env!("CARGO_BIN_EXE_hack-lifecycle-supervisor"), &installed).unwrap();
+ let mut owner = spawn_fresh_executable(&mut fixture.command_binary("serve", &installed));
+ send(
+ owner.stdin.as_mut().unwrap(),
+ &fixture.launch(fixture.process("first", "hold")),
+ );
+ assert_eq!(receive(owner.stdout.as_mut().unwrap())["outcome"], "ready");
+ fixture.owners.push(owner);
+ fs::copy(
+ env!("CARGO_BIN_EXE_hack-lifecycle-supervisor"),
+ &replacement,
+ )
+ .unwrap();
+ fs::rename(replacement, installed).unwrap();
+ let receipt = fixture.add(fixture.process("second", "hold"));
+ assert_eq!(receipt["liveProcessNames"].as_array().unwrap().len(), 2);
+ fixture.control("stop");
+}
+
+#[test]
+fn completed_stop_retires_a_delayed_initial_launch_before_any_authored_effect() {
+ let fixture = Fixture::new();
+ let mut delayed = fixture.command("serve").spawn().unwrap();
+ // Child exists but its launch frame has not arrived: stop must fence this
+ // generation even though no command receipt or guard exists yet.
+ let receipt = fixture.control("stop");
+ assert_eq!(receipt["outcome"], "stopped");
+ send(
+ delayed.stdin.as_mut().unwrap(),
+ &fixture.launch(fixture.process("late", "hold")),
+ );
+ let refused = receive(delayed.stdout.as_mut().unwrap());
+ assert_eq!(refused["outcome"], "refused");
+ assert!(!wait(&mut delayed).success());
+ assert!(!fixture.root.join("late.json").exists());
+}
diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts
index f83baf12f..daf63899d 100644
--- a/scripts/lib/tla-runtime-models.ts
+++ b/scripts/lib/tla-runtime-models.ts
@@ -141,6 +141,34 @@ const restartPolicyStaleUses = [
{ engine: "exited", bound: 3 },
] as const;
const contracts: readonly ModelContract[] = [
+ {
+ name: "lifecycle-controller",
+ module: "LifecycleController",
+ states: 21,
+ invariant: "NoAdoptedStop",
+ action: "Abort",
+ fields: [
+ "unsafeStop = TRUE",
+ "epoch = 2",
+ "creatorEpoch = 1",
+ "adopted = TRUE",
+ 'stage = "Aborted"',
+ ],
+ additionalControls: [
+ {
+ name: "pending",
+ negative: true,
+ invariant: "DurableBeforeLaunch",
+ action: "Launch",
+ fields: [
+ "unsafeLaunch = TRUE",
+ "pending = FALSE",
+ "live = TRUE",
+ 'stage = "AwaitReply"',
+ ],
+ },
+ ],
+ },
{
name: "lifecycle-supervisor",
module: "LifecycleSupervisor",
diff --git a/src/backends/native-authored-host-processes.ts b/src/backends/native-authored-host-processes.ts
index a6cbbfb3b..0c1a15f6b 100644
--- a/src/backends/native-authored-host-processes.ts
+++ b/src/backends/native-authored-host-processes.ts
@@ -376,6 +376,9 @@ async function stopKnown(opts: {
};
await current();
if (opts.savedEntry) {
+ if (opts.savedEntry.backend === "supervisor") {
+ return refuse();
+ }
const backend = getMuxBackends().get(opts.savedEntry.backend) ?? refuse();
const inspected = await inspectLifecycleSession({
backend,
@@ -779,6 +782,9 @@ export async function createNativeAuthoredHostProcesses(input: {
return refuse();
}
if (savedEntry) {
+ if (savedEntry.backend === "supervisor") {
+ return refuse();
+ }
const backend = getMuxBackends().get(savedEntry.backend) ?? refuse();
const inspection = await inspectLifecycleSession({
backend,
diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts
index 92bcbaa7c..feebe783a 100644
--- a/src/commands/doctor.ts
+++ b/src/commands/doctor.ts
@@ -2553,21 +2553,34 @@ async function checkProjectLifecycleHygiene({
};
}
+ const native = await describeLifecycleExecutable(
+ await readProjectConfig(ctx)
+ );
+ if (native.failure) {
+ return native.failure;
+ }
+ const executable = native.description;
const inspection = await inspectProjectLifecycleHygiene({
projectDir: ctx.projectDir,
projectRoot: ctx.projectRoot,
});
if (
- inspection.staleEntries.length === 0 &&
- inspection.orphanedProcessGroups.length === 0 &&
- inspection.orphanedSessions.length === 0 &&
- inspection.unverifiedSessions.length === 0
+ [
+ inspection.staleEntries,
+ inspection.orphanedProcessGroups,
+ inspection.orphanedSessions,
+ inspection.unverifiedSessions,
+ ].every((entries) => entries.length === 0)
) {
return {
name: "lifecycle hygiene",
status: "ok",
- message:
+ message: [
"No stale lifecycle state entries or orphaned lifecycle processes",
+ executable,
+ ]
+ .filter(Boolean)
+ .join("; "),
};
}
@@ -2611,10 +2624,36 @@ async function checkProjectLifecycleHygiene({
return {
name: "lifecycle hygiene",
status: "warn",
- message: `${details.join("; ")} (${nextStep})`,
+ message: [`${details.join("; ")} (${nextStep})`, executable]
+ .filter(Boolean)
+ .join("; "),
};
}
+async function describeLifecycleExecutable(
+ config: Awaited>
+): Promise<{ description: string; failure?: CheckResult }> {
+ if (
+ resolveLifecycleRunner({ runner: config.lifecycle?.runner }) !==
+ "supervisor"
+ ) {
+ return { description: "" };
+ }
+ try {
+ const identity = await preflightLifecycleSupervisor();
+ return { description: `${identity.path} (${identity.version})` };
+ } catch {
+ return {
+ description: "",
+ failure: {
+ name: "lifecycle supervisor",
+ status: "warn",
+ message: `${resolveLifecycleSupervisorBinary()}: missing or incompatible executable`,
+ },
+ };
+ }
+}
+
async function checkProjectEnvMode({
startDir,
}: {
@@ -5796,3 +5835,9 @@ async function checkProxyNetwork(): Promise {
].join("\n"),
};
}
+
+import { resolveLifecycleRunner } from "../lib/lifecycle-runner.ts";
+import {
+ preflightLifecycleSupervisor,
+ resolveLifecycleSupervisorBinary,
+} from "../lib/lifecycle-supervisor.ts";
diff --git a/src/commands/host-ps.ts b/src/commands/host-ps.ts
index c81124734..b08a743d8 100644
--- a/src/commands/host-ps.ts
+++ b/src/commands/host-ps.ts
@@ -11,6 +11,7 @@ import {
type LifecycleStateEntry,
readLifecycleState,
} from "../lib/lifecycle-runtime.ts";
+import { inspectLifecycleSupervisor } from "../lib/lifecycle-supervisor.ts";
import { resolvePersistedLifecycleProcessGroupIds } from "../lib/project-lifecycle-processes.ts";
import { inspectLifecycleSession } from "../lib/project-lifecycle-sessions.ts";
import {
@@ -87,7 +88,7 @@ export const hostPsCommand = withHandler(spec, async ({ args }) => {
return 0;
});
-async function readLifecycleObservations({
+export async function readLifecycleObservations({
registry,
filter,
snapshot,
@@ -97,22 +98,49 @@ async function readLifecycleObservations({
readonly snapshot: readonly ObservedProcess[] | null;
}): Promise[]> {
const lifecycle: Record[] = [];
- const directories = new Map();
+ const directories = new Map<
+ string,
+ { readonly root: string; readonly name: string }
+ >();
for (const project of registry.projects.filter(
(project) => !filter || project.name === filter
)) {
- directories.set(project.projectDir, project.repoRoot);
+ directories.set(project.projectDir, {
+ root: project.repoRoot,
+ name: project.name,
+ });
for (const worktree of project.worktrees ?? []) {
- directories.set(
- resolve(worktree.path, project.projectDirName),
- worktree.path
- );
+ directories.set(resolve(worktree.path, project.projectDirName), {
+ root: worktree.path,
+ name: project.name,
+ });
}
}
- for (const [projectDir, projectRoot] of directories) {
- for (const entry of await readLifecycleState({ projectDir })) {
+ for (const [projectDir, project] of directories) {
+ const projectRoot = project.root;
+ let entries: readonly LifecycleStateEntry[];
+ try {
+ entries = await readLifecycleState({ projectDir });
+ } catch {
+ lifecycle.push({
+ project: project.name,
+ projectRoot,
+ session: null,
+ ownership: "unknown",
+ status: "unknown",
+ reason: "state_unreadable",
+ attention: "state_unreadable",
+ });
+ continue;
+ }
+ for (const entry of entries) {
lifecycle.push(
- await observeLifecycleEntry({ entry, projectRoot, snapshot })
+ await observeLifecycleEntry({
+ entry,
+ projectDir,
+ projectRoot,
+ snapshot,
+ })
);
}
}
@@ -120,13 +148,22 @@ async function readLifecycleObservations({
}
async function observeLifecycleEntry({
entry,
+ projectDir,
projectRoot,
snapshot,
}: {
readonly entry: LifecycleStateEntry;
+ readonly projectDir: string;
readonly projectRoot: string;
readonly snapshot: readonly ObservedProcess[] | null;
}): Promise> {
+ if (entry.backend === "supervisor") {
+ return await observeSupervisorLifecycleEntry({
+ entry,
+ projectDir,
+ projectRoot,
+ });
+ }
const backends = getMuxBackends();
const backend = backends.get(entry.backend);
const inspection = backend
@@ -177,6 +214,62 @@ async function observeLifecycleEntry({
};
}
+/** Read native session ownership without interpreting it as a mux session. */
+export async function observeSupervisorLifecycleEntry(opts: {
+ readonly entry: LifecycleStateEntry;
+ readonly projectDir: string;
+ readonly projectRoot: string;
+ readonly inspect?: typeof inspectLifecycleSupervisor;
+}): Promise> {
+ const base = {
+ project: opts.entry.projectName,
+ projectRoot: opts.projectRoot,
+ branch: opts.entry.branch,
+ session: opts.entry.sessionName,
+ backend: "supervisor",
+ lifetime: "persistent",
+ definitionSource: "persisted",
+ // The generic host snapshot does not include session IDs, so it cannot
+ // qualify every group or descendant after a command changes groups.
+ processGroupIds: [],
+ observedPids: [],
+ liveCpuTimeMs: null,
+ liveRssBytes: null,
+ elapsedMs: null,
+ };
+ if (!opts.entry.ownershipToken) {
+ return { ...base, ownership: "unknown", attention: "ownership_unverified" };
+ }
+ try {
+ const receipt = await (opts.inspect ?? inspectLifecycleSupervisor)({
+ projectDir: opts.projectDir,
+ composeProject: opts.entry.composeProject,
+ ownershipToken: opts.entry.ownershipToken,
+ });
+ if (
+ receipt.outcome !== "observed" ||
+ receipt.unknownProcessNames.length > 0
+ ) {
+ return {
+ ...base,
+ ownership: "unknown",
+ attention: "ownership_unverified",
+ };
+ }
+ const live = new Set(receipt.liveProcessNames);
+ return {
+ ...base,
+ ownership: live.size > 0 ? "owned-healthy" : "absent",
+ sessionIds: receipt.processes
+ .filter((process) => live.has(process.name))
+ .map((process) => process.processGroupId),
+ attention: live.size > 0 ? null : "session_absent_review_state",
+ };
+ } catch {
+ return { ...base, ownership: "unknown", attention: "ownership_unverified" };
+ }
+}
+
function lifecycleAttention(classification: string | undefined): string | null {
if (classification === "absent") {
return "session_absent_review_state";
diff --git a/src/commands/project.ts b/src/commands/project.ts
index 9df3bb385..1d6028eb2 100644
--- a/src/commands/project.ts
+++ b/src/commands/project.ts
@@ -4,6 +4,8 @@ import {
migrateLegacyEnvForCommand as migrateLegacyProjectEnv,
recordEnvMigrationProjectRun,
} from "../lib/env-migration.ts";
+import { resolveLifecycleRunner } from "../lib/lifecycle-runner.ts";
+import { preflightLifecycleSupervisor } from "../lib/lifecycle-supervisor.ts";
import {
createLifecycleProcessController,
installLifecycleSignalCleanup,
@@ -2115,11 +2117,15 @@ function sharedHelperControls(
* runs. Exported for tests.
*/
export async function assertSharedHelperSessionMuxSupported(opts: {
+ readonly runner?: "mux" | "supervisor";
readonly sharedHelpersConfigured: boolean;
readonly needsSession: boolean;
readonly mux: () => Promise;
}): Promise {
- if (!(opts.sharedHelpersConfigured && opts.needsSession)) {
+ if (
+ opts.runner === "supervisor" ||
+ !(opts.sharedHelpersConfigured && opts.needsSession)
+ ) {
return;
}
const mux = await opts.mux();
@@ -2149,6 +2155,9 @@ async function runLifecycleUpBeforeAndProcesses(opts: {
readonly effectiveEnvName: string | null;
readonly composeProject: string;
}): Promise {
+ if (opts.cfg.lifecycleRunnerError) {
+ throw new Error(opts.cfg.lifecycleRunnerError);
+ }
if (opts.cfg.sharedHelpersError) {
// Fail closed before any hook runs: starting without the configured
// helpers would launch services that depend on them.
@@ -2160,6 +2169,7 @@ async function runLifecycleUpBeforeAndProcesses(opts: {
// the hooks run supersedes this operation (see startProjectSharedHelpers).
const sharedHelperConfigs = opts.cfg.lifecycle?.sharedHelpers ?? [];
await assertSharedHelperSessionMuxSupported({
+ runner: resolveLifecycleRunner({ runner: opts.cfg.lifecycle?.runner }),
sharedHelpersConfigured: sharedHelperConfigs.length > 0,
needsSession:
(opts.cfg.lifecycle?.processes ?? []).length > 0 ||
@@ -2243,6 +2253,7 @@ async function runLifecycleUpBeforeAndProcesses(opts: {
}
const starter = createLifecycleProcessController({
+ runner: resolveLifecycleRunner({ runner: opts.cfg.lifecycle?.runner }),
mux: () => resolveMux({ project: opts.project }),
project: opts.project,
projectName: opts.projectName,
@@ -2796,6 +2807,7 @@ export async function startLifecycleProcesses(opts: {
}
const starter = createLifecycleProcessController({
+ runner: resolveLifecycleRunner({ runner: opts.cfg.lifecycle?.runner }),
mux: opts.mux ?? (() => resolveMux({ project: opts.project })),
project: opts.project,
projectName: opts.projectName,
@@ -6685,6 +6697,11 @@ async function runUpCommand({
if (sharedHelperRefusal !== null) {
return sharedHelperRefusal;
}
+ if (
+ resolveLifecycleRunner({ runner: cfg.lifecycle?.runner }) === "supervisor"
+ ) {
+ await preflightLifecycleSupervisor();
+ }
const baseProjectName = await resolveComposeProjectName({ project, cfg });
const composeProjectName = branch ? `${baseProjectName}--${branch}` : null;
@@ -7947,10 +7964,18 @@ function refuseInvalidSharedHelpers(opts: {
readonly cfg: Awaited>;
readonly json: boolean;
}): number | null {
- if (!opts.cfg.sharedHelpersError) {
+ let runnerError = opts.cfg.lifecycleRunnerError;
+ try {
+ resolveLifecycleRunner({ runner: opts.cfg.lifecycle?.runner });
+ } catch {
+ runnerError = "Invalid lifecycle runner; use mux or supervisor.";
+ }
+ if (!(opts.cfg.sharedHelpersError || runnerError)) {
return null;
}
- const message = `Invalid lifecycle.sharedHelpers; nothing was changed: ${opts.cfg.sharedHelpersError}`;
+ const message =
+ runnerError ??
+ `Invalid lifecycle.sharedHelpers; nothing was changed: ${opts.cfg.sharedHelpersError}`;
if (opts.json) {
return emitLifecycleResult({
result: errorResult({ code: "E_LIFECYCLE_FAILED", message }),
@@ -8646,6 +8671,11 @@ async function runRestartCommand({
}
const baseProjectName = await resolveComposeProjectName({ project, cfg });
+ if (
+ resolveLifecycleRunner({ runner: cfg.lifecycle?.runner }) === "supervisor"
+ ) {
+ await preflightLifecycleSupervisor();
+ }
const { composeProjectName, lifecycleComposeProject } = resolveRestartTarget({
baseProjectName,
branch,
diff --git a/src/lib/lifecycle-runner.ts b/src/lib/lifecycle-runner.ts
new file mode 100644
index 000000000..96ac782bd
--- /dev/null
+++ b/src/lib/lifecycle-runner.ts
@@ -0,0 +1,35 @@
+export type LifecycleRunner = "mux" | "supervisor";
+
+export const LIFECYCLE_RUNNER_CONFIG_ERROR =
+ 'Invalid lifecycle.runner; expected "mux" or "supervisor". Values omitted.';
+
+/** Validate without coercion: a misspelled runner must never launch another one. */
+export function parseLifecycleRunner(value: unknown): LifecycleRunner | null {
+ return value === "mux" || value === "supervisor" ? value : null;
+}
+
+/** Explicit test/runtime overrides precede project config; omission keeps mux. */
+export function resolveLifecycleRunner(opts: {
+ readonly runner?: unknown;
+ readonly env?: Readonly>;
+}): LifecycleRunner {
+ const env = opts.env ?? process.env;
+ const override = env.HACK_LIFECYCLE_RUNNER;
+ if (override !== undefined) {
+ const runner = parseLifecycleRunner(override);
+ if (!runner) {
+ throw new Error(
+ 'Invalid HACK_LIFECYCLE_RUNNER; expected "mux" or "supervisor". Values omitted.'
+ );
+ }
+ return runner;
+ }
+ if (opts.runner === undefined) {
+ return "mux";
+ }
+ const runner = parseLifecycleRunner(opts.runner);
+ if (!runner) {
+ throw new Error(LIFECYCLE_RUNNER_CONFIG_ERROR);
+ }
+ return runner;
+}
diff --git a/src/lib/lifecycle-runtime.ts b/src/lib/lifecycle-runtime.ts
index debbea0f2..bd143759e 100644
--- a/src/lib/lifecycle-runtime.ts
+++ b/src/lib/lifecycle-runtime.ts
@@ -1,15 +1,18 @@
+import { lstat, rename } from "node:fs/promises";
import { resolve } from "node:path";
import { readLinesFromStream } from "../ui/lines.ts";
-import { pathExists, readTextFile, writeTextFile } from "./fs.ts";
import { getString, isRecord } from "./guards.ts";
import {
appendPrivateFile,
ensurePrivateDirectory,
openPrivateAppend,
PrivatePathRefusedError,
+ readReplaceablePrivateFile,
+ replacePrivateFileAtomic,
} from "./private-file.ts";
+import { withProjectsRegistryLock } from "./projects-registry-lock.ts";
-export type LifecycleBackend = "tmux" | "zellij";
+export type LifecycleBackend = "tmux" | "zellij" | "supervisor";
export type LifecycleStateProcess = {
readonly name: string;
@@ -18,6 +21,10 @@ export type LifecycleStateProcess = {
readonly panePid?: number;
readonly paneBirth?: string;
readonly processGroupId?: number;
+ readonly guardPid?: number;
+ readonly guardStartTime?: number;
+ readonly commandPid?: number;
+ readonly bootId?: string;
readonly processGroupBirth?: string;
};
@@ -31,6 +38,10 @@ export type LifecycleStateEntry = {
readonly definitionHash?: string;
readonly processes: readonly LifecycleStateProcess[];
readonly updatedAt: string;
+ readonly supervisorPid?: number;
+ readonly supervisorRoot?: string;
+ readonly supervisorLaunchAttempted?: boolean;
+ readonly cleanupEpoch?: string;
};
type LifecycleStateFile = {
@@ -151,34 +162,78 @@ export async function ensurePrivateLifecycleLog(opts: {
export async function readLifecycleState(opts: {
readonly projectDir: string;
+ readonly stateLocked?: boolean;
}): Promise {
- const path = resolveLifecycleStatePath({ projectDir: opts.projectDir });
- if (!(await pathExists(path))) {
+ const root = resolveLifecycleRootDir(opts);
+ const path = resolveLifecycleStatePath(opts);
+ const rootInfo = await lstat(root).catch((error: unknown) => {
+ if (isRecord(error) && error.code === "ENOENT") {
+ return null;
+ }
+ throw error;
+ });
+ if (!rootInfo) {
return [];
}
- const text = await readTextFile(path);
- if (!text) {
+ if (!rootInfo.isDirectory() || rootInfo.uid !== process.getuid?.()) {
+ throw lifecycleStateRefused();
+ }
+ await verifyLifecycleStateTarget(path);
+ const file = await readReplaceablePrivateFile({
+ path,
+ directories: [root],
+ maxBytes: 1024 * 1024,
+ refusal: lifecycleStateRefused,
+ });
+ if (file === null) {
+ // Removal rewrites an empty entry list and never deletes this file. Once
+ // supervisor ownership was recorded, a missing file is lost state, not
+ // absence: native commands may still run.
+ if (await supervisorStateRecorded(root)) {
+ throw lifecycleStateRefused();
+ }
return [];
}
+ await verifyLifecycleStateTarget(path);
+ const text = file.text;
let parsed: unknown;
try {
parsed = JSON.parse(text);
} catch {
+ if (await supervisorStateRecorded(root)) {
+ throw lifecycleStateRefused();
+ }
+ if (!opts.stateLocked) {
+ return await mutateLifecycleState(opts.projectDir, () =>
+ readLifecycleState({ ...opts, stateLocked: true })
+ );
+ }
+ await verifyLifecycleStateTarget(path);
+ await rename(path, `${path}.legacy-${Date.now()}-${crypto.randomUUID()}`);
+ console.error(
+ "Moved unreadable legacy lifecycle JSON aside; continuing without lifecycle state."
+ );
return [];
}
if (!isRecord(parsed)) {
- return [];
+ throw lifecycleStateRefused();
}
const entriesRaw = parsed.entries;
if (!Array.isArray(entriesRaw)) {
- return [];
+ throw lifecycleStateRefused();
}
const entries: LifecycleStateEntry[] = [];
for (const entry of entriesRaw) {
const parsedEntry = parseLifecycleStateEntry(entry);
- if (parsedEntry) {
- entries.push(parsedEntry);
+ if (
+ !parsedEntry ||
+ entries.some(
+ (current) => current.composeProject === parsedEntry.composeProject
+ )
+ ) {
+ throw lifecycleStateRefused();
}
+ entries.push(parsedEntry);
}
return entries;
}
@@ -187,18 +242,33 @@ export async function upsertLifecycleStateEntry(opts: {
readonly projectDir: string;
readonly entry: LifecycleStateEntry;
}): Promise {
- const state = await readLifecycleState({ projectDir: opts.projectDir });
- const withoutCurrent = state.filter(
- (entry) => entry.composeProject !== opts.entry.composeProject
- );
- const next: LifecycleStateFile = {
- entries: [...withoutCurrent, opts.entry].sort((a, b) =>
- a.composeProject.localeCompare(b.composeProject)
- ),
- };
- await writeLifecycleStateFile({
- projectDir: opts.projectDir,
- state: next,
+ return await mutateLifecycleState(opts.projectDir, async () => {
+ // Read before marking: the marker turns a missing state file into a
+ // refusal, and a project's first supervisor entry has no file yet.
+ const state = await readLifecycleState({
+ projectDir: opts.projectDir,
+ stateLocked: true,
+ });
+ const withoutCurrent = state.filter(
+ (entry) => entry.composeProject !== opts.entry.composeProject
+ );
+ const next: LifecycleStateFile = {
+ entries: [...withoutCurrent, opts.entry].sort((a, b) =>
+ a.composeProject.localeCompare(b.composeProject)
+ ),
+ };
+ await writeLifecycleStateFile({
+ projectDir: opts.projectDir,
+ state: next,
+ });
+ // The entry is durable before the marker and before any native launch, so
+ // a crash between them never leaves a marker without a readable file.
+ if (opts.entry.backend === "supervisor") {
+ await appendPrivateFile({
+ path: resolve(resolveLifecycleRootDir(opts), "supervisor-state"),
+ data: "",
+ });
+ }
});
}
@@ -206,16 +276,21 @@ export async function removeLifecycleStateEntry(opts: {
readonly projectDir: string;
readonly composeProject: string;
}): Promise {
- const state = await readLifecycleState({ projectDir: opts.projectDir });
- const nextEntries = state.filter(
- (entry) => entry.composeProject !== opts.composeProject
- );
- if (nextEntries.length === state.length) {
- return;
- }
- await writeLifecycleStateFile({
- projectDir: opts.projectDir,
- state: { entries: nextEntries },
+ return await mutateLifecycleState(opts.projectDir, async () => {
+ const state = await readLifecycleState({
+ projectDir: opts.projectDir,
+ stateLocked: true,
+ });
+ const nextEntries = state.filter(
+ (entry) => entry.composeProject !== opts.composeProject
+ );
+ if (nextEntries.length === state.length) {
+ return;
+ }
+ await writeLifecycleStateFile({
+ projectDir: opts.projectDir,
+ state: { entries: nextEntries },
+ });
});
}
@@ -224,50 +299,61 @@ export async function removeLifecycleStateEntryIfOwned(opts: {
readonly composeProject: string;
readonly ownershipToken: string;
}): Promise {
- const state = await readLifecycleState({ projectDir: opts.projectDir });
- const current = state.find(
- (entry) => entry.composeProject === opts.composeProject
- );
- if (current?.ownershipToken !== opts.ownershipToken) {
- return false;
- }
- await writeLifecycleStateFile({
- projectDir: opts.projectDir,
- state: {
- entries: state.filter(
- (entry) => entry.composeProject !== opts.composeProject
- ),
- },
+ return await mutateLifecycleState(opts.projectDir, async () => {
+ const state = await readLifecycleState({
+ projectDir: opts.projectDir,
+ stateLocked: true,
+ });
+ const current = state.find(
+ (entry) => entry.composeProject === opts.composeProject
+ );
+ if (current?.ownershipToken !== opts.ownershipToken) {
+ return false;
+ }
+ await writeLifecycleStateFile({
+ projectDir: opts.projectDir,
+ state: {
+ entries: state.filter(
+ (entry) => entry.composeProject !== opts.composeProject
+ ),
+ },
+ });
+ return true;
});
- return true;
}
export async function removeLifecycleStateEntryIfMatching(opts: {
readonly projectDir: string;
readonly expectedEntry: LifecycleStateEntry;
}): Promise {
- const state = await readLifecycleState({ projectDir: opts.projectDir });
- const current = state.find(
- (entry) => entry.composeProject === opts.expectedEntry.composeProject
- );
- if (
- !current ||
- current.sessionName !== opts.expectedEntry.sessionName ||
- current.backend !== opts.expectedEntry.backend ||
- current.updatedAt !== opts.expectedEntry.updatedAt ||
- current.ownershipToken !== opts.expectedEntry.ownershipToken
- ) {
- return false;
- }
- await writeLifecycleStateFile({
- projectDir: opts.projectDir,
- state: {
- entries: state.filter(
- (entry) => entry.composeProject !== opts.expectedEntry.composeProject
- ),
- },
+ return await mutateLifecycleState(opts.projectDir, async () => {
+ const state = await readLifecycleState({
+ projectDir: opts.projectDir,
+ stateLocked: true,
+ });
+ const current = state.find(
+ (entry) => entry.composeProject === opts.expectedEntry.composeProject
+ );
+ if (
+ !current ||
+ current.sessionName !== opts.expectedEntry.sessionName ||
+ current.backend !== opts.expectedEntry.backend ||
+ current.updatedAt !== opts.expectedEntry.updatedAt ||
+ current.cleanupEpoch !== opts.expectedEntry.cleanupEpoch ||
+ current.ownershipToken !== opts.expectedEntry.ownershipToken
+ ) {
+ return false;
+ }
+ await writeLifecycleStateFile({
+ projectDir: opts.projectDir,
+ state: {
+ entries: state.filter(
+ (entry) => entry.composeProject !== opts.expectedEntry.composeProject
+ ),
+ },
+ });
+ return true;
});
- return true;
}
function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null {
@@ -286,7 +372,13 @@ function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null {
) {
return null;
}
- if (!(backendRaw === "tmux" || backendRaw === "zellij")) {
+ if (
+ !(
+ backendRaw === "tmux" ||
+ backendRaw === "zellij" ||
+ backendRaw === "supervisor"
+ )
+ ) {
return null;
}
const branchRaw = value.branch;
@@ -301,10 +393,15 @@ function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null {
const processes: LifecycleStateProcess[] = [];
for (const item of processesRaw) {
const parsed = parseLifecycleStateProcess(item);
- if (parsed) {
- processes.push(parsed);
+ if (!parsed) {
+ return null;
}
+ processes.push(parsed);
}
+ const supervisorPid = parseLifecycleOptionalPositiveInteger(
+ value.supervisorPid
+ );
+ const cleanupEpoch = getString(value, "cleanupEpoch")?.trim();
return {
composeProject,
projectName,
@@ -315,6 +412,15 @@ function parseLifecycleStateEntry(value: unknown): LifecycleStateEntry | null {
...(definitionHash ? { definitionHash } : {}),
processes,
updatedAt,
+ ...(supervisorPid ? { supervisorPid } : {}),
+ ...(typeof value.supervisorRoot === "string" &&
+ value.supervisorRoot.startsWith("/")
+ ? { supervisorRoot: value.supervisorRoot }
+ : {}),
+ ...(typeof value.supervisorLaunchAttempted === "boolean"
+ ? { supervisorLaunchAttempted: value.supervisorLaunchAttempted }
+ : {}),
+ ...(cleanupEpoch ? { cleanupEpoch } : {}),
};
}
@@ -335,6 +441,12 @@ function parseLifecycleStateProcess(
const processGroupId = parseLifecycleOptionalPositiveInteger(
value.processGroupId
);
+ const guardPid = parseLifecycleOptionalPositiveInteger(value.guardPid);
+ const guardStartTime = parseLifecycleOptionalPositiveInteger(
+ value.guardStartTime
+ );
+ const commandPid = parseLifecycleOptionalPositiveInteger(value.commandPid);
+ const bootId = getString(value, "bootId");
const processGroupBirth = getString(value, "processGroupBirth")?.trim();
return {
name,
@@ -343,6 +455,10 @@ function parseLifecycleStateProcess(
...(panePid ? { panePid } : {}),
...(paneBirth ? { paneBirth } : {}),
...(processGroupId ? { processGroupId } : {}),
+ ...(guardPid ? { guardPid } : {}),
+ ...(guardStartTime ? { guardStartTime } : {}),
+ ...(commandPid ? { commandPid } : {}),
+ ...(bootId ? { bootId } : {}),
...(processGroupBirth ? { processGroupBirth } : {}),
};
}
@@ -366,8 +482,20 @@ async function writeLifecycleStateFile(opts: {
}): Promise {
const root = resolveLifecycleRootDir({ projectDir: opts.projectDir });
const path = resolveLifecycleStatePath({ projectDir: opts.projectDir });
+ const text = `${JSON.stringify(opts.state, null, 2)}\n`;
+ if (Buffer.byteLength(text) > 1024 * 1024) {
+ throw lifecycleStateRefused();
+ }
await ensurePrivateDirectory(root);
- await writeTextFile(path, `${JSON.stringify(opts.state, null, 2)}\n`);
+ await verifyLifecycleStateTarget(path);
+ await replacePrivateFileAtomic({
+ path,
+ directories: [root],
+ staging: "sibling",
+ text,
+ refusal: lifecycleStateRefused,
+ beforeRename: () => verifyLifecycleStateTarget(path),
+ });
}
function sanitizeLifecycleToken(value: string): string {
@@ -473,3 +601,53 @@ export async function drainLifecycleOutput(opts: {
});
}
}
+
+/** Durable marker written once a supervisor entry has been published. */
+async function supervisorStateRecorded(root: string): Promise {
+ const marker = await lstat(resolve(root, "supervisor-state")).catch(
+ (error: unknown) => {
+ if (isRecord(error) && error.code === "ENOENT") {
+ return null;
+ }
+ throw error;
+ }
+ );
+ return marker !== null;
+}
+
+function lifecycleStateRefused(): Error {
+ return new Error(
+ "Lifecycle ownership state at .hack/.internal/lifecycle/state.json is invalid or unsafe; kept existing state. Inspect the file and stop verified leftover processes before moving it aside; see docs/lifecycle.md. Values omitted."
+ );
+}
+
+async function verifyLifecycleStateTarget(path: string): Promise {
+ const info = await lstat(path).catch((error: unknown) => {
+ if (isRecord(error) && error.code === "ENOENT") {
+ return null;
+ }
+ throw error;
+ });
+ // A concurrent atomic replacement can unlink the observed inode before
+ // the asynchronous stat result is delivered (nlink=0). That is not an
+ // alias; the replaceable reader separately binds its actual opened file.
+ if (
+ info &&
+ (!info.isFile() || info.uid !== process.getuid?.() || info.nlink > 1)
+ ) {
+ throw lifecycleStateRefused();
+ }
+}
+
+async function mutateLifecycleState(
+ projectDir: string,
+ run: () => Promise
+): Promise {
+ const root = resolveLifecycleRootDir({ projectDir });
+ await ensurePrivateDirectory(root);
+ return await withProjectsRegistryLock({
+ lockPath: resolve(root, "state.lock"),
+ label: "Lifecycle state",
+ run,
+ });
+}
diff --git a/src/lib/lifecycle-supervisor.ts b/src/lib/lifecycle-supervisor.ts
new file mode 100644
index 000000000..0a9e751da
--- /dev/null
+++ b/src/lib/lifecycle-supervisor.ts
@@ -0,0 +1,503 @@
+import { createHash } from "node:crypto";
+import { constants, realpathSync } from "node:fs";
+import { access, lstat } from "node:fs/promises";
+import { basename, dirname, isAbsolute, relative, resolve } from "node:path";
+import { resolveGlobalHackDir } from "./config-paths.ts";
+import { isRecord } from "./guards.ts";
+import {
+ readLifecycleState,
+ resolveLifecycleLogPath,
+} from "./lifecycle-runtime.ts";
+import { ensurePrivateDirectory } from "./private-file.ts";
+import { withProjectsRegistryLock } from "./projects-registry-lock.ts";
+
+export type LifecycleSupervisorProcess = {
+ readonly name: string;
+ readonly argv: readonly string[];
+ readonly cwd: string;
+ readonly env: Readonly>;
+};
+
+export type LifecycleSupervisorState = {
+ readonly version: 1;
+ readonly name: string;
+ readonly ownerToken: string;
+ readonly guardPid: number;
+ readonly processGroupId: number;
+ readonly guardStartTime: number;
+ readonly bootId: string;
+ readonly uid: number;
+ readonly commandPid: number | null;
+ readonly phase: string;
+ readonly exitCode: number | null;
+ readonly signal: number | null;
+};
+
+export type LifecycleSupervisorReceipt = {
+ readonly version: 1;
+ readonly supervisorPid: number;
+ readonly outcome: "ready" | "observed" | "stopped" | "unknown" | "refused";
+ readonly processes: readonly LifecycleSupervisorState[];
+ readonly liveProcessNames: readonly string[];
+ readonly unknownProcessNames: readonly string[];
+};
+
+type Instance = {
+ readonly projectDir: string;
+ readonly composeProject: string;
+ readonly ownershipToken: string;
+};
+
+const LIMIT = 1024 * 1024;
+const REQUEST_TIMEOUT_MS = 35_000;
+
+/** Control capabilities must never be published through checkout bind mounts. */
+export function resolveLifecycleSupervisorRoot(opts: {
+ readonly projectDir: string;
+ readonly composeProject: string;
+}): string {
+ const hostRoot = canonicalPath(
+ resolve(resolveGlobalHackDir(), "lifecycle-supervisors")
+ );
+ const inside = relative(dirname(realpathSync(opts.projectDir)), hostRoot);
+ if (inside === "" || !(inside.startsWith("../") || isAbsolute(inside))) {
+ throw new Error(
+ "Lifecycle supervisor state must be outside the project checkout."
+ );
+ }
+ const key = createHash("sha256")
+ .update(
+ JSON.stringify([realpathSync(opts.projectDir), opts.composeProject])
+ )
+ .digest("hex");
+ return resolve(hostRoot, key);
+}
+
+function canonicalPath(path: string): string {
+ try {
+ return realpathSync(path);
+ } catch (error) {
+ if (
+ !(isRecord(error) && error.code === "ENOENT") ||
+ dirname(path) === path
+ ) {
+ throw error;
+ }
+ return resolve(canonicalPath(dirname(path)), basename(path));
+ }
+}
+
+/** Resolve only the current installation or an explicit test executable. */
+export function resolveLifecycleSupervisorBinary(): string {
+ const selected = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+ if (selected !== undefined && !Bun.main.startsWith("/$bunfs/")) {
+ if (
+ !(selected.trim() && selected.startsWith("/")) ||
+ selected.includes("\0")
+ ) {
+ throw new Error(
+ "HACK_LIFECYCLE_SUPERVISOR_BINARY must name an absolute executable path."
+ );
+ }
+ return selected;
+ }
+ return Bun.main.startsWith("/$bunfs/")
+ ? resolve(dirname(process.execPath), "hack-lifecycle-supervisor")
+ : resolve(import.meta.dir, "../../dist/hack-lifecycle-supervisor");
+}
+
+export const LIFECYCLE_SUPERVISOR_VERSION =
+ "hack-lifecycle-supervisor 0.1.0 protocol=1 incremental=1";
+
+/** Validate the public, value-free handshake before publishing any ownership. */
+export async function preflightLifecycleSupervisor(): Promise<{
+ path: string;
+ version: string;
+}> {
+ const path = resolveLifecycleSupervisorBinary();
+ try {
+ await access(path, constants.X_OK);
+ if (!(await lstat(realpathSync(path))).isFile()) {
+ throw failure();
+ }
+ const child = Bun.spawn([path, "--version"], {
+ env: {},
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "ignore",
+ timeout: 2000,
+ });
+ const version = (await new Response(child.stdout).text()).trim();
+ if (
+ (await child.exited) !== 0 ||
+ version !== LIFECYCLE_SUPERVISOR_VERSION
+ ) {
+ throw failure();
+ }
+ return { path, version };
+ } catch {
+ throw new Error(
+ "Lifecycle supervisor executable is missing, incompatible or not executable; install the matching companion beside this CLI before opting in."
+ );
+ }
+}
+
+async function recordedRoot(
+ opts: Instance
+): Promise<{ root: string; neverStarted: boolean }> {
+ const entry = (await readLifecycleState(opts)).find(
+ (entry) => entry.composeProject === opts.composeProject
+ );
+ const root = entry?.supervisorRoot ?? resolveLifecycleSupervisorRoot(opts);
+ const inside = relative(
+ dirname(realpathSync(opts.projectDir)),
+ canonicalPath(root)
+ );
+ if (inside === "" || !(inside.startsWith("../") || isAbsolute(inside))) {
+ throw failure();
+ }
+ return {
+ root,
+ neverStarted:
+ !entry ||
+ (entry.supervisorLaunchAttempted === false &&
+ !entry.supervisorPid &&
+ entry.processes.length === 0),
+ };
+}
+
+/** Native observations, never a persisted `running` phase, establish liveness. */
+export async function inspectLifecycleSupervisor(
+ opts: Instance
+): Promise {
+ if (await lifecycleSupervisorProvenUnstarted(opts)) {
+ return {
+ version: 1,
+ supervisorPid: 0,
+ outcome: "observed",
+ processes: [],
+ liveProcessNames: [],
+ unknownProcessNames: [],
+ };
+ }
+ return await request({ ...opts, action: "status" });
+}
+
+/** A failed or unknown stop keeps the caller's ownership and dependency records. */
+export async function stopLifecycleSupervisor(
+ opts: Instance
+): Promise {
+ if (await lifecycleSupervisorProvenUnstarted(opts)) {
+ return {
+ version: 1,
+ supervisorPid: 0,
+ outcome: "stopped",
+ processes: [],
+ liveProcessNames: [],
+ unknownProcessNames: [],
+ };
+ }
+ return await request({ ...opts, action: "stop" });
+}
+
+async function lifecycleSupervisorProvenUnstarted(
+ opts: Instance
+): Promise {
+ const recorded = await recordedRoot(opts);
+ if (recorded.neverStarted) {
+ return true;
+ }
+ try {
+ await lstat(recorded.root);
+ return false;
+ } catch (error) {
+ if (isRecord(error) && error.code === "ENOENT") {
+ // Never turn an earlier experimental checkout-local owner into absence.
+ try {
+ await lstat(`${resolveLifecycleLogPath(opts)}.supervisor`);
+ } catch (legacyError) {
+ if (isRecord(legacyError) && legacyError.code === "ENOENT") {
+ throw failure();
+ }
+ throw failure();
+ }
+ throw new Error(
+ "Stop the earlier experimental supervisor with its original CLI before upgrading."
+ );
+ }
+ throw failure();
+ }
+}
+
+/** First launch creates the project owner; later launches preserve hook ordering. */
+export async function launchLifecycleSupervisorProcess(
+ opts: Instance & {
+ readonly action: "serve" | "add";
+ readonly process: LifecycleSupervisorProcess;
+ }
+): Promise {
+ return await request({ ...opts, processes: [opts.process] });
+}
+
+async function request(
+ opts: Instance & {
+ readonly action: "serve" | "add" | "status" | "stop";
+ readonly processes?: readonly LifecycleSupervisorProcess[];
+ }
+): Promise {
+ await preflightLifecycleSupervisor();
+ const payload = new TextEncoder().encode(
+ JSON.stringify({
+ version: 1,
+ root: (await recordedRoot(opts)).root,
+ ownerToken: opts.ownershipToken,
+ ...(opts.processes
+ ? {
+ processes: opts.processes.map((process) => ({
+ ...process,
+ name: `service-${createHash("sha256").update(process.name).digest("hex")}`,
+ displayName: process.name,
+ })),
+ logPath: resolveLifecycleLogPath(opts),
+ }
+ : {}),
+ })
+ );
+ if (payload.length > LIMIT) {
+ payload.fill(0);
+ throw failure();
+ }
+ const frame = new Uint8Array(payload.length + 4);
+ new DataView(frame.buffer).setUint32(0, payload.length);
+ frame.set(payload, 4);
+ payload.fill(0);
+ let child: Bun.Subprocess<"pipe", "pipe", "ignore">;
+ try {
+ // The Rust owner calls setsid. `detached: true` would make it a group
+ // leader before that call and is deliberately absent here.
+ child = Bun.spawn([resolveLifecycleSupervisorBinary(), opts.action], {
+ env: {},
+ stdin: "pipe",
+ stdout: "pipe",
+ stderr: "ignore",
+ });
+ } catch {
+ frame.fill(0);
+ throw new Error(
+ "Lifecycle supervisor is unavailable; build or install it alongside Hack."
+ );
+ }
+ child.unref();
+ let timeout: ReturnType | undefined;
+ const deadline = new Promise((_, reject) => {
+ timeout = setTimeout(() => reject(failure()), REQUEST_TIMEOUT_MS);
+ });
+ const reader = child.stdout.getReader();
+ try {
+ const result = await Promise.race([
+ (async () => {
+ await child.stdin.write(frame);
+ await child.stdin.end();
+ const receipt = await readReceipt(reader);
+ if (
+ new Set(receipt.processes.map((state) => state.name)).size !==
+ receipt.processes.length ||
+ (opts.processes &&
+ (receipt.supervisorPid <= 1 ||
+ opts.processes.some(
+ (process) =>
+ !receipt.processes.some(
+ (state) => state.name === process.name
+ )
+ ))) ||
+ (opts.action === "stop" &&
+ (receipt.liveProcessNames.length > 0 ||
+ receipt.unknownProcessNames.length > 0))
+ ) {
+ throw failure();
+ }
+ if (
+ receipt.processes.some(
+ (state) => state.ownerToken !== opts.ownershipToken
+ ) ||
+ receipt.liveProcessNames.some(
+ (name) => !receipt.processes.some((state) => state.name === name)
+ ) ||
+ receipt.unknownProcessNames.some(
+ (name) => !receipt.processes.some((state) => state.name === name)
+ )
+ ) {
+ throw failure();
+ }
+ if (opts.action !== "serve") {
+ const exitCode = await child.exited;
+ if (exitCode !== 0) {
+ throw failure();
+ }
+ }
+ const expected = {
+ status: "observed",
+ stop: "stopped",
+ serve: "ready",
+ add: "ready",
+ }[opts.action];
+ if (receipt.outcome !== expected) {
+ throw failure();
+ }
+ return receipt;
+ })(),
+ deadline,
+ ]);
+ return result;
+ } catch {
+ try {
+ child.kill("SIGTERM");
+ } catch {
+ /* The finite client may already have exited. */
+ }
+ throw failure();
+ } finally {
+ if (timeout !== undefined) {
+ clearTimeout(timeout);
+ }
+ frame.fill(0);
+ await reader.cancel().catch(() => undefined);
+ reader.releaseLock();
+ }
+}
+
+/** Frame completion is explicit; a caller never waits for a detached owner's EOF. */
+async function readReceipt(
+ reader: ReadableStreamDefaultReader
+): Promise {
+ let bytes = new Uint8Array(0);
+ let length: number | undefined;
+ for (;;) {
+ const next = await reader.read();
+ if (next.done || bytes.length + next.value.length > LIMIT + 4) {
+ throw failure();
+ }
+ const combined = new Uint8Array(bytes.length + next.value.length);
+ combined.set(bytes);
+ combined.set(next.value, bytes.length);
+ bytes = combined;
+ if (length === undefined && bytes.length >= 4) {
+ length = new DataView(bytes.buffer).getUint32(0);
+ if (length === 0 || length > LIMIT) {
+ throw failure();
+ }
+ }
+ if (length !== undefined && bytes.length >= length + 4) {
+ const value: unknown = JSON.parse(
+ new TextDecoder("utf-8", { fatal: true }).decode(
+ bytes.subarray(4, length + 4)
+ )
+ );
+ return parseReceipt(value);
+ }
+ }
+}
+
+function integer(value: unknown, minimum = 0): value is number {
+ return (
+ typeof value === "number" && Number.isSafeInteger(value) && value >= minimum
+ );
+}
+function optionalInteger(value: unknown): number | null {
+ if (value === null || value === undefined) {
+ return null;
+ }
+ if (!integer(value)) {
+ throw failure();
+ }
+ return value;
+}
+function strings(value: unknown): readonly string[] {
+ if (
+ !(Array.isArray(value) && value.every((item) => typeof item === "string"))
+ ) {
+ throw failure();
+ }
+ return value;
+}
+function state(value: unknown): LifecycleSupervisorState {
+ if (
+ !isRecord(value) ||
+ value.version !== 1 ||
+ typeof value.name !== "string" ||
+ typeof value.ownerToken !== "string" ||
+ !integer(value.guardPid, 2) ||
+ value.processGroupId !== value.guardPid ||
+ !integer(value.guardStartTime, 1) ||
+ typeof value.bootId !== "string" ||
+ !integer(value.uid) ||
+ typeof value.phase !== "string"
+ ) {
+ throw failure();
+ }
+ return {
+ version: 1,
+ name:
+ typeof value.displayName === "string" ? value.displayName : value.name,
+ ownerToken: value.ownerToken,
+ guardPid: value.guardPid,
+ processGroupId: value.guardPid,
+ guardStartTime: value.guardStartTime,
+ bootId: value.bootId,
+ uid: value.uid,
+ commandPid: optionalInteger(value.commandPid),
+ phase: value.phase,
+ exitCode: optionalInteger(value.exitCode),
+ signal: optionalInteger(value.signal),
+ };
+}
+function parseReceipt(value: unknown): LifecycleSupervisorReceipt {
+ if (
+ !isRecord(value) ||
+ value.version !== 1 ||
+ !integer(value.supervisorPid) ||
+ !Array.isArray(value.processes)
+ ) {
+ throw failure();
+ }
+ const outcome = value.outcome;
+ if (
+ outcome !== "ready" &&
+ outcome !== "observed" &&
+ outcome !== "stopped" &&
+ outcome !== "unknown" &&
+ outcome !== "refused"
+ ) {
+ throw failure();
+ }
+ return {
+ version: 1,
+ supervisorPid: value.supervisorPid,
+ outcome,
+ processes: value.processes.map(state),
+ liveProcessNames: strings(value.liveProcessNames),
+ unknownProcessNames: strings(value.unknownProcessNames),
+ };
+}
+function failure(): Error {
+ return new Error(
+ "Lifecycle supervisor could not confirm the operation; ownership was retained. Values omitted."
+ );
+}
+
+/** Serialize CLI publication and native effects for one instance. Native locks
+ * remain independent and protect the binary's own state after CLI loss. */
+export async function withLifecycleSupervisorControllerLock(opts: {
+ readonly projectDir: string;
+ readonly composeProject: string;
+ readonly run: () => Promise;
+}): Promise {
+ const logPath = resolveLifecycleLogPath(opts);
+ await ensurePrivateDirectory(dirname(logPath));
+ return await withProjectsRegistryLock({
+ lockPath: `${logPath}.controller.lock`,
+ label: "lifecycle supervisor controller",
+ timeoutMs: 40_000,
+ run: opts.run,
+ });
+}
diff --git a/src/lib/project-lifecycle-controller.ts b/src/lib/project-lifecycle-controller.ts
index 072902f4c..21e58709b 100644
--- a/src/lib/project-lifecycle-controller.ts
+++ b/src/lib/project-lifecycle-controller.ts
@@ -25,6 +25,10 @@ import {
resolveLifecycleLogPath,
upsertLifecycleStateEntry,
} from "./lifecycle-runtime.ts";
+import {
+ stopLifecycleSupervisor,
+ withLifecycleSupervisorControllerLock,
+} from "./lifecycle-supervisor.ts";
import { readProcessIdentities } from "./process-tree.ts";
import {
type ProjectLifecycleCommand,
@@ -50,6 +54,7 @@ import {
inspectListeningTcpPorts,
resolveLifecycleSingletonDecision,
} from "./project-lifecycle-singleton.ts";
+import { createSupervisorLifecycleController } from "./project-lifecycle-supervisor.ts";
import type { SharedHelperSessionOwnership } from "./project-shared-helpers.ts";
import { exec } from "./shell.ts";
export type LifecycleControllerProject = {
@@ -151,6 +156,16 @@ async function reconcileChangedLifecycleBackend(opts: {
if (!opts.entry || opts.entry.backend === opts.resolvedBackend) {
return opts.entry;
}
+ if (opts.entry.backend === "supervisor") {
+ await stopLifecycleProcessController({
+ project: { projectRoot: opts.projectRoot, projectDir: opts.projectDir },
+ projectName: opts.entry.projectName,
+ branch: opts.entry.branch,
+ composeProject: opts.composeProject,
+ definitionHash: opts.definitionHash,
+ });
+ return null;
+ }
const previousBackend = opts.mux.backends.get(opts.entry.backend);
if (!previousBackend?.available) {
throw new Error(
@@ -233,6 +248,7 @@ function describeSessionCreateFailure(opts: {
/** The shared Hack mux owner. Callers supply selection; this owner retains session/token checks. */
export function createLifecycleProcessController(opts: {
+ readonly runner?: "mux" | "supervisor";
readonly mux: () => Promise;
readonly project: LifecycleControllerProject;
readonly projectName: string;
@@ -287,6 +303,16 @@ export function createLifecycleProcessController(opts: {
getOperationCleanup: () => LifecycleOperationCleanup | null;
getSignalCleanup: () => { readonly dispose: () => void } | null;
} {
+ if (opts.runner === "supervisor") {
+ return createSupervisorLifecycleController({
+ ...opts,
+ stopExisting: () =>
+ stopLifecycleProcessController({
+ ...opts,
+ supervisorControllerLocked: true,
+ }),
+ });
+ }
const sessionName = resolveLifecycleSessionName({
projectName: opts.projectName,
branch: opts.branch,
@@ -958,7 +984,7 @@ export class LifecycleSessionDisplacedError extends Error {
}
}
-async function resolveLifecycleSingletonDecisionForProcess(opts: {
+export async function resolveLifecycleSingletonDecisionForProcess(opts: {
readonly process: ProjectLifecycleProcess;
readonly projectDir: string;
readonly composeProject: string;
@@ -1156,6 +1182,8 @@ async function startLifecycleProcess(opts: {
}
export async function stopLifecycleProcessController(opts: {
+ /** Internal: the caller already serializes this supervisor transition. */
+ readonly supervisorControllerLocked?: boolean;
readonly project: LifecycleControllerProject;
readonly definitionHash: string;
readonly projectName: string;
@@ -1181,6 +1209,10 @@ export async function stopLifecycleProcessController(opts: {
(entry) => entry.composeProject === opts.composeProject
) ?? null;
+ if (lifecycleEntry?.backend === "supervisor") {
+ await stopSupervisorEntry(opts, lifecycleEntry);
+ return;
+ }
const backend = lifecycleEntry
? (opts.mux ? (await opts.mux()).backends : getMuxBackends()).get(
lifecycleEntry.backend
@@ -1263,6 +1295,40 @@ export async function stopLifecycleProcessController(opts: {
}
}
+async function stopSupervisorEntry(
+ opts: Parameters[0],
+ lifecycleEntry: LifecycleStateEntry
+): Promise {
+ if (!opts.supervisorControllerLocked) {
+ return await withLifecycleSupervisorControllerLock({
+ projectDir: opts.project.projectDir,
+ composeProject: opts.composeProject,
+ run: () =>
+ stopLifecycleProcessController({
+ ...opts,
+ supervisorControllerLocked: true,
+ }),
+ });
+ }
+ if (!lifecycleEntry.ownershipToken) {
+ throw new Error(
+ "Lifecycle supervisor ownership is missing; cleanup was refused."
+ );
+ }
+ await opts.assertFresh?.();
+ await stopLifecycleSupervisor({
+ projectDir: opts.project.projectDir,
+ composeProject: opts.composeProject,
+ ownershipToken: lifecycleEntry.ownershipToken,
+ });
+ await opts.assertFresh?.();
+ await removeLifecycleStateEntryIfOwned({
+ projectDir: opts.project.projectDir,
+ composeProject: opts.composeProject,
+ ownershipToken: lifecycleEntry.ownershipToken,
+ });
+}
+
/** Missing mux state grants no authority to kill a possibly reused group ID. */
async function assertAbsentSessionGroupsRetired(opts: {
readonly entry: LifecycleStateEntry;
diff --git a/src/lib/project-lifecycle-hygiene.ts b/src/lib/project-lifecycle-hygiene.ts
index d6a86ab10..a98da9ea4 100644
--- a/src/lib/project-lifecycle-hygiene.ts
+++ b/src/lib/project-lifecycle-hygiene.ts
@@ -10,6 +10,12 @@ import {
readLifecycleState,
removeLifecycleStateEntryIfMatching,
} from "./lifecycle-runtime.ts";
+import {
+ inspectLifecycleSupervisor,
+ type LifecycleSupervisorReceipt,
+ stopLifecycleSupervisor,
+ withLifecycleSupervisorControllerLock,
+} from "./lifecycle-supervisor.ts";
import { readProcessIdentities } from "./process-tree.ts";
import {
createLifecycleSessionRetirement,
@@ -80,8 +86,12 @@ export async function inspectProjectLifecycleHygiene(opts: {
const [sessionsByBackend, snapshot, activeComposeProjects] =
await Promise.all([
- listSessionsByBackend(),
- readProcessSnapshot(),
+ entries.some((entry) => entry.backend !== "supervisor")
+ ? listSessionsByBackend()
+ : Promise.resolve(new Map>()),
+ entries.some((entry) => entry.backend !== "supervisor")
+ ? readProcessSnapshot()
+ : Promise.resolve([]),
readActiveComposeProjects({ projectDir: opts.projectDir }),
]);
const stateInspection = inspectLifecycleStateEntries({
@@ -94,14 +104,44 @@ export async function inspectProjectLifecycleHygiene(opts: {
projectRoot: opts.projectRoot,
});
const sessionHygiene = classifyLifecycleSessionHygiene({
- entries,
+ entries: entries.filter((entry) => entry.backend !== "supervisor"),
sessionInspections,
activeComposeProjects,
});
+ const supervisorReceipts = new Map<
+ string,
+ LifecycleSupervisorReceipt | null
+ >();
+ for (const entry of entries.filter((item) => item.backend === "supervisor")) {
+ const receipt = entry.ownershipToken
+ ? await inspectLifecycleSupervisor({
+ projectDir: opts.projectDir,
+ composeProject: entry.composeProject,
+ ownershipToken: entry.ownershipToken,
+ }).catch(() => null)
+ : null;
+ supervisorReceipts.set(entry.composeProject, receipt);
+ }
+ const supervisorInspection = classifySupervisorLifecycleHygiene({
+ entries,
+ receipts: supervisorReceipts,
+ activeComposeProjects,
+ });
return {
- ...stateInspection,
- ...sessionHygiene,
+ staleEntries: [
+ ...stateInspection.staleEntries,
+ ...supervisorInspection.staleEntries,
+ ],
+ orphanedProcessGroups: stateInspection.orphanedProcessGroups,
+ orphanedSessions: [
+ ...sessionHygiene.orphanedSessions,
+ ...supervisorInspection.orphanedSessions,
+ ],
+ unverifiedSessions: [
+ ...sessionHygiene.unverifiedSessions,
+ ...supervisorInspection.unverifiedSessions,
+ ],
runtimeAvailable: activeComposeProjects !== null,
};
}
@@ -119,6 +159,11 @@ export function inspectLifecycleStateEntries(opts: {
const orphanedProcessGroups = new Set();
for (const entry of opts.entries) {
+ // Native status owns the supervisor's identity/liveness proof. Absence
+ // from a mux inventory never grants stale-state or cleanup authority.
+ if (entry.backend === "supervisor") {
+ continue;
+ }
const sessions = opts.sessionsByBackend.get(entry.backend);
const sessionPresent = sessions?.has(entry.sessionName) ?? false;
if (sessionPresent) {
@@ -150,6 +195,61 @@ export function inspectLifecycleStateEntries(opts: {
};
}
+/** Classify native read-only evidence without treating unknown as absence. */
+export function classifySupervisorLifecycleHygiene(opts: {
+ readonly entries: readonly LifecycleStateEntry[];
+ readonly receipts: ReadonlyMap;
+ readonly activeComposeProjects: ReadonlySet | null;
+ readonly now?: Date;
+}): Pick<
+ LifecycleHygieneInspection,
+ "staleEntries" | "orphanedSessions" | "unverifiedSessions"
+> {
+ const staleEntries: StaleLifecycleStateEntry[] = [];
+ const orphanedSessions: OrphanedLifecycleSession[] = [];
+ const unverifiedSessions: UnverifiedLifecycleSession[] = [];
+ for (const entry of opts.entries) {
+ if (entry.backend !== "supervisor") {
+ continue;
+ }
+ const receipt = opts.receipts.get(entry.composeProject);
+ if (
+ !receipt ||
+ receipt.unknownProcessNames.length > 0 ||
+ receipt.outcome !== "observed"
+ ) {
+ unverifiedSessions.push({
+ entry,
+ reason:
+ "Lifecycle supervisor ownership or liveness could not be verified.",
+ });
+ continue;
+ }
+ if (receipt.liveProcessNames.length === 0) {
+ staleEntries.push({ entry, liveProcessGroups: [] });
+ continue;
+ }
+ if (
+ opts.activeComposeProjects === null ||
+ opts.activeComposeProjects.has(entry.composeProject)
+ ) {
+ continue;
+ }
+ const age =
+ (opts.now ?? new Date()).getTime() - Date.parse(entry.updatedAt);
+ if (!Number.isFinite(age) || age < LIFECYCLE_SESSION_ORPHAN_GRACE_MS) {
+ unverifiedSessions.push({
+ entry,
+ reason:
+ "Lifecycle supervisor is recent enough to belong to an in-flight startup.",
+ });
+ continue;
+ }
+ orphanedSessions.push({ entry, classification: "owned-healthy" });
+ }
+ return { staleEntries, orphanedSessions, unverifiedSessions };
+}
+
export function classifyLifecycleSessionHygiene(opts: {
readonly entries: readonly LifecycleStateEntry[];
readonly sessionInspections: ReadonlyMap;
@@ -220,6 +320,8 @@ export async function repairProjectLifecycleSessions(opts: {
readonly retireSession?: ReturnType<
typeof createLifecycleSessionRetirement
>;
+ readonly inspectSupervisor?: typeof inspectLifecycleSupervisor;
+ readonly stopSupervisor?: typeof stopLifecycleSupervisor;
};
}): Promise {
const {
@@ -229,6 +331,8 @@ export async function repairProjectLifecycleSessions(opts: {
readPanes = readRepairPanes,
readIdentities = readProcessIdentities,
retireSession,
+ inspectSupervisor = inspectLifecycleSupervisor,
+ stopSupervisor = stopLifecycleSupervisor,
} = opts.boundary ?? {};
const initial = await inspect(opts);
const repairedSessions: string[] = [];
@@ -245,6 +349,25 @@ export async function repairProjectLifecycleSessions(opts: {
);
continue;
}
+ if (!sameRepairEntry(candidate.entry, orphan.entry)) {
+ failures.push(
+ `${candidate.entry.sessionName}: lifecycle generation changed before repair`
+ );
+ continue;
+ }
+ if (orphan.entry.backend === "supervisor") {
+ const result = await repairSupervisorSession({
+ projectDir: opts.projectDir,
+ projectRoot: opts.projectRoot,
+ entry: orphan.entry,
+ inspect,
+ inspectSupervisor,
+ stopSupervisor,
+ });
+ repairedSessions.push(...result.repairedSessions);
+ failures.push(...result.failures);
+ continue;
+ }
const backend = backends().get(orphan.entry.backend);
if (!backend?.available) {
failures.push(`${orphan.entry.sessionName}: mux backend unavailable`);
@@ -257,10 +380,7 @@ export async function repairProjectLifecycleSessions(opts: {
expectedProjectRoot: opts.projectRoot,
expectedDefinitionHash: orphan.entry.definitionHash ?? "",
});
- if (
- inspection.decision.kind === "block" ||
- inspection.decision.kind === "create"
- ) {
+ if (["block", "create"].includes(inspection.decision.kind)) {
failures.push(
`${orphan.entry.sessionName}: ownership proof changed before repair`
);
@@ -309,6 +429,104 @@ export async function repairProjectLifecycleSessions(opts: {
return { repairedSessions, failures };
}
+async function repairSupervisorSession(opts: {
+ readonly projectDir: string;
+ readonly projectRoot: string;
+ readonly entry: LifecycleStateEntry;
+ readonly inspect: typeof inspectProjectLifecycleHygiene;
+ readonly inspectSupervisor: typeof inspectLifecycleSupervisor;
+ readonly stopSupervisor: typeof stopLifecycleSupervisor;
+}): Promise {
+ return await withLifecycleSupervisorControllerLock({
+ projectDir: opts.projectDir,
+ composeProject: opts.entry.composeProject,
+ run: () => repairSupervisorSessionLocked(opts),
+ });
+}
+
+async function repairSupervisorSessionLocked(opts: {
+ readonly projectDir: string;
+ readonly projectRoot: string;
+ readonly entry: LifecycleStateEntry;
+ readonly inspect: typeof inspectProjectLifecycleHygiene;
+ readonly inspectSupervisor: typeof inspectLifecycleSupervisor;
+ readonly stopSupervisor: typeof stopLifecycleSupervisor;
+}): Promise {
+ try {
+ const ownershipToken = opts.entry.ownershipToken;
+ if (!ownershipToken) {
+ throw new Error("Lifecycle supervisor ownership token is missing");
+ }
+ const control = {
+ projectDir: opts.projectDir,
+ composeProject: opts.entry.composeProject,
+ ownershipToken,
+ };
+ const observed = await opts.inspectSupervisor(control);
+ if (
+ observed.outcome !== "observed" ||
+ observed.unknownProcessNames.length > 0
+ ) {
+ throw new Error("Lifecycle supervisor ownership is unverified");
+ }
+ // Native observation can wait on a process census. Refresh runtime absence
+ // and startup grace after that wait, then fence the exact stored generation.
+ const fresh = await opts.inspect({
+ projectDir: opts.projectDir,
+ projectRoot: opts.projectRoot,
+ });
+ if (
+ !fresh.orphanedSessions.some((orphan) =>
+ sameRepairEntry(orphan.entry, opts.entry)
+ )
+ ) {
+ throw new Error("Lifecycle supervisor is no longer runtime-orphaned");
+ }
+ await assertRepairEntryCurrent({
+ projectDir: opts.projectDir,
+ entry: opts.entry,
+ });
+ const stopped = await opts.stopSupervisor(control);
+ if (
+ stopped.outcome !== "stopped" ||
+ stopped.liveProcessNames.length > 0 ||
+ stopped.unknownProcessNames.length > 0
+ ) {
+ throw new Error("Lifecycle supervisor stop was not confirmed");
+ }
+ if (
+ !(await removeLifecycleStateEntryIfMatching({
+ projectDir: opts.projectDir,
+ expectedEntry: opts.entry,
+ }))
+ ) {
+ throw new Error("Lifecycle state changed after supervisor stop");
+ }
+ return { repairedSessions: [opts.entry.sessionName], failures: [] };
+ } catch {
+ return {
+ repairedSessions: [],
+ failures: [
+ `${opts.entry.sessionName}: supervisor retirement unconfirmed; kept ownership state`,
+ ],
+ };
+ }
+}
+
+function sameRepairEntry(
+ left: LifecycleStateEntry,
+ right: LifecycleStateEntry
+): boolean {
+ return (
+ left.composeProject === right.composeProject &&
+ left.backend === right.backend &&
+ left.sessionName === right.sessionName &&
+ left.ownershipToken === right.ownershipToken &&
+ left.cleanupEpoch === right.cleanupEpoch &&
+ left.updatedAt === right.updatedAt
+ );
+}
+
function repairFailureMessage(error: unknown): string {
return error instanceof Error
? error.message
@@ -436,6 +654,7 @@ async function assertRepairEntryCurrent(opts: {
current.sessionName !== opts.entry.sessionName ||
current.backend !== opts.entry.backend ||
current.updatedAt !== opts.entry.updatedAt ||
+ current.cleanupEpoch !== opts.entry.cleanupEpoch ||
current.ownershipToken !== opts.entry.ownershipToken
) {
throw new Error("Lifecycle ownership changed before repair retirement");
@@ -446,9 +665,15 @@ async function inspectOwnedLifecycleSessions(opts: {
readonly entries: readonly LifecycleStateEntry[];
readonly projectRoot: string;
}): Promise> {
- const backends = getMuxBackends();
const inspections = new Map();
+ if (!opts.entries.some((entry) => entry.backend !== "supervisor")) {
+ return inspections;
+ }
+ const backends = getMuxBackends();
for (const entry of opts.entries) {
+ if (entry.backend === "supervisor") {
+ continue;
+ }
const backend = backends.get(entry.backend);
if (!backend?.available) {
continue;
diff --git a/src/lib/project-lifecycle-supervisor.ts b/src/lib/project-lifecycle-supervisor.ts
new file mode 100644
index 000000000..881f87217
--- /dev/null
+++ b/src/lib/project-lifecycle-supervisor.ts
@@ -0,0 +1,461 @@
+import { dirname } from "node:path";
+import { logger } from "../ui/logger.ts";
+import { childEnvironment } from "./child-environment.ts";
+import {
+ ensurePrivateLifecycleLog,
+ type LifecycleStateEntry,
+ type LifecycleStateProcess,
+ readLifecycleState,
+ removeLifecycleStateEntryIfOwned,
+ resolveLifecycleLogPath,
+ upsertLifecycleStateEntry,
+} from "./lifecycle-runtime.ts";
+import {
+ inspectLifecycleSupervisor,
+ type LifecycleSupervisorReceipt,
+ launchLifecycleSupervisorProcess,
+ preflightLifecycleSupervisor,
+ resolveLifecycleSupervisorRoot,
+ stopLifecycleSupervisor,
+ withLifecycleSupervisorControllerLock,
+} from "./lifecycle-supervisor.ts";
+import { ensurePrivateDirectory } from "./private-file.ts";
+import type { ProjectLifecycleProcess } from "./project.ts";
+import {
+ type createLifecycleProcessController,
+ installLifecycleSignalCleanup,
+ LIFECYCLE_SESSION_RETAINED,
+ LifecycleSessionDisplacedError,
+ resolveLifecycleCwd,
+ resolveLifecycleSessionName,
+ resolveLifecycleSingletonDecisionForProcess,
+} from "./project-lifecycle-controller.ts";
+import { createLifecycleOwnershipToken } from "./project-lifecycle-sessions.ts";
+
+type Options = Parameters[0];
+type Controller = ReturnType;
+
+function hasRunningCommand(
+ receipt: LifecycleSupervisorReceipt,
+ name: string
+): boolean {
+ return (
+ receipt.liveProcessNames.includes(name) &&
+ receipt.processes.some(
+ (state) => state.name === name && state.phase === "running"
+ )
+ );
+}
+
+/** The native owner preserves the mux controller's adoption and rollback fences. */
+export function createSupervisorLifecycleController(
+ opts: Options & { readonly stopExisting: () => Promise }
+): Controller {
+ const sessionName = resolveLifecycleSessionName(opts);
+ const logPath = resolveLifecycleLogPath({
+ projectDir: opts.project.projectDir,
+ composeProject: opts.composeProject,
+ });
+ const instance = {
+ projectDir: opts.project.projectDir,
+ composeProject: opts.composeProject,
+ };
+ let token: string | null = null;
+ let created = false;
+ let decided = false;
+ let ownerPid: number | undefined;
+ let launchAttempted = false;
+ let nativeRoot: string | undefined;
+ let cleanupEpoch: string | undefined;
+ let processes: readonly LifecycleStateProcess[] = [];
+ let signals: ReturnType | null = null;
+ let mutation: Promise = Promise.resolve();
+
+ async function underLock(run: () => Promise): Promise {
+ const locked = () => withControllerLock(run);
+ return opts.sessionOwnership
+ ? await opts.sessionOwnership.withLock(locked)
+ : await locked();
+ }
+ async function withControllerLock(run: () => Promise): Promise {
+ return await withLifecycleSupervisorControllerLock({ ...instance, run });
+ }
+ async function current(): Promise {
+ return (await readLifecycleState(instance)).find(
+ (entry) => entry.composeProject === opts.composeProject
+ );
+ }
+ async function assertOwned(): Promise {
+ await opts.assertFresh?.();
+ const observed = await current();
+ if (
+ !token ||
+ observed?.ownershipToken !== token ||
+ observed.cleanupEpoch !== cleanupEpoch
+ ) {
+ created = false;
+ throw new LifecycleSessionDisplacedError(sessionName);
+ }
+ }
+ async function publish(): Promise {
+ if (!token) {
+ return;
+ }
+ await assertOwned();
+ await upsertLifecycleStateEntry({
+ projectDir: instance.projectDir,
+ entry: entry(),
+ });
+ }
+ function entry(): LifecycleStateEntry {
+ return {
+ composeProject: opts.composeProject,
+ projectName: opts.projectName,
+ branch: opts.branch,
+ sessionName,
+ backend: "supervisor",
+ ownershipToken: token ?? undefined,
+ definitionHash: opts.definitionHash,
+ processes,
+ updatedAt: new Date().toISOString(),
+ ...(ownerPid ? { supervisorPid: ownerPid } : {}),
+ ...(nativeRoot ? { supervisorRoot: nativeRoot } : {}),
+ supervisorLaunchAttempted: launchAttempted,
+ ...(cleanupEpoch ? { cleanupEpoch } : {}),
+ };
+ }
+ async function decide(): Promise {
+ if (decided) {
+ return;
+ }
+ await underLock(async () => {
+ await opts.assertFresh?.();
+ const existing = await current();
+ if (existing) {
+ if (existing.backend === "supervisor" && (await adopt(existing))) {
+ decided = true;
+ return;
+ }
+ await opts.stopExisting();
+ }
+ decided = true;
+ });
+ }
+ async function adopt(existing: LifecycleStateEntry): Promise {
+ if (!existing.ownershipToken) {
+ throw new Error(
+ "Lifecycle supervisor ownership is missing; nothing was started."
+ );
+ }
+ const status = await inspectLifecycleSupervisor({
+ ...instance,
+ ownershipToken: existing.ownershipToken,
+ });
+ if (status.unknownProcessNames.length > 0) {
+ throw new Error(
+ "Lifecycle supervisor identity could not be verified; ownership was retained."
+ );
+ }
+ if (canAdopt(existing, status)) {
+ token = existing.ownershipToken;
+ processes = existing.processes;
+ ownerPid = existing.supervisorPid;
+ nativeRoot =
+ existing.supervisorRoot ?? resolveLifecycleSupervisorRoot(instance);
+ launchAttempted = true;
+ cleanupEpoch = createLifecycleOwnershipToken();
+ await upsertLifecycleStateEntry({
+ projectDir: instance.projectDir,
+ entry: entry(),
+ });
+ await opts.sessionOwnership?.record("adopted");
+ return true;
+ }
+ return false;
+ }
+ function canAdopt(
+ existing: LifecycleStateEntry,
+ status: LifecycleSupervisorReceipt
+ ): boolean {
+ return (
+ existing.definitionHash === opts.definitionHash &&
+ existing.processes.length > 0 &&
+ existing.processes.every((process) =>
+ hasRunningCommand(status, process.name)
+ )
+ );
+ }
+ async function begin(): Promise {
+ if (token) {
+ return;
+ }
+ // decide() may have released the lock while a singleton probe ran. Never
+ // publish a new token over a start that won that interval.
+ if (await current()) {
+ throw new LifecycleSessionDisplacedError(sessionName);
+ }
+ await ensurePrivateLifecycleLog({ root: dirname(logPath), logPath });
+ await preflightLifecycleSupervisor();
+ nativeRoot = resolveLifecycleSupervisorRoot(instance);
+ await ensurePrivateDirectory(nativeRoot);
+ token = createLifecycleOwnershipToken();
+ cleanupEpoch = createLifecycleOwnershipToken();
+ created = true;
+ signals = (opts.installSignalCleanup ?? installLifecycleSignalCleanup)({
+ cleanup: abort,
+ });
+ // Publication precedes even the native owner's startup. Every error after
+ // this point leaves an owner token that down can use to inspect/retire it.
+ await upsertLifecycleStateEntry({
+ projectDir: instance.projectDir,
+ entry: entry(),
+ });
+ await opts.sessionOwnership?.record("created");
+ }
+ function accept(receipt: LifecycleSupervisorReceipt): void {
+ if (receipt.supervisorPid > 1) {
+ ownerPid = receipt.supervisorPid;
+ }
+ processes = receipt.processes.map((process) => ({
+ name: process.name,
+ windowName: process.name,
+ logPath,
+ processGroupId: process.processGroupId,
+ guardPid: process.guardPid,
+ guardStartTime: process.guardStartTime,
+ bootId: process.bootId,
+ ...(process.commandPid ? { commandPid: process.commandPid } : {}),
+ }));
+ }
+ async function start(process: ProjectLifecycleProcess): Promise {
+ await preflightLifecycleSupervisor();
+ await decide();
+ if (processes.some((started) => started.name === process.name)) {
+ await underLock(assertOwned);
+ return;
+ }
+ const singleton = await resolveLifecycleSingletonDecisionForProcess({
+ process,
+ projectDir: instance.projectDir,
+ composeProject: opts.composeProject,
+ });
+ if (singleton.kind === "adopt") {
+ logger.info({ message: singleton.message });
+ return;
+ }
+ if (singleton.kind === "fail") {
+ throw new Error(singleton.message);
+ }
+ if (opts.prepareLaunch) {
+ throw new Error(
+ "Lifecycle supervisor does not support this launch adapter."
+ );
+ }
+ await underLock(async () => {
+ await begin();
+ await assertOwned();
+ if (!token) {
+ throw new Error("Lifecycle supervisor ownership was not established.");
+ }
+ const observed = await inspectLifecycleSupervisor({
+ ...instance,
+ ownershipToken: token,
+ });
+ if (observed.liveProcessNames.includes(process.name)) {
+ if (!hasRunningCommand(observed, process.name)) {
+ throw new Error(
+ "Lifecycle command is retiring; retry after shutdown completes."
+ );
+ }
+ accept(observed);
+ await publish();
+ return;
+ }
+ if (observed.unknownProcessNames.includes(process.name)) {
+ throw new Error(
+ "Lifecycle command identity is unknown; ownership was retained."
+ );
+ }
+ const launch = {
+ ...instance,
+ ownershipToken: token,
+ process: {
+ name: process.name,
+ argv: ["/bin/sh", "-c", process.command],
+ cwd: resolveLifecycleCwd({
+ projectRoot: opts.project.projectRoot,
+ cwd: process.cwd,
+ }),
+ env: childEnvironment(undefined, opts.env),
+ },
+ };
+ launchAttempted = true;
+ await publish();
+ // An owner can naturally retire between hooks. Retry serve only after
+ // inspecting native receipts; never infer absence from a socket error.
+ let receipt: LifecycleSupervisorReceipt;
+ if (processes.length === 0) {
+ receipt = await serveAfterAbsence(launch);
+ } else {
+ try {
+ receipt = await launchLifecycleSupervisorProcess({
+ ...launch,
+ action: "add",
+ });
+ } catch {
+ receipt = await reconcileAdd(launch);
+ }
+ }
+ accept(receipt);
+ await publish();
+ });
+ }
+ async function reconcileAdd(
+ launch: Omit<
+ Parameters[0],
+ "action"
+ >
+ ): Promise {
+ await assertOwned();
+ const status = await inspectLifecycleSupervisor({
+ ...instance,
+ ownershipToken: launch.ownershipToken,
+ });
+ if (status.liveProcessNames.includes(launch.process.name)) {
+ accept(status);
+ await publish();
+ throw new Error(
+ "Lifecycle supervisor launch completed after its reply was lost; ownership was retained."
+ );
+ }
+ if (
+ status.liveProcessNames.length > 0 ||
+ status.unknownProcessNames.length > 0
+ ) {
+ throw new Error(
+ "Lifecycle supervisor launch could not be confirmed; ownership was retained."
+ );
+ }
+ return await serveAfterAbsence(launch);
+ }
+ async function serveAfterAbsence(
+ launch: Omit<
+ Parameters[0],
+ "action"
+ >
+ ): Promise {
+ for (let attempt = 0; attempt < 3; attempt += 1) {
+ await assertOwned();
+ try {
+ return await launchLifecycleSupervisorProcess({
+ ...launch,
+ action: "serve",
+ });
+ } catch {
+ const status = await inspectLifecycleSupervisor({
+ ...instance,
+ ownershipToken: launch.ownershipToken,
+ });
+ if (
+ status.liveProcessNames.length ||
+ status.unknownProcessNames.length ||
+ attempt === 2
+ ) {
+ throw new Error(
+ "Lifecycle supervisor launch could not be confirmed; ownership was retained."
+ );
+ }
+ await Bun.sleep(50);
+ }
+ }
+ throw new Error("Lifecycle supervisor launch could not be confirmed.");
+ }
+ async function stopOwned(): Promise<
+ undefined | typeof LIFECYCLE_SESSION_RETAINED
+ > {
+ if (!(created && token)) {
+ return undefined;
+ }
+ if ((await current())?.cleanupEpoch !== cleanupEpoch) {
+ created = false;
+ return undefined;
+ }
+ await assertOwned();
+ try {
+ await stopLifecycleSupervisor({ ...instance, ownershipToken: token });
+ await assertOwned();
+ await removeLifecycleStateEntryIfOwned({
+ ...instance,
+ ownershipToken: token,
+ });
+ created = false;
+ return undefined;
+ } catch {
+ logger.warn({
+ message:
+ "Could not confirm lifecycle supervisor shutdown; ownership and dependencies were retained. Run hack down to retry.",
+ });
+ return LIFECYCLE_SESSION_RETAINED;
+ }
+ }
+ async function abort(): Promise<
+ undefined | typeof LIFECYCLE_SESSION_RETAINED
+ > {
+ await mutation.catch(() => undefined);
+ if (!created) {
+ return undefined;
+ }
+ if (!opts.sessionOwnership) {
+ return await withControllerLock(stopOwned);
+ }
+ let stopped: undefined | typeof LIFECYCLE_SESSION_RETAINED;
+ let result: Awaited<
+ ReturnType["runIfOwned"]>
+ >;
+ try {
+ result = await opts.sessionOwnership.runIfOwned(async () => {
+ stopped = await withControllerLock(stopOwned);
+ });
+ } catch {
+ result = "undecidable";
+ }
+ if (result === "adopted") {
+ created = false;
+ return undefined;
+ }
+ return result === "undecidable" ? LIFECYCLE_SESSION_RETAINED : stopped;
+ }
+ async function startTracked(process: ProjectLifecycleProcess): Promise {
+ const pending = start(process);
+ mutation = pending;
+ await pending;
+ }
+ return {
+ sessionName,
+ startFromCommand: ({ command, serviceName }) =>
+ startTracked({
+ name: serviceName,
+ command: command.command,
+ ...(command.cwd ? { cwd: command.cwd } : {}),
+ ...(command.singleton ? { singleton: command.singleton } : {}),
+ }),
+ startMany: async ({ processes }) => {
+ await decide();
+ for (const process of processes) {
+ await startTracked(process);
+ }
+ },
+ finalize: async () => {
+ await underLock(publish);
+ },
+ abort: async () => {
+ const result = await abort();
+ signals?.dispose();
+ return result;
+ },
+ hasStarted: () => processes.length > 0,
+ getOperationCleanup: () =>
+ created ? () => withControllerLock(stopOwned) : null,
+ getSignalCleanup: () => signals,
+ };
+}
diff --git a/src/lib/project.ts b/src/lib/project.ts
index 261c538e3..66e4dea75 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -13,6 +13,11 @@ import {
import { parseDotEnv } from "./env.ts";
import { pathExists, readTextFile } from "./fs.ts";
import { getRecord, getString, isRecord } from "./guards.ts";
+import {
+ LIFECYCLE_RUNNER_CONFIG_ERROR,
+ type LifecycleRunner,
+ parseLifecycleRunner,
+} from "./lifecycle-runner.ts";
import {
type OpenHostPreference,
parseOpenHostPreference,
@@ -267,6 +272,8 @@ export interface ProjectConfig {
* `parseError`, which would otherwise launch services with no helper.
*/
readonly sharedHelpersError?: string;
+ /** Invalid runner selection must refuse lifecycle effects, not fall back. */
+ readonly lifecycleRunnerError?: string;
}
export interface ProjectOpenConfig {
@@ -301,6 +308,8 @@ export interface ProjectSessionsConfig {
}
export interface ProjectLifecycleConfig {
+ /** Persistent host-process runner; omitted means mux. */
+ readonly runner?: LifecycleRunner;
readonly up?: ProjectLifecycleHooks;
readonly down?: ProjectLifecycleHooks;
readonly processes?: readonly ProjectLifecycleProcess[];
@@ -563,6 +572,11 @@ function parseProjectConfigRecord(value: unknown, path: string): ProjectConfig {
const worktree = parseWorktreeConfig(getRecord(value, "worktree"));
const sessions = parseSessionsConfig(getRecord(value, "sessions"));
const lifecycleRecord = getRecord(value, "lifecycle");
+ const lifecycleRunnerError =
+ lifecycleRecord?.runner !== undefined &&
+ parseLifecycleRunner(lifecycleRecord.runner) === null
+ ? LIFECYCLE_RUNNER_CONFIG_ERROR
+ : undefined;
const lifecycleBase = parseLifecycleConfig(lifecycleRecord);
const sharedHelpers = parseProjectSharedHelpers(
lifecycleRecord?.sharedHelpers
@@ -598,6 +612,7 @@ function parseProjectConfigRecord(value: unknown, path: string): ProjectConfig {
...(parsedOwnership.parseError ? [parsedOwnership.parseError] : []),
...(parsedOpen.parseError ? [parsedOpen.parseError] : []),
...sharedHelpers.errors,
+ ...(lifecycleRunnerError ? [lifecycleRunnerError] : []),
];
return {
@@ -619,6 +634,7 @@ function parseProjectConfigRecord(value: unknown, path: string): ProjectConfig {
...(sharedHelpers.errors.length > 0
? { sharedHelpersError: sharedHelpers.errors.join(" ") }
: {}),
+ ...(lifecycleRunnerError ? { lifecycleRunnerError } : {}),
configPath: path,
};
}
@@ -829,6 +845,7 @@ function mergeLifecycleWithStartup(opts: {
const processes = [...lifecycleProcesses, ...startup.processes];
return buildLifecycleConfig({
+ runner: opts.lifecycle?.runner,
upBefore,
upAfter: lifecycleUpAfter,
downBefore: lifecycleDownBefore,
@@ -871,6 +888,7 @@ function mapStartupEntriesToLifecycle(opts: {
}
function buildLifecycleConfig(opts: {
+ readonly runner?: LifecycleRunner;
readonly upBefore: readonly ProjectLifecycleCommand[];
readonly upAfter: readonly ProjectLifecycleCommand[] | undefined;
readonly downBefore: readonly ProjectLifecycleCommand[] | undefined;
@@ -878,6 +896,7 @@ function buildLifecycleConfig(opts: {
readonly processes: readonly ProjectLifecycleProcess[];
}): ProjectLifecycleConfig | undefined {
const lifecycle: ProjectLifecycleConfig = {
+ ...(opts.runner ? { runner: opts.runner } : {}),
...(opts.upBefore.length > 0 || opts.upAfter
? {
up: {
@@ -910,8 +929,10 @@ function parseLifecycleConfig(
const up = parseLifecycleHooks(getRecord(value, "up"));
const down = parseLifecycleHooks(getRecord(value, "down"));
const processes = parseLifecycleProcesses(value.processes);
+ const runner = parseLifecycleRunner(value.runner);
const out: ProjectLifecycleConfig = {
+ ...(runner ? { runner } : {}),
...(up ? { up } : {}),
...(down ? { down } : {}),
...(processes ? { processes } : {}),
diff --git a/src/lib/runtime-projects.ts b/src/lib/runtime-projects.ts
index b5ea948b9..33b0b6c3f 100644
--- a/src/lib/runtime-projects.ts
+++ b/src/lib/runtime-projects.ts
@@ -9,6 +9,7 @@ import { pathExists } from "./fs.ts";
import { getString, isRecord } from "./guards.ts";
import { parseJsonLines } from "./json-lines.ts";
import { readLifecycleState } from "./lifecycle-runtime.ts";
+import { inspectLifecycleSupervisor } from "./lifecycle-supervisor.ts";
import {
createOperationTimings,
type OperationTimings,
@@ -376,10 +377,11 @@ export async function autoRegisterRuntimeHackProjects(opts: {
type LifecycleActivity = {
readonly sessionActive: boolean;
readonly runningWindows: ReadonlySet | null;
+ readonly unknownWindows?: ReadonlySet;
};
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Lifecycle pseudo-services are derived from several runtime sources in one normalization step.
-async function addLifecycleProcessServices(opts: {
+export async function addLifecycleProcessServices(opts: {
readonly runtime: readonly RuntimeProject[];
}): Promise {
if (opts.runtime.length === 0) {
@@ -390,13 +392,18 @@ async function addLifecycleProcessServices(opts: {
string,
Awaited>
>();
+ const unreadable = new Set();
for (const runtime of opts.runtime) {
const workingDir = runtime.workingDir;
if (!workingDir || stateByWorkingDir.has(workingDir)) {
continue;
}
- const state = await readLifecycleState({ projectDir: workingDir });
- stateByWorkingDir.set(workingDir, state);
+ try {
+ const state = await readLifecycleState({ projectDir: workingDir });
+ stateByWorkingDir.set(workingDir, state);
+ } catch {
+ unreadable.add(workingDir);
+ }
}
const tmuxWindowsBySession = new Map | null>();
@@ -409,6 +416,36 @@ async function addLifecycleProcessServices(opts: {
out.push(runtime);
continue;
}
+ if (unreadable.has(workingDir)) {
+ const services = new Map(runtime.services);
+ const service = "lifecycle-state";
+ const existing = services.get(service);
+ services.set(service, {
+ service,
+ containers: [
+ ...(existing?.containers ?? []),
+ {
+ id: `lifecycle-state-${runtime.project}`,
+ project: runtime.project,
+ service,
+ state: "unknown",
+ status: "Unknown (lifecycle: state_unreadable)",
+ name: `${runtime.project}-lifecycle-state`,
+ ports: "",
+ workingDir,
+ image: null,
+ labels: {
+ "hack.lifecycle.process": "true",
+ "hack.lifecycle.error": "state_unreadable",
+ },
+ mounts: [],
+ networks: [],
+ },
+ ],
+ });
+ out.push({ ...runtime, services });
+ continue;
+ }
const state = stateByWorkingDir.get(workingDir) ?? [];
const entry = state.find((item) => item.composeProject === runtime.project);
if (!entry) {
@@ -416,18 +453,24 @@ async function addLifecycleProcessServices(opts: {
continue;
}
- const activity = await resolveLifecycleActivity({
- backend: entry.backend,
- sessionName: entry.sessionName,
- tmuxWindowsBySession,
- getZellijSessions: async () => {
- if (zellijSessions) {
- return zellijSessions;
- }
- zellijSessions = await readZellijSessionNames();
- return zellijSessions;
- },
- });
+ const activity =
+ entry.backend === "supervisor"
+ ? await readSupervisorLifecycleActivity({
+ entry,
+ projectDir: workingDir,
+ })
+ : await resolveLifecycleActivity({
+ backend: entry.backend,
+ sessionName: entry.sessionName,
+ tmuxWindowsBySession,
+ getZellijSessions: async () => {
+ if (zellijSessions) {
+ return zellijSessions;
+ }
+ zellijSessions = await readZellijSessionNames();
+ return zellijSessions;
+ },
+ });
const services = new Map(runtime.services);
for (const process of entry.processes) {
@@ -435,8 +478,15 @@ async function addLifecycleProcessServices(opts: {
activity.runningWindows !== null
? activity.runningWindows.has(process.windowName)
: activity.sessionActive;
- const stateValue = running ? "running" : "exited";
- const status = running ? "Up (lifecycle)" : "Exited (lifecycle)";
+ const unknown = activity.unknownWindows?.has(process.windowName) === true;
+ const observedState = running ? "running" : "exited";
+ const stateValue = unknown ? "unknown" : observedState;
+ const statusByState = {
+ unknown: "Unknown (lifecycle)",
+ running: "Up (lifecycle)",
+ exited: "Exited (lifecycle)",
+ };
+ const status = statusByState[stateValue];
const id = `lifecycle-${runtime.project}-${sanitizeLifecycleToken(process.name)}`;
const syntheticContainer: RuntimeContainer = {
id,
@@ -481,6 +531,51 @@ async function addLifecycleProcessServices(opts: {
return out;
}
+/** Native observations alone qualify supervisor service liveness. */
+export async function readSupervisorLifecycleActivity(opts: {
+ readonly entry: Awaited>[number];
+ readonly projectDir: string;
+ readonly inspect?: typeof inspectLifecycleSupervisor;
+}): Promise {
+ const unknown = () => ({
+ sessionActive: false,
+ runningWindows: new Set(),
+ unknownWindows: new Set(
+ opts.entry.processes.map((process) => process.windowName)
+ ),
+ });
+ if (!opts.entry.ownershipToken) {
+ return unknown();
+ }
+ try {
+ const receipt = await (opts.inspect ?? inspectLifecycleSupervisor)({
+ projectDir: opts.projectDir,
+ composeProject: opts.entry.composeProject,
+ ownershipToken: opts.entry.ownershipToken,
+ });
+ if (receipt.outcome !== "observed") {
+ return unknown();
+ }
+ return {
+ sessionActive: receipt.liveProcessNames.length > 0,
+ runningWindows: new Set(
+ opts.entry.processes
+ .filter((process) => receipt.liveProcessNames.includes(process.name))
+ .map((process) => process.windowName)
+ ),
+ unknownWindows: new Set(
+ opts.entry.processes
+ .filter((process) =>
+ receipt.unknownProcessNames.includes(process.name)
+ )
+ .map((process) => process.windowName)
+ ),
+ };
+ } catch {
+ return unknown();
+ }
+}
+
async function resolveLifecycleActivity(opts: {
readonly backend: "tmux" | "zellij";
readonly sessionName: string;
diff --git a/src/templates.ts b/src/templates.ts
index 5704fb493..86e2ef05c 100644
--- a/src/templates.ts
+++ b/src/templates.ts
@@ -663,6 +663,13 @@ export function renderProjectConfigSchemaJson(): string {
type: "object",
additionalProperties: false,
properties: {
+ runner: {
+ type: "string",
+ enum: ["mux", "supervisor"],
+ default: "mux",
+ description:
+ "Runner for persistent host lifecycle processes. Interactive workspaces still use sessions.mux.",
+ },
up: {
type: "object",
additionalProperties: false,
diff --git a/tests/e2e/scenarios/lifecycle-host-process.ts b/tests/e2e/scenarios/lifecycle-host-process.ts
index 51ea01049..a5a88d020 100644
--- a/tests/e2e/scenarios/lifecycle-host-process.ts
+++ b/tests/e2e/scenarios/lifecycle-host-process.ts
@@ -1,4 +1,9 @@
-import { join } from "node:path";
+import { readdir } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { isRecord } from "../../../src/lib/guards.ts";
+import { resolveLifecycleSupervisorRoot } from "../../../src/lib/lifecycle-supervisor.ts";
+import { restoreEnv } from "../../helpers/env.ts";
+import { registerPrivateTmuxRoot } from "../../helpers/tmux-socket-guard.ts";
import { createMonorepoFixture } from "../fixture.ts";
import { expect, expectExit, runCommand, type Scenario } from "../harness.ts";
@@ -20,6 +25,7 @@ export const lifecycleHostProcessScenario: Scenario = {
summary: "lifecycle hooks and command-group teardown run during up/down",
run: async (ctx) => {
await requireDockerPreconditions({ ctx });
+ const supervisor = process.env.HACK_LIFECYCLE_RUNNER === "supervisor";
const markerFile = join(ctx.tempRoot, "lifecycle-marker.txt");
const envMarkerFile = join(ctx.tempRoot, "lifecycle-env-marker.txt");
@@ -33,9 +39,42 @@ export const lifecycleHostProcessScenario: Scenario = {
standaloneContainers: true,
},
});
+ if (supervisor) {
+ const binary =
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY ??
+ resolve(
+ ctx.repoRoot,
+ ".hack-local/target/lifecycle-integration/debug/hack-lifecycle-supervisor"
+ );
+ expect({
+ that: await Bun.file(binary).exists(),
+ message:
+ "supervisor host scenario requires its current-source executable",
+ });
+ // Store only the runner choice in this disposable fixture. The native
+ // executable remains an explicit test environment override.
+ await runSupervisorHostProcess({
+ ctx,
+ fixture,
+ markerFile,
+ envMarkerFile,
+ binary,
+ });
+ return;
+ }
+ const previousTmuxRoot = process.env.TMUX_TMPDIR;
+ // The run's tmux socket guard admits only registered private roots; this
+ // scenario owns its fixture root (mkdtemp, 0700) for its own tmux server.
+ const unregisterTmuxRoot = registerPrivateTmuxRoot(ctx.tempRoot);
+ process.env.TMUX_TMPDIR = ctx.tempRoot;
+ const cli: typeof ctx.cli = (invocation) =>
+ ctx.cli({
+ ...invocation,
+ env: { ...invocation.env, TMUX_TMPDIR: ctx.tempRoot },
+ });
try {
- const up = await ctx.cli({
+ const up = await cli({
args: ["up", "--detach"],
cwd: fixture.root,
timeoutMs: UP_TIMEOUT_MS,
@@ -140,7 +179,7 @@ export const lifecycleHostProcessScenario: Scenario = {
result: owner,
});
- const repeatedUp = await ctx.cli({
+ const repeatedUp = await cli({
args: ["up", "--detach"],
cwd: fixture.root,
timeoutMs: UP_TIMEOUT_MS,
@@ -202,7 +241,7 @@ export const lifecycleHostProcessScenario: Scenario = {
message: "legacy fixture should have no session ownership token",
});
- const recoveredUp = await ctx.cli({
+ const recoveredUp = await cli({
args: ["up", "--detach"],
cwd: fixture.root,
timeoutMs: UP_TIMEOUT_MS,
@@ -220,7 +259,7 @@ export const lifecycleHostProcessScenario: Scenario = {
result: recoveredUp,
});
- const down = await ctx.cli({
+ const down = await cli({
args: ["down"],
cwd: fixture.root,
timeoutMs: UP_TIMEOUT_MS,
@@ -260,7 +299,7 @@ export const lifecycleHostProcessScenario: Scenario = {
standaloneContainers: true,
},
});
- const failedUp = await ctx.cli({
+ const failedUp = await cli({
args: ["up", "--detach", "--json"],
cwd: failingFixture.root,
timeoutMs: UP_TIMEOUT_MS,
@@ -298,11 +337,228 @@ export const lifecycleHostProcessScenario: Scenario = {
result: failedUp,
});
} finally {
- await downBestEffort({ ctx, fixture });
+ try {
+ await downBestEffort({ ctx: { ...ctx, cli }, fixture });
+ } finally {
+ try {
+ await runCommand({
+ argv: ["tmux", "kill-server"],
+ cwd: fixture.root,
+ });
+ } finally {
+ // Never leak this fixture's socket root into later scenarios.
+ restoreEnv("TMUX_TMPDIR", previousTmuxRoot);
+ unregisterTmuxRoot();
+ }
+ }
}
},
};
+async function runSupervisorHostProcess(opts: {
+ readonly ctx: Parameters[0];
+ readonly fixture: Awaited>;
+ readonly markerFile: string;
+ readonly envMarkerFile: string;
+ readonly binary: string;
+}): Promise {
+ const env = {
+ HACK_LIFECYCLE_RUNNER: "supervisor",
+ HACK_LIFECYCLE_SUPERVISOR_BINARY: opts.binary,
+ };
+ const statePath = join(
+ opts.fixture.hackDir,
+ ".internal/lifecycle/state.json"
+ );
+ const cli = (args: readonly string[]) =>
+ opts.ctx.cli({
+ args,
+ cwd: opts.fixture.root,
+ env,
+ timeoutMs: UP_TIMEOUT_MS,
+ });
+ try {
+ const up = await cli(["up", "--detach"]);
+ expectExit({
+ result: up,
+ codes: [0],
+ message: "supervisor host fixture should start real Compose services",
+ });
+ expect({
+ that: (await Bun.file(opts.markerFile).text()).includes("up-before-ran"),
+ message: "supervisor before hook did not precede Compose",
+ result: up,
+ });
+ expect({
+ that:
+ (await Bun.file(opts.envMarkerFile).text()) === "host-value|host-only",
+ message: "supervisor before hook lost resolved environment",
+ result: up,
+ });
+ const first = (await readLifecycleState({ path: statePath })).entries[0];
+ const guard = first?.processes?.find(
+ (process) => process.name === "e2e-sleeper"
+ )?.processGroupId;
+ expect({
+ that:
+ first?.backend === "supervisor" &&
+ Boolean(first.ownershipToken) &&
+ Number.isInteger(guard) &&
+ (guard ?? 0) > 1,
+ message: "native lifecycle state lost owner/session identity",
+ result: up,
+ });
+ const leader = await runCommand({
+ argv: ["ps", "-o", "pid=,pgid=", "-p", String(guard)],
+ cwd: opts.fixture.root,
+ });
+ expectExit({
+ result: leader,
+ codes: [0],
+ message: "recorded native guard should be alive",
+ });
+ expect({
+ that: leader.stdout
+ .trim()
+ .split(/\s+/)
+ .every((value) => Number(value) === guard),
+ message: "native guard should lead its own group",
+ result: leader,
+ });
+ const repeated = await cli(["up", "--detach"]);
+ expectExit({
+ result: repeated,
+ codes: [0],
+ message: "supervisor host fixture should adopt healthy owner",
+ });
+ expect({
+ that:
+ (await readLifecycleState({ path: statePath })).entries[0]
+ ?.ownershipToken === first?.ownershipToken,
+ message: "native healthy adoption replaced ownership",
+ result: repeated,
+ });
+ const down = await cli(["down"]);
+ expectExit({
+ result: down,
+ codes: [0],
+ message: "native down should confirm host session retirement",
+ });
+ expect({
+ that:
+ (await readLifecycleState({ path: statePath })).entries.length === 0,
+ message: "native down retained stopped state",
+ result: down,
+ });
+ if (guard && guard > 1) {
+ let alive = true;
+ try {
+ process.kill(guard, 0);
+ } catch {
+ alive = false;
+ }
+ expect({
+ that: !alive,
+ message: "native guard survived confirmed down",
+ result: down,
+ });
+ }
+ const failing = await createMonorepoFixture({
+ parentDir: opts.ctx.tempRoot,
+ withHackConfig: true,
+ lifecycle: {
+ persistentProcess: true,
+ composeFailure: true,
+ standaloneContainers: true,
+ },
+ });
+ const failingState = join(
+ failing.hackDir,
+ ".internal/lifecycle/state.json"
+ );
+ try {
+ const failedUp = await opts.ctx.cli({
+ args: ["up", "--detach", "--json"],
+ cwd: failing.root,
+ env,
+ timeoutMs: UP_TIMEOUT_MS,
+ });
+ expect({
+ that: failedUp.exitCode !== 0 && !failedUp.timedOut,
+ message:
+ "real Compose failure must propagate after native lifecycle startup",
+ result: failedUp,
+ });
+ expect({
+ that:
+ (await readLifecycleState({ path: failingState })).entries.length ===
+ 0,
+ message: "real Compose failure left native lifecycle ownership",
+ result: failedUp,
+ });
+ const priorHome = process.env.HACK_HOME;
+ let nativeRoot: string;
+ try {
+ process.env.HACK_HOME = opts.ctx.hackHome;
+ nativeRoot = resolveLifecycleSupervisorRoot({
+ projectDir: failing.hackDir,
+ composeProject: failing.name,
+ });
+ } finally {
+ restoreEnv("HACK_HOME", priorHome);
+ }
+ const receipts = (await readdir(nativeRoot)).filter((name) =>
+ name.endsWith(".json")
+ );
+ expect({
+ that: receipts.length > 0,
+ message: "failure fixture did not reach native lifecycle startup",
+ result: failedUp,
+ });
+ for (const name of receipts) {
+ const state: unknown = await Bun.file(join(nativeRoot, name)).json();
+ expect({
+ that:
+ isRecord(state) &&
+ Number.isInteger(state.commandPid) &&
+ (state.phase === "exited" || state.phase === "killed"),
+ message:
+ "Compose failure did not retain confirmed native retirement evidence",
+ result: failedUp,
+ });
+ }
+ } finally {
+ const cleanup = await opts.ctx.cli({
+ args: ["down"],
+ cwd: failing.root,
+ env,
+ timeoutMs: UP_TIMEOUT_MS,
+ });
+ if (
+ cleanup.timedOut ||
+ (await readLifecycleState({ path: failingState })).entries.length > 0
+ ) {
+ opts.ctx.retainFixtures(
+ "Failed Compose cleanup retained native ownership evidence."
+ );
+ }
+ await downBestEffort({ ctx: opts.ctx, fixture: failing });
+ }
+ } finally {
+ const cleanup = await cli(["down"]);
+ if (
+ cleanup.exitCode !== 0 ||
+ cleanup.timedOut ||
+ (await readLifecycleState({ path: statePath })).entries.length > 0
+ ) {
+ opts.ctx.retainFixtures(
+ "Supervisor host cleanup was not confirmed; retained private ownership evidence."
+ );
+ }
+ await downBestEffort({ ctx: opts.ctx, fixture: opts.fixture });
+ }
+}
+
type LifecycleStateFixture = {
readonly entries: ReadonlyArray<{
readonly ownershipToken?: string;
diff --git a/tests/e2e/scenarios/lifecycle-session-recovery.ts b/tests/e2e/scenarios/lifecycle-session-recovery.ts
index ea57f6dae..cdd6a4d0d 100644
--- a/tests/e2e/scenarios/lifecycle-session-recovery.ts
+++ b/tests/e2e/scenarios/lifecycle-session-recovery.ts
@@ -1,5 +1,7 @@
-import { chmod, mkdir } from "node:fs/promises";
-import { join } from "node:path";
+import { chmod, copyFile, mkdir } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { restoreEnv } from "../../helpers/env.ts";
+import { registerPrivateTmuxRoot } from "../../helpers/tmux-socket-guard.ts";
import { resolveExecutableCapability } from "../capabilities.ts";
import { createMonorepoFixture } from "../fixture.ts";
@@ -20,6 +22,10 @@ export const lifecycleSessionRecoveryScenario: Scenario = {
tier: "local",
summary: "owned lifecycle sessions reconcile and clean up without Docker",
run: async (ctx) => {
+ if (process.env.HACK_LIFECYCLE_RUNNER === "supervisor") {
+ await runSupervisorRecovery({ ctx });
+ return;
+ }
const tmux = resolveExecutableCapability({
executable: "tmux",
executablePath: Bun.which("tmux"),
@@ -44,7 +50,16 @@ export const lifecycleSessionRecoveryScenario: Scenario = {
await writeFakeDocker({ fakeBin });
const path = `${fakeBin}:${process.env.PATH ?? ""}`;
const dockerLog = join(ctx.tempRoot, "fake-docker.log");
- const cliEnv = { PATH: path, E2E_DOCKER_LOG: dockerLog };
+ const previousTmuxRoot = process.env.TMUX_TMPDIR;
+ // The run's tmux socket guard admits only registered private roots; this
+ // scenario owns its fixture root (mkdtemp, 0700) for its own tmux server.
+ const unregisterTmuxRoot = registerPrivateTmuxRoot(ctx.tempRoot);
+ process.env.TMUX_TMPDIR = ctx.tempRoot;
+ const cliEnv = {
+ PATH: path,
+ E2E_DOCKER_LOG: dockerLog,
+ TMUX_TMPDIR: ctx.tempRoot,
+ };
const sessionName = `${fixture.name}--lifecycle`;
const statePath = join(
fixture.hackDir,
@@ -551,14 +566,325 @@ export const lifecycleSessionRecoveryScenario: Scenario = {
});
}
} finally {
- await runCommand({
- argv: ["tmux", "kill-session", "-t", sessionName],
- cwd: fixture.root,
- });
+ try {
+ await runCommand({
+ argv: ["tmux", "kill-session", "-t", sessionName],
+ cwd: fixture.root,
+ });
+ // This server is scoped to the scenario's private TMUX_TMPDIR; remove
+ // branch sessions too if an earlier assertion interrupted their cleanup.
+ await runCommand({ argv: ["tmux", "kill-server"], cwd: fixture.root });
+ } finally {
+ // Never leak this fixture's socket root into later scenarios.
+ restoreEnv("TMUX_TMPDIR", previousTmuxRoot);
+ unregisterTmuxRoot();
+ }
}
},
};
+/** The supervisor path needs no mux. Exercise the same public CLI ownership,
+ * overlays, branches, restart, rollback and doctor boundaries with native state. */
+async function runSupervisorRecovery(opts: {
+ readonly ctx: ScenarioContext;
+}): Promise {
+ const { ctx } = opts;
+ const binary =
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY ??
+ resolve(
+ ctx.repoRoot,
+ ".hack-local/target/lifecycle-integration/debug/hack-lifecycle-supervisor"
+ );
+ expect({
+ that: binary.startsWith("/") && (await Bun.file(binary).exists()),
+ message:
+ "supervisor recovery requires a built current-source supervisor executable",
+ });
+ const fakeBin = join(ctx.tempRoot, "fake-supervisor-bin");
+ await writeFakeDocker({ fakeBin });
+ const cliEnv = {
+ PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
+ HACK_LIFECYCLE_RUNNER: "supervisor",
+ HACK_LIFECYCLE_SUPERVISOR_BINARY: binary,
+ };
+ const fixture = await createMonorepoFixture({
+ parentDir: ctx.tempRoot,
+ withHackConfig: true,
+ lifecycle: {
+ persistentUpBefore: true,
+ persistentProcess: true,
+ disableInternal: true,
+ persistentUpBeforeCommand:
+ 'bun -e \'console.log("supervisor-ready " + process.env.E2E_PLAIN); let n=0; setInterval(() => console.log("supervisor-tick " + ++n), 100)\'',
+ },
+ });
+ const statePath = join(fixture.hackDir, ".internal/lifecycle/state.json");
+ const cli = (
+ args: readonly string[],
+ env: Readonly> = cliEnv
+ ) => ctx.cli({ args, cwd: fixture.root, env });
+ const assertOk = (result: Awaited>, message: string) =>
+ expectExit({ result, codes: [0], message });
+ const readEntry = async (project = fixture.name) =>
+ (await readState({ statePath })).entries.find(
+ (entry) => entry.composeProject === project
+ );
+ const branches = ["feat-supervisor-handoff"];
+ try {
+ const originalCli = process.env.HACK_E2E_CLI_BIN;
+ const missingBinary = join(ctx.tempRoot, "missing-supervisor");
+ if (originalCli) {
+ const isolatedInstall = join(ctx.tempRoot, "cli-without-companion");
+ await mkdir(isolatedInstall);
+ const isolatedCli = join(isolatedInstall, "hack");
+ await copyFile(originalCli, isolatedCli);
+ await chmod(isolatedCli, 0o700);
+ process.env.HACK_E2E_CLI_BIN = isolatedCli;
+ }
+ try {
+ expectExit({
+ result: await cli(["up", "--detach", "--json"], {
+ ...cliEnv,
+ HACK_LIFECYCLE_SUPERVISOR_BINARY: originalCli
+ ? binary
+ : missingBinary,
+ }),
+ codes: [1],
+ message:
+ "missing companion must fail before ownership even with a production override",
+ });
+ expect({
+ that: !(await readEntry()),
+ message: "failed preflight published ownership",
+ });
+ assertOk(
+ await cli(["down", "--json"]),
+ "missing companion must not wedge down before any launch"
+ );
+ } finally {
+ restoreEnv("HACK_E2E_CLI_BIN", originalCli);
+ }
+ assertOk(
+ await cli(["up", "--detach", "--json"]),
+ "supervisor up should start persistent hook and process"
+ );
+ const initial = await readEntry();
+ expect({
+ that:
+ initial?.backend === "supervisor" &&
+ Boolean(initial.ownershipToken) &&
+ initial.processes?.length === 2,
+ message:
+ "supervisor state must persist both ordered commands and ownership",
+ });
+ assertOk(
+ await cli(["up", "--detach", "--json"]),
+ "supervisor repeated up should adopt"
+ );
+ expect({
+ that: (await readEntry())?.ownershipToken === initial?.ownershipToken,
+ message: "supervisor adoption replaced healthy ownership",
+ });
+ const logPath = join(
+ fixture.hackDir,
+ ".internal/lifecycle",
+ `${fixture.name}.log`
+ );
+ await waitForText({ path: logPath, text: "supervisor-ready host-value" });
+ const logs = await cli(["logs", "e2e-tunnel", "--no-follow"]);
+ assertOk(logs, "supervisor logs should read the common private log");
+ expect({
+ that: logs.combined.includes("supervisor-ready host-value"),
+ message: "supervisor output did not reach hack logs",
+ result: logs,
+ });
+ await expectSupervisorLogFollow({
+ ctx,
+ cwd: fixture.root,
+ env: cliEnv,
+ logPath,
+ });
+ await writeEnvOverlay({
+ hackDir: fixture.hackDir,
+ envName: "qa",
+ value: "qa-first",
+ });
+ assertOk(
+ await cli(["up", "--detach", "--json", "--env", "qa"]),
+ "supervisor overlay replacement should succeed"
+ );
+ expect({
+ that: (await readEntry())?.ownershipToken !== initial?.ownershipToken,
+ message: "supervisor overlay change reused old command environment",
+ });
+ await waitForText({ path: logPath, text: "supervisor-ready qa-first" });
+ const branch = branches[0] ?? "feat-supervisor-handoff";
+ assertOk(
+ await cli(["up", "--detach", "--json", "--branch", branch]),
+ "supervisor branch should start"
+ );
+ const branchProject = `${fixture.name}--${branch}`;
+ const branchToken = (await readEntry(branchProject))?.ownershipToken;
+ assertOk(
+ await cli(["restart", "--json"]),
+ "supervisor restart should succeed"
+ );
+ expect({
+ that: (await readEntry(branchProject))?.ownershipToken === branchToken,
+ message: "primary supervisor restart changed branch ownership",
+ });
+ const beforeKill = await readEntry();
+ const owner = beforeKill?.supervisorPid;
+ expect({
+ that: Number.isInteger(owner) && (owner ?? 0) > 1,
+ message: "supervisor owner PID was not recorded",
+ });
+ if (owner && owner > 1) {
+ // This PID is read immediately from the fixture's own current launch receipt.
+ process.kill(owner, "SIGKILL");
+ }
+ await Bun.sleep(150);
+ assertOk(
+ await cli(["down", "--json"]),
+ "down must recover native groups after project supervisor SIGKILL"
+ );
+ expect({
+ that: !(await readEntry()),
+ message: "recovery down retained confirmed-stopped supervisor entry",
+ });
+ assertOk(
+ await cli(["down", "--json", "--branch", branch]),
+ "branch supervisor down should succeed"
+ );
+ const failedUp = await cli(["up", "--detach", "--json"], {
+ ...cliEnv,
+ E2E_DOCKER_FAIL_UP: "1",
+ });
+ expect({
+ that: failedUp.exitCode !== 0 && !failedUp.timedOut,
+ message: "compose failure must propagate under supervisor",
+ result: failedUp,
+ });
+ expect({
+ that: !(await readEntry()),
+ message:
+ "compose failure retained confirmed-stopped supervisor ownership",
+ });
+ assertOk(
+ await cli(["up", "--detach", "--json"]),
+ "supervisor doctor fixture should start"
+ );
+ await ageLifecycleState({ statePath });
+ const doctor = await cli(["doctor"]);
+ expect({
+ that:
+ doctor.combined.includes("owned lifecycle session") &&
+ doctor.combined.includes("hack doctor --fix"),
+ message: "doctor did not classify native ownership-proven orphan",
+ result: doctor,
+ });
+ const fixed = await cli(["doctor", "--fix"]);
+ expect({
+ that: !fixed.timedOut && fixed.combined.includes("lifecycle repair"),
+ message: "doctor fix did not inspect supervisor retirement",
+ result: fixed,
+ });
+ expect({
+ that: !(await readEntry()),
+ message: "doctor repair did not retire native ownership",
+ });
+ } finally {
+ const cleanup = [await cli(["down", "--json"])];
+ for (const branch of branches) {
+ cleanup.push(await cli(["down", "--json", "--branch", branch]));
+ }
+ if (
+ cleanup.some((result) => result.exitCode !== 0 || result.timedOut) ||
+ (await readState({ statePath })).entries.length > 0
+ ) {
+ ctx.retainFixtures(
+ "Supervisor recovery cleanup was not confirmed; retained private ownership evidence."
+ );
+ }
+ }
+}
+
+async function expectSupervisorLogFollow(opts: {
+ readonly ctx: ScenarioContext;
+ readonly cwd: string;
+ readonly env: Readonly>;
+ readonly logPath: string;
+}): Promise {
+ const previous = Math.max(
+ 0,
+ ...[
+ ...(await Bun.file(opts.logPath).text()).matchAll(
+ /supervisor-tick (\d+)/g
+ ),
+ ].map((match) => Number(match[1]))
+ );
+ const follower = Bun.spawn(
+ resolveCliSpawnArgs(["logs", "-f", "e2e-tunnel"]),
+ {
+ cwd: opts.cwd,
+ env: buildCliEnv({ hackHome: opts.ctx.hackHome, extra: opts.env }),
+ stdin: "ignore",
+ stdout: "pipe",
+ stderr: "pipe",
+ }
+ );
+ const stderr = new Response(follower.stderr).text();
+ const reader = follower.stdout.getReader();
+ let text = "";
+ let timer: ReturnType | undefined;
+ try {
+ await Promise.race([
+ (async () => {
+ while (
+ ![...text.matchAll(/supervisor-tick (\d+)/g)].some(
+ (match) => Number(match[1]) > previous
+ )
+ ) {
+ const result = await reader.read();
+ if (result.done) {
+ throw new Error("supervisor log follower ended before output");
+ }
+ text = (text + new TextDecoder().decode(result.value)).slice(-4096);
+ }
+ })(),
+ new Promise((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error("supervisor log follower timed out")),
+ SESSION_WAIT_TIMEOUT_MS
+ );
+ }),
+ ]);
+ } finally {
+ clearTimeout(timer);
+ follower.kill("SIGINT");
+ await follower.exited;
+ await reader.cancel();
+ await stderr;
+ }
+}
+
+async function waitForText(opts: {
+ readonly path: string;
+ readonly text: string;
+}): Promise {
+ const deadline = Date.now() + SESSION_WAIT_TIMEOUT_MS;
+ while (Date.now() < deadline) {
+ const file = Bun.file(opts.path);
+ if ((await file.exists()) && (await file.text()).includes(opts.text)) {
+ return;
+ }
+ await Bun.sleep(100);
+ }
+ throw new Error(
+ "native lifecycle output did not reach the private log before deadline"
+ );
+}
+
/**
* HACK-1797: without a UTF-8 locale tmux rewrote the session listing's tab
* separators, so `down` missed the owned session (leaving it and its tunnel
@@ -798,6 +1124,7 @@ async function runSignalProbe(opts: {
hackHome: opts.hackHome,
extra: {
PATH: opts.path,
+ TMUX_TMPDIR: process.env.TMUX_TMPDIR ?? "",
E2E_DOCKER_BLOCK_UP: "1",
},
}),
@@ -891,6 +1218,8 @@ async function inspectSessionPresence(opts: {
type LifecycleState = {
readonly entries: ReadonlyArray<{
readonly composeProject: string;
+ readonly backend?: string;
+ readonly supervisorPid?: number;
readonly ownershipToken?: string;
readonly processes?: ReadonlyArray<{
readonly processGroupId?: number;
diff --git a/tests/lifecycle-runner-config.test.ts b/tests/lifecycle-runner-config.test.ts
new file mode 100644
index 000000000..3756fc977
--- /dev/null
+++ b/tests/lifecycle-runner-config.test.ts
@@ -0,0 +1,169 @@
+import { expect, test } from "bun:test";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+ PROJECT_COMPOSE_FILENAME,
+ PROJECT_CONFIG_FILENAME,
+ PROJECT_ENV_FILENAME,
+} from "../src/constants.ts";
+import { resolveLifecycleRunner } from "../src/lib/lifecycle-runner.ts";
+import { readProjectConfig } from "../src/lib/project.ts";
+import { renderProjectConfigSchemaJson } from "../src/templates.ts";
+
+async function config(value: unknown) {
+ const root = await mkdtemp(join(tmpdir(), "hack-runner-config-"));
+ try {
+ const projectRoot = join(root, "repo");
+ const projectDir = join(projectRoot, ".hack");
+ await mkdir(projectDir, { recursive: true });
+ const configFile = join(projectDir, PROJECT_CONFIG_FILENAME);
+ const composeFile = join(projectDir, PROJECT_COMPOSE_FILENAME);
+ const envFile = join(projectDir, PROJECT_ENV_FILENAME);
+ await writeFile(configFile, JSON.stringify(value));
+ await writeFile(composeFile, "services: {}\n");
+ await writeFile(envFile, "");
+ return await readProjectConfig({
+ projectRoot,
+ projectDir,
+ projectDirName: ".hack",
+ composeFile,
+ configFile,
+ envFile,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+}
+
+test("runner-only lifecycle config is retained", async () => {
+ const cfg = await config({ lifecycle: { runner: "supervisor" } });
+ expect(cfg.lifecycle).toEqual({ runner: "supervisor" });
+ expect(cfg.lifecycleRunnerError).toBeUndefined();
+});
+
+test("startup merging preserves runner and command ordering", async () => {
+ const cfg = await config({
+ lifecycle: {
+ runner: "supervisor",
+ up: { before: ["echo before"] },
+ processes: [{ name: "existing", command: "example-worker" }],
+ },
+ startup: [
+ "echo shorthand",
+ { name: "additional", run: "example-helper", persistent: true },
+ ],
+ });
+ expect(cfg.lifecycle?.runner).toBe("supervisor");
+ expect(cfg.lifecycle?.up?.before).toEqual([
+ { command: "echo before" },
+ { command: "echo shorthand" },
+ ]);
+ expect(cfg.lifecycle?.processes).toEqual([
+ { name: "existing", command: "example-worker" },
+ { name: "additional", command: "example-helper" },
+ ]);
+});
+
+test("omitted runner defaults to mux without changing parsed config", async () => {
+ const cfg = await config({
+ startup: [{ run: "example-worker", persistent: true }],
+ });
+ expect(cfg.lifecycle?.runner).toBeUndefined();
+ expect(
+ resolveLifecycleRunner({ runner: cfg.lifecycle?.runner, env: {} })
+ ).toBe("mux");
+ expect(resolveLifecycleRunner({ env: {} })).toBe("mux");
+});
+
+test("environment override precedes either configured runner", () => {
+ expect(
+ resolveLifecycleRunner({
+ runner: "mux",
+ env: { HACK_LIFECYCLE_RUNNER: "supervisor" },
+ })
+ ).toBe("supervisor");
+ expect(
+ resolveLifecycleRunner({
+ runner: "supervisor",
+ env: { HACK_LIFECYCLE_RUNNER: "mux" },
+ })
+ ).toBe("mux");
+});
+
+test.each([
+ "",
+ "auto",
+ "SUPERVISOR",
+ "synthetic-invalid-input",
+ null,
+ 7,
+])("invalid explicit runner config is marked for fail-closed startup (%p)", async (runner) => {
+ const cfg = await config({
+ lifecycle: { runner },
+ startup: ["echo preparation"],
+ });
+ expect(cfg.lifecycleRunnerError).toContain("lifecycle.runner");
+ expect(cfg.lifecycleRunnerError).toContain('"mux" or "supervisor"');
+ expect(cfg.parseError).toContain(
+ cfg.lifecycleRunnerError ?? "missing-validation"
+ );
+ expect(cfg.lifecycleRunnerError).not.toContain("synthetic-invalid-input");
+ expect(cfg.lifecycle?.up?.before).toEqual([{ command: "echo preparation" }]);
+ expect(() => resolveLifecycleRunner({ runner, env: {} })).toThrow(
+ "lifecycle.runner"
+ );
+});
+
+test.each([
+ "",
+ "auto",
+ "SUPERVISOR",
+ "synthetic-invalid-input",
+])("invalid explicit environment override never falls back (%p)", (override) => {
+ expect(() =>
+ resolveLifecycleRunner({
+ runner: "mux",
+ env: { HACK_LIFECYCLE_RUNNER: override },
+ })
+ ).toThrow("HACK_LIFECYCLE_RUNNER");
+ try {
+ resolveLifecycleRunner({
+ runner: "mux",
+ env: { HACK_LIFECYCLE_RUNNER: override },
+ });
+ } catch (error: unknown) {
+ expect(error instanceof Error && error.message).not.toContain(
+ "synthetic-invalid-input"
+ );
+ }
+});
+
+test("runner schema accepts only the named options and documents mux default", () => {
+ const schema = JSON.parse(renderProjectConfigSchemaJson());
+ expect(schema.properties.lifecycle.properties.runner).toMatchObject({
+ type: "string",
+ enum: ["mux", "supervisor"],
+ default: "mux",
+ });
+});
+
+test("runner-only config survives shorthand and shared-helper reconstruction", async () => {
+ const cfg = await config({
+ lifecycle: {
+ runner: "mux",
+ sharedHelpers: [
+ {
+ name: "example-helper",
+ command: ["example-helper"],
+ port: 4100,
+ upstream: "https://example.invalid",
+ },
+ ],
+ },
+ startup: ["echo preparation"],
+ });
+ expect(cfg.lifecycle?.runner).toBe("mux");
+ expect(cfg.lifecycle?.sharedHelpers).toHaveLength(1);
+ expect(cfg.lifecycle?.up?.before).toEqual([{ command: "echo preparation" }]);
+});
diff --git a/tests/lifecycle-state-publication.test.ts b/tests/lifecycle-state-publication.test.ts
new file mode 100644
index 000000000..25e4a6b7d
--- /dev/null
+++ b/tests/lifecycle-state-publication.test.ts
@@ -0,0 +1,319 @@
+import { expect, test } from "bun:test";
+import { existsSync } from "node:fs";
+import {
+ chmod,
+ link,
+ lstat,
+ mkdir,
+ mkdtemp,
+ readFile,
+ rm,
+ symlink,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+ type LifecycleStateEntry,
+ readLifecycleState,
+ removeLifecycleStateEntry,
+ removeLifecycleStateEntryIfMatching,
+ removeLifecycleStateEntryIfOwned,
+ resolveLifecycleRootDir,
+ resolveLifecycleStatePath,
+ upsertLifecycleStateEntry,
+} from "../src/lib/lifecycle-runtime.ts";
+
+function entry(name: string, cleanupEpoch = "epoch-a"): LifecycleStateEntry {
+ return {
+ composeProject: name,
+ projectName: "example-project",
+ branch: name,
+ sessionName: `example-lifecycle-${name}`,
+ backend: "supervisor",
+ ownershipToken: "synthetic-owner",
+ cleanupEpoch,
+ supervisorPid: 42,
+ processes: [
+ {
+ name: "worker",
+ windowName: "worker",
+ logPath: "example.log",
+ guardPid: 43,
+ guardStartTime: 44,
+ commandPid: 45,
+ processGroupId: 43,
+ bootId: "synthetic-boot",
+ },
+ ],
+ updatedAt: "2026-01-01T00:00:00Z",
+ };
+}
+
+async function fixture(
+ run: (opts: {
+ projectDir: string;
+ root: string;
+ path: string;
+ }) => Promise
+) {
+ const projectDir = await mkdtemp(join(tmpdir(), "hack-state-publication-"));
+ const root = resolveLifecycleRootDir({ projectDir });
+ const path = resolveLifecycleStatePath({ projectDir });
+ try {
+ await mkdir(root, { recursive: true, mode: 0o700 });
+ await run({ projectDir, root, path });
+ } finally {
+ await rm(projectDir, { recursive: true, force: true });
+ }
+}
+
+test("parallel branch publication and all deletion variants preserve unrelated ownership", async () => {
+ await fixture(async ({ projectDir }) => {
+ const names = ["branch-a", "branch-b", "branch-c", "branch-d"];
+ await Promise.all(
+ names.map((name) =>
+ upsertLifecycleStateEntry({ projectDir, entry: entry(name) })
+ )
+ );
+ expect(
+ (await readLifecycleState({ projectDir })).map(
+ (item) => item.composeProject
+ )
+ ).toEqual(names);
+ await Promise.all([
+ removeLifecycleStateEntry({ projectDir, composeProject: "branch-a" }),
+ removeLifecycleStateEntryIfOwned({
+ projectDir,
+ composeProject: "branch-b",
+ ownershipToken: "synthetic-owner",
+ }),
+ removeLifecycleStateEntryIfMatching({
+ projectDir,
+ expectedEntry: entry("branch-c"),
+ }),
+ upsertLifecycleStateEntry({ projectDir, entry: entry("branch-e") }),
+ ]);
+ expect(
+ (await readLifecycleState({ projectDir })).map(
+ (item) => item.composeProject
+ )
+ ).toEqual(["branch-d", "branch-e"]);
+ });
+});
+
+test("old cleanup epoch cannot delete an adopted entry with the same owner token", async () => {
+ await fixture(async ({ projectDir }) => {
+ await upsertLifecycleStateEntry({
+ projectDir,
+ entry: entry("branch-a", "adopted"),
+ });
+ expect(
+ await removeLifecycleStateEntryIfMatching({
+ projectDir,
+ expectedEntry: entry("branch-a"),
+ })
+ ).toBe(false);
+ expect(
+ await removeLifecycleStateEntryIfOwned({
+ projectDir,
+ composeProject: "branch-a",
+ ownershipToken: "foreign-owner",
+ })
+ ).toBe(false);
+ const states = await readLifecycleState({ projectDir });
+ expect(states).toEqual([entry("branch-a", "adopted")]);
+ });
+});
+
+test("atomic readers observe a complete old or new snapshot", async () => {
+ await fixture(async ({ projectDir }) => {
+ await upsertLifecycleStateEntry({
+ projectDir,
+ entry: entry("branch-a", "0"),
+ });
+ let writing = true;
+ const writer = (async () => {
+ for (let index = 1; index <= 25; index += 1) {
+ await upsertLifecycleStateEntry({
+ projectDir,
+ entry: entry("branch-a", String(index)),
+ });
+ }
+ })().finally(() => {
+ writing = false;
+ });
+ let observations = 0;
+ while (writing) {
+ const states = await readLifecycleState({ projectDir });
+ expect(states).toHaveLength(1);
+ expect(states[0]?.composeProject).toBe("branch-a");
+ expect(Number(states[0]?.cleanupEpoch)).toBeWithin(0, 26);
+ observations += 1;
+ }
+ await writer;
+ expect(observations).toBeGreaterThan(0);
+ expect((await readLifecycleState({ projectDir }))[0]?.cleanupEpoch).toBe(
+ "25"
+ );
+ });
+});
+
+test.each([
+ "null",
+ '{"entries":{}}',
+ '{"entries":[{}]}',
+])("malformed state refuses mutation without overwriting (%p)", async (text) => {
+ await fixture(async ({ projectDir, path }) => {
+ await writeFile(path, text, { mode: 0o600 });
+ await expect(readLifecycleState({ projectDir })).rejects.toThrow(
+ "Lifecycle ownership state"
+ );
+ await expect(
+ upsertLifecycleStateEntry({ projectDir, entry: entry("branch-a") })
+ ).rejects.toThrow("Lifecycle ownership state");
+ expect(await readFile(path, "utf8")).toBe(text);
+ });
+});
+
+test.each([
+ "symlink",
+ "hardlink",
+])("aliased ownership state is refused without changing either name (%s)", async (kind) => {
+ await fixture(async ({ projectDir, root, path }) => {
+ const target = join(root, "retained.json");
+ const contents = JSON.stringify({ entries: [entry("branch-a")] });
+ await writeFile(target, contents, { mode: 0o600 });
+ if (kind === "symlink") {
+ await symlink(target, path);
+ } else {
+ await link(target, path);
+ }
+ await expect(
+ upsertLifecycleStateEntry({ projectDir, entry: entry("branch-b") })
+ ).rejects.toThrow();
+ await expect(readLifecycleState({ projectDir })).rejects.toThrow();
+ expect(await readFile(target, "utf8")).toBe(contents);
+ expect(await readFile(path, "utf8")).toBe(contents);
+ });
+});
+
+test("legacy owner-only regular state migrates to private atomic publication", async () => {
+ await fixture(async ({ projectDir, root, path }) => {
+ await chmod(root, 0o755);
+ await writeFile(path, JSON.stringify({ entries: [entry("branch-a")] }), {
+ mode: 0o644,
+ });
+ expect(await readLifecycleState({ projectDir })).toEqual([
+ entry("branch-a"),
+ ]);
+ await upsertLifecycleStateEntry({ projectDir, entry: entry("branch-b") });
+ expect((await lstat(root)).mode & 0o777).toBe(0o700);
+ expect((await lstat(path)).mode & 0o777).toBe(0o600);
+ expect(await readLifecycleState({ projectDir })).toEqual([
+ entry("branch-a"),
+ entry("branch-b"),
+ ]);
+ });
+});
+
+test("duplicate ownership entries and malformed process rows fail closed", async () => {
+ await fixture(async ({ projectDir, path }) => {
+ for (const entries of [
+ [entry("branch-a"), entry("branch-a")],
+ [{ ...entry("branch-a"), processes: [{}] }],
+ ]) {
+ const text = JSON.stringify({ entries });
+ await writeFile(path, text, { mode: 0o600 });
+ await expect(
+ removeLifecycleStateEntry({ projectDir, composeProject: "branch-a" })
+ ).rejects.toThrow("Lifecycle ownership state");
+ expect(await readFile(path, "utf8")).toBe(text);
+ }
+ });
+});
+
+test("oversized existing and proposed state are refused without replacement", async () => {
+ await fixture(async ({ projectDir, path }) => {
+ const oversized = " ".repeat(1024 * 1024 + 1);
+ await writeFile(path, oversized, { mode: 0o600 });
+ await expect(
+ upsertLifecycleStateEntry({ projectDir, entry: entry("branch-a") })
+ ).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(oversized);
+ const text = JSON.stringify({ entries: [entry("branch-a")] });
+ await writeFile(path, text, { mode: 0o600 });
+ await expect(
+ upsertLifecycleStateEntry({
+ projectDir,
+ entry: { ...entry("branch-b"), definitionHash: oversized },
+ })
+ ).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(text);
+ });
+});
+
+test.each([
+ "",
+ "{",
+])("legacy non-atomic mux JSON is preserved and recovered (%p)", async (text) => {
+ await fixture(async ({ projectDir, path, root }) => {
+ const { readdir } = await import("node:fs/promises");
+ await writeFile(path, text, { mode: 0o600 });
+ expect(await readLifecycleState({ projectDir })).toEqual([]);
+ const archived = (await readdir(root)).find((name) =>
+ name.startsWith("state.json.legacy-")
+ );
+ expect(archived).toBeDefined();
+ expect(await readFile(join(root, archived ?? "missing"), "utf8")).toBe(
+ text
+ );
+ await upsertLifecycleStateEntry({
+ projectDir,
+ entry: { ...entry("mux"), backend: "tmux" },
+ });
+ expect((await readLifecycleState({ projectDir }))[0]?.backend).toBe("tmux");
+ });
+});
+
+test("supervisor ownership disables legacy JSON recovery", async () => {
+ await fixture(async ({ projectDir, path }) => {
+ await upsertLifecycleStateEntry({ projectDir, entry: entry("native") });
+ await writeFile(path, "{", { mode: 0o600 });
+ await expect(readLifecycleState({ projectDir })).rejects.toThrow(
+ "Lifecycle ownership state"
+ );
+ expect(await readFile(path, "utf8")).toBe("{");
+ });
+});
+
+test("missing state after supervisor ownership refuses instead of proving absence", async () => {
+ await fixture(async ({ projectDir, root, path }) => {
+ // First supervisor publication starts with no state file at all.
+ await upsertLifecycleStateEntry({ projectDir, entry: entry("native") });
+ expect(await lstat(join(root, "supervisor-state"))).toBeTruthy();
+ await rm(path);
+ await expect(readLifecycleState({ projectDir })).rejects.toThrow(
+ "Lifecycle ownership state"
+ );
+ await expect(
+ upsertLifecycleStateEntry({ projectDir, entry: entry("other") })
+ ).rejects.toThrow("Lifecycle ownership state");
+ await expect(
+ removeLifecycleStateEntry({ projectDir, composeProject: "native" })
+ ).rejects.toThrow("Lifecycle ownership state");
+ expect(existsSync(path)).toBe(false);
+ });
+});
+
+test("missing state without supervisor history remains empty", async () => {
+ await fixture(async ({ projectDir, root }) => {
+ await upsertLifecycleStateEntry({
+ projectDir,
+ entry: { ...entry("mux"), backend: "tmux" },
+ });
+ await rm(resolveLifecycleStatePath({ projectDir }));
+ expect(existsSync(join(root, "supervisor-state"))).toBe(false);
+ expect(await readLifecycleState({ projectDir })).toEqual([]);
+ });
+});
diff --git a/tests/lifecycle-supervisor-controller.test.ts b/tests/lifecycle-supervisor-controller.test.ts
new file mode 100644
index 000000000..e89132be2
--- /dev/null
+++ b/tests/lifecycle-supervisor-controller.test.ts
@@ -0,0 +1,685 @@
+import { afterAll, beforeAll, expect, test } from "bun:test";
+import { existsSync } from "node:fs";
+import {
+ mkdir,
+ mkdtemp,
+ readFile,
+ realpath,
+ rename,
+ rm,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import {
+ readLifecycleState,
+ resolveLifecycleLogPath,
+ resolveLifecycleStatePath,
+} from "../src/lib/lifecycle-runtime.ts";
+import {
+ inspectLifecycleSupervisor,
+ resolveLifecycleSupervisorRoot,
+} from "../src/lib/lifecycle-supervisor.ts";
+import {
+ createLifecycleProcessController,
+ stopLifecycleProcessController,
+} from "../src/lib/project-lifecycle-controller.ts";
+import { restoreEnv } from "./helpers/env.ts";
+
+const previous = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+const supervisorBinary =
+ process.env.HACK_TEST_SUPERVISOR_BINARY ??
+ resolve(
+ ".hack-local/target/lifecycle-integration/debug/hack-lifecycle-supervisor"
+ );
+if (
+ process.env.HACK_TEST_REQUIRE_SUPERVISOR === "1" &&
+ !existsSync(supervisorBinary)
+) {
+ throw new Error(
+ "The required current-source supervisor executable was not prepared."
+ );
+}
+const nativeTest = existsSync(supervisorBinary) ? test : test.skip;
+beforeAll(() => {
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY = supervisorBinary;
+});
+afterAll(() => restoreEnv("HACK_LIFECYCLE_SUPERVISOR_BINARY", previous));
+
+async function fixture(
+ run: (project: { projectRoot: string; projectDir: string }) => Promise
+) {
+ const root = await realpath(
+ await mkdtemp(join(tmpdir(), "supervisor-controller-"))
+ );
+ const projectRoot = join(root, "checkout");
+ const project = { projectRoot, projectDir: join(projectRoot, ".hack") };
+ await mkdir(project.projectDir, { recursive: true });
+ const savedHackHome = process.env.HACK_HOME;
+ process.env.HACK_HOME = join(root, "host-home");
+ try {
+ // Compare canonical fixture paths (Darwin temp paths may alias /private).
+ // Fail before native effects if resolution reaches the real home.
+ expect(
+ resolveLifecycleSupervisorRoot({
+ ...project,
+ composeProject: "fixture",
+ }).startsWith(`${process.env.HACK_HOME}/`)
+ ).toBe(true);
+ await run(project);
+ } finally {
+ await stopLifecycleProcessController({
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "synthetic",
+ }).catch(() => undefined);
+ restoreEnv("HACK_HOME", savedHackHome);
+ await rm(root, { recursive: true, force: true });
+ }
+}
+function controller(
+ project: { projectRoot: string; projectDir: string },
+ hash = "synthetic"
+) {
+ return createLifecycleProcessController({
+ runner: "supervisor",
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: hash,
+ env: { HOME: project.projectRoot, HACK_DAEMON_AUTOSTART_DISABLE: "1" },
+ mux: async () => {
+ throw new Error("mux must not be consulted");
+ },
+ installSignalCleanup: () => ({ dispose() {} }),
+ });
+}
+const command = "printf 'controller-ready\\n'; exec sleep 60";
+async function eventually(check: () => Promise) {
+ const end = performance.now() + 5000;
+ while (!(await check())) {
+ if (performance.now() > end) {
+ throw new Error("fixture observation timed out");
+ }
+ await Bun.sleep(20);
+ }
+}
+
+nativeTest(
+ "supervisor controller starts persistent hooks incrementally without mux and keeps common logs",
+ () =>
+ fixture(async (project) => {
+ const owner = controller(project);
+ await owner.startFromCommand({
+ command: { command, persistent: true },
+ serviceName: "first",
+ });
+ let entries = await readLifecycleState(project);
+ expect(entries[0]?.processes.map((p) => p.name)).toEqual(["first"]);
+ await owner.startMany({ processes: [{ name: "second", command }] });
+ await owner.finalize();
+ entries = await readLifecycleState(project);
+ expect(entries[0]?.backend).toBe("supervisor");
+ expect(entries[0]?.processes.map((p) => p.name).sort()).toEqual([
+ "first",
+ "second",
+ ]);
+ const logPath = resolveLifecycleLogPath({
+ ...project,
+ composeProject: "fixture",
+ });
+ await eventually(async () =>
+ (await readFile(logPath, "utf8")).includes("controller-ready")
+ );
+ expect(await owner.abort()).toBeUndefined();
+ expect(await readLifecycleState(project)).toHaveLength(0);
+ }),
+ 30_000
+);
+
+nativeTest(
+ "adopting a healthy supervisor retains its token and does not acquire rollback authority",
+ () =>
+ fixture(async (project) => {
+ const first = controller(project);
+ await first.startMany({ processes: [{ name: "worker", command }] });
+ await first.finalize();
+ const original = (await readLifecycleState(project))[0];
+ const second = controller(project);
+ await second.startMany({ processes: [{ name: "worker", command }] });
+ await second.finalize();
+ expect((await readLifecycleState(project))[0]?.ownershipToken).toBe(
+ original?.ownershipToken
+ );
+ expect(second.getOperationCleanup()).toBeNull();
+ await second.abort();
+ expect((await readLifecycleState(project))[0]?.ownershipToken).toBe(
+ original?.ownershipToken
+ );
+ }),
+ 30_000
+);
+
+nativeTest(
+ "persisted supervisor stop works after owner SIGKILL and current runner changes",
+ () =>
+ fixture(async (project) => {
+ const owner = controller(project);
+ await owner.startMany({ processes: [{ name: "worker", command }] });
+ await owner.finalize();
+ const entry = (await readLifecycleState(project))[0];
+ if (!(entry?.supervisorPid && entry.ownershipToken)) {
+ throw new Error("fixture missing owner");
+ }
+ process.kill(entry.supervisorPid, "SIGKILL");
+ const status = await inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ });
+ expect(status.liveProcessNames).toEqual(["worker"]);
+ await stopLifecycleProcessController({
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "changed",
+ mux: async () => {
+ throw new Error("mux must not be consulted");
+ },
+ });
+ expect(await readLifecycleState(project)).toHaveLength(0);
+ }),
+ 30_000
+);
+
+nativeTest(
+ "creator rollback cannot stop a supervisor another operation adopted",
+ () =>
+ fixture(async (project) => {
+ const first = controller(project);
+ await first.startMany({ processes: [{ name: "worker", command }] });
+ await first.finalize();
+ const second = controller(project);
+ await second.startMany({ processes: [{ name: "worker", command }] });
+ await second.finalize();
+ await first.abort();
+ const entry = (await readLifecycleState(project))[0];
+ if (!entry?.ownershipToken) {
+ throw new Error("adopted ownership discarded");
+ }
+ const status = await inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ });
+ expect(status.liveProcessNames).toEqual(["worker"]);
+ }),
+ 30_000
+);
+
+nativeTest(
+ "missing native root is absent while missing executable with retained root stays unknown",
+ () =>
+ fixture(async (project) => {
+ const empty = await inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: "synthetic",
+ });
+ expect(empty.liveProcessNames).toEqual([]);
+ const owner = controller(project);
+ await owner.startMany({ processes: [{ name: "worker", command }] });
+ await owner.finalize();
+ const entry = (await readLifecycleState(project))[0];
+ if (!entry?.ownershipToken) {
+ throw new Error("fixture missing token");
+ }
+ const binary = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY =
+ "/missing-synthetic-supervisor";
+ try {
+ await expect(
+ inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ })
+ ).rejects.toThrow("executable");
+ expect((await readLifecycleState(project))[0]?.ownershipToken).toBe(
+ entry.ownershipToken
+ );
+ } finally {
+ restoreEnv("HACK_LIFECYCLE_SUPERVISOR_BINARY", binary);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "human service names stay distinct while native state names remain safe",
+ () =>
+ fixture(async (project) => {
+ const owner = controller(project);
+ const names = ["service with spaces/../label", "Web", "web"];
+ await owner.startMany({
+ processes: names.map((name) => ({ name, command })),
+ });
+ await owner.finalize();
+ const entry = (await readLifecycleState(project))[0];
+ expect(entry?.processes.map((p) => p.name).sort()).toEqual(
+ [...names].sort()
+ );
+ if (!entry?.ownershipToken) {
+ throw new Error("fixture missing token");
+ }
+ const observed = await inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ });
+ expect([...observed.liveProcessNames].sort()).toEqual([...names].sort());
+ expect(await owner.abort()).toBeUndefined();
+ }),
+ 30_000
+);
+
+nativeTest(
+ "lost incremental reply publishes the verified late command before failure",
+ () =>
+ fixture(async (project) => {
+ const wrapper = join(project.projectRoot, "drop-add-reply");
+ await Bun.write(
+ wrapper,
+ `#!${process.execPath} --no-env-file
+if (process.argv.at(-1) === "--version") {
+ const child = Bun.spawn([${JSON.stringify(supervisorBinary)}, "--version"], {env:{},stdin:"ignore",stdout:"inherit",stderr:"ignore"});
+ process.exit(await child.exited);
+}
+const reader = Bun.stdin.stream().getReader();
+let frame = Buffer.alloc(0);
+while (true) {
+ const part = await reader.read();
+ if (part.done) process.exit(91);
+ frame = Buffer.concat([frame, part.value]);
+ if (frame.length >= 4 && frame.length >= frame.readUInt32BE(0) + 4) break;
+}
+await reader.cancel();
+const action = process.argv.at(-1);
+const child = Bun.spawn([${JSON.stringify(supervisorBinary)}, action], {env:{},stdin:frame,stdout:"pipe",stderr:"ignore"});
+child.unref();
+const reply = await new Response(child.stdout).arrayBuffer();
+if (action === "add") { await child.exited; process.exit(1); }
+await Bun.write(Bun.stdout, reply);
+if (action !== "serve") process.exit(await child.exited);
+`
+ );
+ const { chmod } = await import("node:fs/promises");
+ await chmod(wrapper, 0o700);
+ const saved = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY = wrapper;
+ const owner = controller(project);
+ try {
+ await owner.startMany({ processes: [{ name: "first", command }] });
+ await expect(
+ owner.startMany({ processes: [{ name: "late", command }] })
+ ).rejects.toThrow("reply was lost");
+ const entry = (await readLifecycleState(project))[0];
+ expect(entry?.processes.map((p) => p.name).sort()).toEqual([
+ "first",
+ "late",
+ ]);
+ if (!entry?.ownershipToken) {
+ throw new Error("late ownership missing");
+ }
+ const status = await inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ });
+ expect([...status.liveProcessNames].sort()).toEqual(["first", "late"]);
+ await owner.abort();
+ } finally {
+ restoreEnv("HACK_LIFECYCLE_SUPERVISOR_BINARY", saved);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "a native command published after adoption is refreshed before another add",
+ () =>
+ fixture(async (project) => {
+ const { launchLifecycleSupervisorProcess } = await import(
+ "../src/lib/lifecycle-supervisor.ts"
+ );
+ const creator = controller(project);
+ await creator.startMany({ processes: [{ name: "first", command }] });
+ await creator.finalize();
+ const adopter = controller(project);
+ await adopter.startMany({ processes: [{ name: "first", command }] });
+ const entry = (await readLifecycleState(project))[0];
+ if (!entry?.ownershipToken) {
+ throw new Error("fixture missing token");
+ }
+ // Schedule the protected native effect after the adopter's initial snapshot.
+ await launchLifecycleSupervisorProcess({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ action: "add",
+ process: {
+ name: "second",
+ argv: ["/bin/sh", "-c", command],
+ cwd: project.projectRoot,
+ env: {
+ HOME: project.projectRoot,
+ PATH: "/usr/bin:/bin",
+ HACK_DAEMON_AUTOSTART_DISABLE: "1",
+ },
+ },
+ });
+ await adopter.startMany({
+ processes: [{ name: "second", command: "exit 99" }],
+ });
+ await adopter.finalize();
+ expect(
+ (await readLifecycleState(project))[0]?.processes
+ .map((p) => p.name)
+ .sort()
+ ).toEqual(["first", "second"]);
+ }),
+ 30_000
+);
+
+nativeTest(
+ "missing companion fails before ownership and down remains usable",
+ () =>
+ fixture(async (project) => {
+ const saved = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY = join(
+ project.projectRoot,
+ "missing-companion"
+ );
+ try {
+ await expect(
+ controller(project).startMany({
+ processes: [{ name: "worker", command }],
+ })
+ ).rejects.toThrow("executable");
+ expect(await readLifecycleState(project)).toEqual([]);
+ await stopLifecycleProcessController({
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "synthetic",
+ });
+ expect(await readLifecycleState(project)).toEqual([]);
+ } finally {
+ restoreEnv("HACK_LIFECYCLE_SUPERVISOR_BINARY", saved);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "recorded native root survives a changed home and a symlinked checkout",
+ () =>
+ fixture(async (project) => {
+ const { symlink } = await import("node:fs/promises");
+ const owner = controller(project);
+ await owner.startMany({ processes: [{ name: "worker", command }] });
+ await owner.finalize();
+ const entry = (await readLifecycleState(project))[0];
+ if (!(entry?.ownershipToken && entry.supervisorRoot)) {
+ throw new Error("missing fixture identity");
+ }
+ const savedHome = process.env.HACK_HOME;
+ const alias = join(project.projectRoot, "alias");
+ await symlink(project.projectDir, alias);
+ process.env.HACK_HOME = join(project.projectRoot, "unused-home");
+ try {
+ const observed = await inspectLifecycleSupervisor({
+ projectDir: alias,
+ composeProject: "fixture",
+ ownershipToken: entry.ownershipToken,
+ });
+ expect(observed.liveProcessNames).toEqual(["worker"]);
+ await stopLifecycleProcessController({
+ project: { ...project, projectDir: alias },
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "synthetic",
+ });
+ expect(await readLifecycleState(project)).toEqual([]);
+ } finally {
+ restoreEnv("HACK_HOME", savedHome);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "missing recorded root never grants retirement authority",
+ () =>
+ fixture(async (project) => {
+ const { rename } = await import("node:fs/promises");
+ const owner = controller(project);
+ await owner.startMany({ processes: [{ name: "worker", command }] });
+ await owner.finalize();
+ const entry = (await readLifecycleState(project))[0];
+ if (!(entry?.ownershipToken && entry.supervisorRoot)) {
+ throw new Error("missing fixture identity");
+ }
+ const moved = `${entry.supervisorRoot}.retained`;
+ await rename(entry.supervisorRoot, moved);
+ try {
+ await expect(
+ stopLifecycleProcessController({
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "synthetic",
+ })
+ ).rejects.toThrow();
+ expect((await readLifecycleState(project))[0]?.ownershipToken).toBe(
+ entry.ownershipToken
+ );
+ } finally {
+ await rename(moved, entry.supervisorRoot);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "never-attempted ownership can be retired without the companion",
+ () =>
+ fixture(async (project) => {
+ const { upsertLifecycleStateEntry } = await import(
+ "../src/lib/lifecycle-runtime.ts"
+ );
+ await upsertLifecycleStateEntry({
+ projectDir: project.projectDir,
+ entry: {
+ composeProject: "fixture",
+ projectName: "fixture",
+ branch: null,
+ sessionName: "fixture-lifecycle",
+ backend: "supervisor",
+ ownershipToken: "synthetic-never-started",
+ cleanupEpoch: "synthetic-epoch",
+ processes: [],
+ supervisorLaunchAttempted: false,
+ updatedAt: new Date().toISOString(),
+ },
+ });
+ const saved = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY = join(
+ project.projectRoot,
+ "missing-companion"
+ );
+ try {
+ await stopLifecycleProcessController({
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "synthetic",
+ });
+ expect(await readLifecycleState(project)).toEqual([]);
+ } finally {
+ restoreEnv("HACK_LIFECYCLE_SUPERVISOR_BINARY", saved);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "incompatible companion fails before command environment delivery",
+ () =>
+ fixture(async (project) => {
+ const { chmod } = await import("node:fs/promises");
+ const wrapper = join(project.projectRoot, "wrong-version");
+ await Bun.write(
+ wrapper,
+ `#!${process.execPath} --no-env-file\nconsole.log("synthetic incompatible version");\n`
+ );
+ await chmod(wrapper, 0o700);
+ const saved = process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY;
+ process.env.HACK_LIFECYCLE_SUPERVISOR_BINARY = wrapper;
+ try {
+ await expect(
+ controller(project).startMany({
+ processes: [{ name: "worker", command }],
+ })
+ ).rejects.toThrow("incompatible");
+ expect(await readLifecycleState(project)).toEqual([]);
+ } finally {
+ restoreEnv("HACK_LIFECYCLE_SUPERVISOR_BINARY", saved);
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "definition replacement waits for the previous project owner to retire",
+ () =>
+ fixture(async (project) => {
+ for (const hash of [
+ "synthetic-first",
+ "synthetic-second",
+ "synthetic-third",
+ ]) {
+ const owner = controller(project, hash);
+ await owner.startMany({ processes: [{ name: "worker", command }] });
+ await owner.finalize();
+ expect((await readLifecycleState(project))[0]?.definitionHash).toBe(
+ hash
+ );
+ }
+ }),
+ 30_000
+);
+
+nativeTest(
+ "a command already retiring cannot be adopted as healthy",
+ () =>
+ fixture(async (project) => {
+ const owner = controller(project);
+ await owner.startMany({
+ processes: [
+ {
+ name: "worker",
+ command:
+ "trap '' TERM; printf 'retirement-ready\\n'; exec sleep 60",
+ },
+ ],
+ });
+ await owner.finalize();
+ const path = resolveLifecycleLogPath({
+ ...project,
+ composeProject: "fixture",
+ });
+ await eventually(async () =>
+ (await readFile(path, "utf8")).includes("retirement-ready")
+ );
+ const previous = (await readLifecycleState(project))[0];
+ const guard = previous?.processes[0]?.guardPid;
+ if (!(guard && previous?.ownershipToken)) {
+ throw new Error("missing owned guard identity");
+ }
+ process.kill(guard, "SIGTERM");
+ await eventually(async () => {
+ const status = await inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken: previous.ownershipToken ?? "",
+ });
+ return status.processes.some((process) => process.phase === "stopping");
+ });
+ const replacement = controller(project);
+ await replacement.startMany({ processes: [{ name: "worker", command }] });
+ await replacement.finalize();
+ expect((await readLifecycleState(project))[0]?.ownershipToken).not.toBe(
+ previous.ownershipToken
+ );
+ }),
+ 30_000
+);
+
+nativeTest(
+ "lost common state keeps native work unconfirmed instead of stopped",
+ () =>
+ fixture(async (project) => {
+ const owner = controller(project);
+ await owner.startMany({ processes: [{ name: "worker", command }] });
+ await owner.finalize();
+ const entry = (await readLifecycleState(project))[0];
+ const commandPid = entry?.processes[0]?.commandPid;
+ const ownershipToken = entry?.ownershipToken;
+ if (!(commandPid && ownershipToken)) {
+ throw new Error("fixture command was not recorded");
+ }
+ const statePath = resolveLifecycleStatePath(project);
+ const aside = `${statePath}.test-aside`;
+ await rename(statePath, aside);
+ const stop = () =>
+ stopLifecycleProcessController({
+ project,
+ projectName: "fixture",
+ composeProject: "fixture",
+ branch: null,
+ definitionHash: "synthetic",
+ mux: async () => {
+ throw new Error("mux must not be consulted");
+ },
+ });
+ await expect(stop()).rejects.toThrow("Lifecycle ownership state");
+ await expect(
+ inspectLifecycleSupervisor({
+ ...project,
+ composeProject: "fixture",
+ ownershipToken,
+ })
+ ).rejects.toThrow();
+ // Nothing was signalled: the native command still runs.
+ expect(() => process.kill(commandPid, 0)).not.toThrow();
+ await rename(aside, statePath);
+ await stop();
+ await eventually(async () => {
+ try {
+ process.kill(commandPid, 0);
+ return false;
+ } catch {
+ return true;
+ }
+ });
+ }),
+ 30_000
+);
diff --git a/tests/lifecycle-supervisor-observers.test.ts b/tests/lifecycle-supervisor-observers.test.ts
new file mode 100644
index 000000000..f3ce3338a
--- /dev/null
+++ b/tests/lifecycle-supervisor-observers.test.ts
@@ -0,0 +1,508 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { observeSupervisorLifecycleEntry } from "../src/commands/host-ps.ts";
+import {
+ type LifecycleStateEntry,
+ readLifecycleState,
+ upsertLifecycleStateEntry,
+} from "../src/lib/lifecycle-runtime.ts";
+import {
+ type LifecycleSupervisorReceipt,
+ resolveLifecycleSupervisorRoot,
+ withLifecycleSupervisorControllerLock,
+} from "../src/lib/lifecycle-supervisor.ts";
+import {
+ classifySupervisorLifecycleHygiene,
+ inspectLifecycleStateEntries,
+ type LifecycleHygieneInspection,
+ repairProjectLifecycleSessions,
+} from "../src/lib/project-lifecycle-hygiene.ts";
+import { readSupervisorLifecycleActivity } from "../src/lib/runtime-projects.ts";
+
+function entry(
+ overrides: Partial = {}
+): LifecycleStateEntry {
+ return {
+ composeProject: "sample",
+ projectName: "sample",
+ branch: null,
+ sessionName: "sample--lifecycle",
+ backend: "supervisor",
+ ownershipToken: "synthetic-owner",
+ definitionHash: "synthetic-definition",
+ updatedAt: "2026-01-01T00:00:00.000Z",
+ processes: [
+ {
+ name: "worker",
+ windowName: "worker-window",
+ logPath: "/fixture/sample.log",
+ },
+ ],
+ ...overrides,
+ };
+}
+
+function receipt(
+ overrides: Partial = {}
+): LifecycleSupervisorReceipt {
+ return {
+ version: 1,
+ supervisorPid: 500,
+ outcome: "observed",
+ processes: [
+ {
+ version: 1,
+ name: "worker",
+ ownerToken: "synthetic-owner",
+ guardPid: 501,
+ processGroupId: 501,
+ guardStartTime: 123,
+ bootId: "synthetic-boot",
+ uid: 1000,
+ commandPid: 502,
+ phase: "running",
+ exitCode: null,
+ signal: null,
+ },
+ ],
+ liveProcessNames: ["worker"],
+ unknownProcessNames: [],
+ ...overrides,
+ };
+}
+
+function classify(
+ observation: LifecycleSupervisorReceipt | null,
+ options: {
+ active?: ReadonlySet | null;
+ state?: LifecycleStateEntry;
+ } = {}
+) {
+ const state = options.state ?? entry();
+ return classifySupervisorLifecycleHygiene({
+ entries: [state],
+ receipts: new Map([[state.composeProject, observation]]),
+ activeComposeProjects:
+ options.active === undefined ? new Set() : options.active,
+ now: new Date("2026-02-01T00:00:00.000Z"),
+ });
+}
+
+test("mux absence grants no stale-state authority over supervisor ownership", () => {
+ const result = inspectLifecycleStateEntries({
+ entries: [entry()],
+ sessionsByBackend: new Map(),
+ snapshot: [],
+ });
+ expect(result.staleEntries).toEqual([]);
+ expect(result.orphanedProcessGroups).toEqual([]);
+});
+
+test("supervisor hygiene distinguishes native absence from missing and unknown evidence", () => {
+ for (const observation of [
+ null,
+ receipt({ unknownProcessNames: ["worker"] }),
+ receipt({ outcome: "refused" }),
+ ]) {
+ const result = classify(observation);
+ expect(result.staleEntries).toEqual([]);
+ expect(result.orphanedSessions).toEqual([]);
+ expect(result.unverifiedSessions).toHaveLength(1);
+ }
+ expect(classify(receipt({ liveProcessNames: [] })).staleEntries).toHaveLength(
+ 1
+ );
+});
+
+test("supervisor orphan repair admission requires absent runtime and elapsed startup grace", () => {
+ expect(classify(receipt()).orphanedSessions).toHaveLength(1);
+ expect(
+ classify(receipt(), { active: new Set(["sample"]) }).orphanedSessions
+ ).toEqual([]);
+ expect(classify(receipt(), { active: null }).orphanedSessions).toEqual([]);
+ const recent = classify(receipt(), {
+ state: entry({ updatedAt: "2026-02-01T00:00:00.000Z" }),
+ });
+ expect(recent.orphanedSessions).toEqual([]);
+ expect(recent.unverifiedSessions).toHaveLength(1);
+});
+
+test("runtime lifecycle service status preserves unknown without consulting mux", async () => {
+ const state = entry();
+ const active = await readSupervisorLifecycleActivity({
+ entry: state,
+ projectDir: "/fixture",
+ inspect: () => Promise.resolve(receipt()),
+ });
+ expect(active.runningWindows?.has("worker-window")).toBe(true);
+ const unknown = await readSupervisorLifecycleActivity({
+ entry: state,
+ projectDir: "/fixture",
+ inspect: () => Promise.reject(new Error("native evidence unavailable")),
+ });
+ expect(unknown.runningWindows?.size).toBe(0);
+ expect(unknown.unknownWindows?.has("worker-window")).toBe(true);
+});
+
+test("host ps uses native ownership and does not invent process-group CPU totals", async () => {
+ const opts = {
+ entry: entry(),
+ projectDir: "/fixture",
+ projectRoot: "/fixture",
+ };
+ const active = await observeSupervisorLifecycleEntry({
+ ...opts,
+ inspect: () => Promise.resolve(receipt()),
+ });
+ expect(active.ownership).toBe("owned-healthy");
+ expect(active.sessionIds).toEqual([501]);
+ expect(active.liveCpuTimeMs).toBeNull();
+ expect(active.processGroupIds).toEqual([]);
+ const unknown = await observeSupervisorLifecycleEntry({
+ ...opts,
+ inspect: () =>
+ Promise.resolve(receipt({ unknownProcessNames: ["worker"] })),
+ });
+ expect(unknown.ownership).toBe("unknown");
+ expect(unknown.attention).toBe("ownership_unverified");
+});
+
+function repairInspection(
+ state: LifecycleStateEntry
+): LifecycleHygieneInspection {
+ return {
+ staleEntries: [],
+ orphanedProcessGroups: [],
+ orphanedSessions: [{ entry: state, classification: "owned-healthy" }],
+ unverifiedSessions: [],
+ runtimeAvailable: true,
+ };
+}
+
+for (const outcome of ["stopped", "unknown"] as const) {
+ test(`supervisor doctor repair retains state unless native stop is confirmed (${outcome})`, async () => {
+ const root = await mkdtemp(join(tmpdir(), "supervisor-observer-"));
+ const state = entry();
+ let effects = 0;
+ try {
+ await upsertLifecycleStateEntry({ projectDir: root, entry: state });
+ const result = await repairProjectLifecycleSessions({
+ projectDir: root,
+ projectRoot: root,
+ boundary: {
+ inspect: () => Promise.resolve(repairInspection(state)),
+ backends: () => {
+ throw new Error("Supervisor repair consulted mux");
+ },
+ inspectSupervisor: () => Promise.resolve(receipt()),
+ stopSupervisor: ({ ownershipToken }) => {
+ expect(ownershipToken).toBe("synthetic-owner");
+ effects += 1;
+ return Promise.resolve(
+ receipt({
+ outcome,
+ liveProcessNames: outcome === "stopped" ? [] : ["worker"],
+ })
+ );
+ },
+ },
+ });
+ expect(effects).toBe(1);
+ expect(result.failures.length).toBe(outcome === "stopped" ? 0 : 1);
+ expect((await readLifecycleState({ projectDir: root })).length).toBe(
+ outcome === "stopped" ? 0 : 1
+ );
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+}
+
+test("doctor rechecks runtime absence after native inspection before stop", async () => {
+ const root = await mkdtemp(join(tmpdir(), "supervisor-observer-"));
+ const state = entry();
+ let inspections = 0;
+ let effects = 0;
+ try {
+ await upsertLifecycleStateEntry({ projectDir: root, entry: state });
+ const result = await repairProjectLifecycleSessions({
+ projectDir: root,
+ projectRoot: root,
+ boundary: {
+ inspect: () => {
+ inspections += 1;
+ return Promise.resolve({
+ ...repairInspection(state),
+ orphanedSessions:
+ inspections < 3
+ ? [{ entry: state, classification: "owned-healthy" as const }]
+ : [],
+ });
+ },
+ inspectSupervisor: () => Promise.resolve(receipt()),
+ stopSupervisor: () => {
+ effects += 1;
+ return Promise.resolve(
+ receipt({ outcome: "stopped", liveProcessNames: [] })
+ );
+ },
+ },
+ });
+ expect(inspections).toBe(3);
+ expect(effects).toBe(0);
+ expect(result.failures).toHaveLength(1);
+ expect(await readLifecycleState({ projectDir: root })).toHaveLength(1);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("doctor cannot stop a newly adopted cleanup epoch with the same token and timestamp", async () => {
+ const root = await mkdtemp(join(tmpdir(), "supervisor-observer-"));
+ const state = entry({ cleanupEpoch: "synthetic-first-epoch" });
+ const adopted = { ...state, cleanupEpoch: "synthetic-adopted-epoch" };
+ let effects = 0;
+ try {
+ await upsertLifecycleStateEntry({ projectDir: root, entry: state });
+ const result = await repairProjectLifecycleSessions({
+ projectDir: root,
+ projectRoot: root,
+ boundary: {
+ inspect: () => Promise.resolve(repairInspection(state)),
+ inspectSupervisor: async () => {
+ await upsertLifecycleStateEntry({ projectDir: root, entry: adopted });
+ return receipt();
+ },
+ stopSupervisor: () => {
+ effects += 1;
+ return Promise.resolve(
+ receipt({ outcome: "stopped", liveProcessNames: [] })
+ );
+ },
+ },
+ });
+ expect(effects).toBe(0);
+ expect(result.failures).toHaveLength(1);
+ expect(
+ (await readLifecycleState({ projectDir: root }))[0]?.cleanupEpoch
+ ).toBe(adopted.cleanupEpoch);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("doctor rechecks exact persisted owner after native inspection and before stop", async () => {
+ const root = await mkdtemp(join(tmpdir(), "supervisor-observer-"));
+ const state = entry();
+ const replacement = entry({
+ ownershipToken: "synthetic-replacement",
+ updatedAt: "2026-01-02T00:00:00.000Z",
+ });
+ let effects = 0;
+ try {
+ await upsertLifecycleStateEntry({ projectDir: root, entry: state });
+ const result = await repairProjectLifecycleSessions({
+ projectDir: root,
+ projectRoot: root,
+ boundary: {
+ inspect: () => Promise.resolve(repairInspection(state)),
+ backends: () => {
+ throw new Error("Supervisor repair consulted mux");
+ },
+ inspectSupervisor: async () => {
+ await upsertLifecycleStateEntry({
+ projectDir: root,
+ entry: replacement,
+ });
+ return receipt();
+ },
+ stopSupervisor: () => {
+ effects += 1;
+ return Promise.resolve(
+ receipt({ outcome: "stopped", liveProcessNames: [] })
+ );
+ },
+ },
+ });
+ expect(effects).toBe(0);
+ expect(result.failures).toHaveLength(1);
+ expect(
+ (await readLifecycleState({ projectDir: root }))[0]?.ownershipToken
+ ).toBe(replacement.ownershipToken);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("one malformed project stays unknown without aborting global observers", async () => {
+ const { mkdir, writeFile } = await import("node:fs/promises");
+ const { readLifecycleObservations } = await import(
+ "../src/commands/host-ps.ts"
+ );
+ const { addLifecycleProcessServices } = await import(
+ "../src/lib/runtime-projects.ts"
+ );
+ const root = await mkdtemp(join(tmpdir(), "lifecycle-global-unreadable-"));
+ const bad = join(root, "bad", ".hack");
+ const good = join(root, "good", ".hack");
+ try {
+ await mkdir(join(bad, ".internal", "lifecycle"), { recursive: true });
+ await mkdir(good, { recursive: true });
+ await writeFile(
+ join(bad, ".internal", "lifecycle", "state.json"),
+ '{"entries":[{}]}'
+ );
+ const common = {
+ projectDirName: ".hack" as const,
+ createdAt: "2026-01-01T00:00:00Z",
+ };
+ const registry = {
+ version: 1 as const,
+ projects: [
+ {
+ ...common,
+ id: "bad",
+ name: "bad",
+ projectDir: bad,
+ repoRoot: join(root, "bad"),
+ },
+ {
+ ...common,
+ id: "good",
+ name: "good",
+ projectDir: good,
+ repoRoot: join(root, "good"),
+ },
+ ],
+ };
+ const host = await readLifecycleObservations({
+ registry,
+ filter: undefined,
+ snapshot: [],
+ });
+ expect(host).toHaveLength(1);
+ expect(host[0]?.reason).toBe("state_unreadable");
+ expect(host[0]?.ownership).toBe("unknown");
+ const runtime = await addLifecycleProcessServices({
+ runtime: [
+ {
+ project: "bad",
+ workingDir: bad,
+ services: new Map(),
+ isGlobal: false,
+ },
+ {
+ project: "good",
+ workingDir: good,
+ services: new Map(),
+ isGlobal: false,
+ },
+ ],
+ });
+ expect(runtime).toHaveLength(2);
+ expect(
+ runtime[0]?.services.get("lifecycle-state")?.containers[0]?.state
+ ).toBe("unknown");
+ expect(runtime[1]?.services.size).toBe(0);
+ const { findOrphanRuntimeProjects } = await import(
+ "../src/lib/project-runtime-hygiene.ts"
+ );
+ expect(await findOrphanRuntimeProjects({ runtime })).toEqual([]);
+ const real = {
+ id: "synthetic-real-container",
+ project: "bad",
+ service: "lifecycle-state",
+ state: "running",
+ status: "Up",
+ name: "synthetic-real-container",
+ ports: "",
+ workingDir: bad,
+ image: null,
+ labels: {},
+ mounts: [],
+ networks: [],
+ };
+ const withExisting = await addLifecycleProcessServices({
+ runtime: [
+ {
+ project: "bad",
+ workingDir: bad,
+ services: new Map([
+ [
+ "lifecycle-state",
+ { service: "lifecycle-state", containers: [real] },
+ ],
+ ]),
+ isGlobal: false,
+ },
+ ],
+ });
+ expect(
+ withExisting[0]?.services.get("lifecycle-state")?.containers
+ ).toHaveLength(2);
+ expect(
+ (await findOrphanRuntimeProjects({ runtime: withExisting }))[0]
+ ?.containerIds
+ ).toEqual([real.id]);
+ await expect(readLifecycleState({ projectDir: bad })).rejects.toThrow(
+ "state.json"
+ );
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("supervisor control state is outside the checkout", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "supervisor-root-"));
+ try {
+ const root = resolveLifecycleSupervisorRoot({
+ projectDir: directory,
+ composeProject: "sample",
+ });
+ expect(root.startsWith("/synthetic-checkout/")).toBe(false);
+ expect(root).toContain("/lifecycle-supervisors/");
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
+test("doctor holds the controller lock through native stop and state removal", async () => {
+ const root = await mkdtemp(join(tmpdir(), "supervisor-observer-"));
+ const state = entry();
+ let acquired = false;
+ let competitor: Promise | undefined;
+ try {
+ await upsertLifecycleStateEntry({ projectDir: root, entry: state });
+ const result = await repairProjectLifecycleSessions({
+ projectDir: root,
+ projectRoot: root,
+ boundary: {
+ inspect: () => Promise.resolve(repairInspection(state)),
+ inspectSupervisor: () => Promise.resolve(receipt()),
+ stopSupervisor: async () => {
+ competitor = withLifecycleSupervisorControllerLock({
+ projectDir: root,
+ composeProject: state.composeProject,
+ run: async () => {
+ acquired = true;
+ expect(
+ await readLifecycleState({ projectDir: root })
+ ).toHaveLength(0);
+ },
+ });
+ await Bun.sleep(50);
+ expect(acquired).toBe(false);
+ return receipt({ outcome: "stopped", liveProcessNames: [] });
+ },
+ },
+ });
+ expect(result.failures).toHaveLength(0);
+ await competitor;
+ expect(acquired).toBe(true);
+ } finally {
+ await competitor;
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/tests/models/tla/lifecycle-controller/COVERAGE.md b/tests/models/tla/lifecycle-controller/COVERAGE.md
new file mode 100644
index 000000000..46fa76951
--- /dev/null
+++ b/tests/models/tla/lifecycle-controller/COVERAGE.md
@@ -0,0 +1,79 @@
+# Opt-in lifecycle controller ownership model
+
+This finite model covers the CLI controller above the standalone physical-process
+model. It has one instance, one creating operation and at most one adopting
+operation. There is one initial launch and at most one incremental add. The native
+owner token remains stable (`token = 1`), while adoption advances the cleanup epoch
+from 1 to 2. It does not model arbitrary strings or independent project instances.
+
+| Transition/property | Source owner | Regression boundary |
+| --- | --- | --- |
+| `Begin`, `PublishPending`, `Launch`; `DurableBeforeLaunch` | `createSupervisorLifecycleController.begin`, controller lock, common state publication before native `serve` | `tests/lifecycle-supervisor-controller.test.ts`; native launch failure retains a durable cleanup owner |
+| `Accept`; `StableNativeOwner` | Controller `accept` and `publish`; native framed launch receipt | Controller readback and incremental real-process tests |
+| `Adopt`, `Abort`; `NoAdoptedStop` | `decide` preserves the native ownership token and writes a new cleanup epoch; `stopOwned` checks that epoch before signaling | Creator abort after another controller adopts cannot stop the adopted process |
+| `LoseReply`, `PublishLate`, `FailUnknown`; `LateFailurePublished` | Incremental launch failure inspects native state, accepts verified receipts and publishes before reporting the uncertain outcome | The real controller regression drops an incremental reply after its native effect and asserts both common-state publication before failure and native liveness of the late command |
+| `Add`, `Stop`; `StoppedPreventsAdd` | Per-instance controller lock plus native launch/stop admission lock | `tests/lifecycle-supervisor-controller.test.ts`; `packages/lifecycle-supervisor/tests/incremental.rs` concurrent add/stop and add after completed stop |
+
+Positive TLC exploration: **21 distinct states**, queue exhausted, depth 10.
+`negative.cfg` removes only the cleanup epoch fence: `Abort` violates
+`NoAdoptedStop`, with `unsafeStop = TRUE`, `epoch = 2`, `creatorEpoch = 1`,
+`adopted = TRUE` and `stage = "Aborted"` in the same state. `pending.cfg` removes
+only the pre-launch publication fence: `Launch` violates `DurableBeforeLaunch`
+with `unsafeLaunch = TRUE`, `pending = FALSE`, `live = TRUE` and
+`stage = "AwaitReply"` in the same state. Both controls exit 12 for the intended
+invariant, rather than a parser or incomplete-state error. The maintained registry
+pins the count and both complete same-state witnesses.
+
+Checker: pinned TLA+ 1.7.4 jar (TLC reports 2.19), Java 17, two workers, 512 MiB
+maximum heap and breadth-first search. Apalache is not qualified here.
+
+The model treats each controller/native lock region as atomic. Therefore stop can
+order before or after add but cannot interrupt its protected launch/publication.
+The abstract stop proves absence; real unknown outcomes retain ownership and do
+not authorize deletion. `nativePublished` represents the command receipt for the
+current launch; it is cleared for add so a lost incremental reply must publish a
+fresh verified receipt, rather than rely on the earlier launch's publication.
+No fairness or eventual command exit is assumed.
+
+Assumptions: private lock integrity, serialized writers for this instance, atomic
+state publication, native readback that correctly separates live/unknown/absent,
+and no hidden late effect after the protected native admission has settled. A
+transport timeout can precede settlement: implementation reconciliation and native
+locks handle that boundary, not an inference from socket absence.
+
+Omissions: actual PID/session identity, native guardian failure, byte framing,
+socket replacement, credentials, filesystem sync/power loss, scheduler timing,
+multiple processes' independent liveness, multiple instances, command output and
+default-runner migration. Physical-process ownership remains covered separately
+by `LifecycleSupervisor.tla` and Rust tests. This small controller model is not
+whole-product, macOS runtime, packaging or installed-artifact qualification.
+
+The finite model assumes `Begin` is admitted only after the executable preflight.
+It does not prove executable selection, version compatibility, canonical paths,
+legacy JSON recovery, or the meaning of a missing native directory. Those are
+implementation boundaries, covered by these real-controller regressions:
+
+| Boundary / transition | Required outcome | Regression |
+| --- | --- | --- |
+| Absent → preflight fails | No ownership entry; down remains usable | `missing companion fails before ownership and down remains usable` |
+| Pending, no launch attempted → down | Retire without requiring an executable | `never-attempted ownership can be retired without the companion` |
+| Pending → launch attempted | Publish the attempt before native dispatch; an uncertain result retains ownership | Lost incremental reply regression and atomic state-publication tests |
+| Running → HOME or path spelling changes | Use the persisted native root, never create a replacement owner | `recorded native root survives a changed home and a symlinked checkout` |
+| Running → recorded root missing | Unconfirmed; retain ownership | `missing recorded root never grants retirement authority` |
+
+`supervisorLaunchAttempted = false` is an explicit durable observation, not an
+inference from an empty process array. Older entries without that field remain
+conservative. An absent PID alone cannot rule out a lost launch reply. Native
+root loss is outside the model's abstract successful-stop action.
+
+The native observation assumption includes serialization with in-flight stop
+admission and a `running` phase for adoption. Real regressions are
+`status_waits_for_inflight_stop_before_reporting_adoptable_liveness` and
+`a command already retiring cannot be adopted as healthy`. The finite model's
+atomic lock region does not simulate the phase-publication interval itself.
+
+Generation retirement fences delayed initial dispatch before native admission;
+this is outside the model's atomic native-launch step. The implementation test
+`completed_stop_retires_a_delayed_initial_launch_before_any_authored_effect`
+stops before delivering the launch frame, then proves no command receipt was
+created. New generations require fresh tokens; retired markers are not pruned.
diff --git a/tests/models/tla/lifecycle-controller/LifecycleController.tla b/tests/models/tla/lifecycle-controller/LifecycleController.tla
new file mode 100644
index 000000000..cd89601e2
--- /dev/null
+++ b/tests/models/tla/lifecycle-controller/LifecycleController.tla
@@ -0,0 +1,79 @@
+------------------------- MODULE LifecycleController -------------------------
+EXTENDS Naturals
+CONSTANTS SkipEpochFence, SkipPendingFence
+VARIABLES stage, token, epoch, creatorEpoch, adopted, live, pending,
+ nativePublished, stopped, unsafeStop, unsafeLaunch, unknown
+vars == <>
+
+Init == /\ stage = "Idle" /\ token = 0 /\ epoch = 0 /\ creatorEpoch = 0
+ /\ adopted = FALSE /\ live = FALSE /\ pending = FALSE
+ /\ nativePublished = FALSE /\ stopped = FALSE
+ /\ unsafeStop = FALSE /\ unsafeLaunch = FALSE /\ unknown = FALSE
+
+Begin == /\ stage = "Idle" /\ ~stopped
+ /\ stage' = "Starting" /\ token' = 1
+ /\ epoch' = 1 /\ creatorEpoch' = 1
+ /\ UNCHANGED <>
+PublishPending == /\ stage = "Starting" /\ ~pending
+ /\ pending' = TRUE
+ /\ UNCHANGED <>
+Launch == /\ stage = "Starting" /\ (pending \/ SkipPendingFence)
+ /\ stage' = "AwaitReply" /\ live' = TRUE
+ /\ unsafeLaunch' = (unsafeLaunch \/ ~pending)
+ /\ UNCHANGED <>
+Accept == /\ stage = "AwaitReply"
+ /\ stage' = "Ready" /\ nativePublished' = TRUE
+ /\ UNCHANGED <>
+LoseReply == /\ stage = "AwaitReply"
+ /\ stage' = "InspectLate"
+ /\ UNCHANGED <>
+PublishLate == /\ stage = "InspectLate" /\ live
+ /\ stage' = "LatePublished" /\ nativePublished' = TRUE
+ /\ UNCHANGED <>
+FailUnknown == /\ stage = "LatePublished" /\ nativePublished
+ /\ stage' = "Failed" /\ unknown' = TRUE
+ /\ UNCHANGED <>
+Adopt == /\ stage \in {"Ready", "Failed"} /\ live /\ ~adopted
+ /\ epoch = 1 /\ stage' = "Adopted" /\ epoch' = 2
+ /\ adopted' = TRUE
+ /\ UNCHANGED <>
+Abort == /\ stage \in {"Ready", "Failed", "Adopted"} /\ live
+ /\ (epoch = creatorEpoch \/ SkipEpochFence)
+ /\ stage' = "Aborted" /\ live' = FALSE
+ /\ unsafeStop' = (unsafeStop \/ (epoch # creatorEpoch))
+ /\ UNCHANGED <>
+Stop == /\ stage \in {"Ready", "Failed", "Adopted"}
+ /\ stage' = "Stopped" /\ live' = FALSE /\ stopped' = TRUE
+ /\ UNCHANGED <>
+Add == /\ stage = "Adopted" /\ live /\ ~stopped
+ /\ stage' = "AwaitReply" /\ nativePublished' = FALSE /\ unknown' = FALSE
+ /\ UNCHANGED <>
+Next == Begin \/ PublishPending \/ Launch \/ Accept \/ LoseReply
+ \/ PublishLate \/ FailUnknown \/ Adopt \/ Abort \/ Stop \/ Add
+Spec == Init /\ [][Next]_vars
+TypeOK == /\ stage \in {"Idle", "Starting", "AwaitReply", "Ready", "InspectLate",
+ "LatePublished", "Failed", "Adopted", "Aborted", "Stopped"}
+ /\ token \in 0..1 /\ epoch \in 0..2 /\ creatorEpoch \in 0..1
+ /\ \A b \in {adopted, live, pending, nativePublished, stopped,
+ unsafeStop, unsafeLaunch, unknown} : b \in BOOLEAN
+NoAdoptedStop == ~unsafeStop
+DurableBeforeLaunch == ~unsafeLaunch
+LateFailurePublished == unknown => nativePublished
+StoppedPreventsAdd == stopped => ~live
+StableNativeOwner == (epoch > 0) => token = 1
+=============================================================================
diff --git a/tests/models/tla/lifecycle-controller/negative.cfg b/tests/models/tla/lifecycle-controller/negative.cfg
new file mode 100644
index 000000000..f6748721f
--- /dev/null
+++ b/tests/models/tla/lifecycle-controller/negative.cfg
@@ -0,0 +1,4 @@
+SPECIFICATION Spec
+CONSTANTS SkipEpochFence = TRUE SkipPendingFence = FALSE
+INVARIANTS TypeOK NoAdoptedStop DurableBeforeLaunch LateFailurePublished StoppedPreventsAdd StableNativeOwner
+CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-controller/pending.cfg b/tests/models/tla/lifecycle-controller/pending.cfg
new file mode 100644
index 000000000..6ab799b7b
--- /dev/null
+++ b/tests/models/tla/lifecycle-controller/pending.cfg
@@ -0,0 +1,4 @@
+SPECIFICATION Spec
+CONSTANTS SkipEpochFence = FALSE SkipPendingFence = TRUE
+INVARIANTS TypeOK NoAdoptedStop DurableBeforeLaunch LateFailurePublished StoppedPreventsAdd StableNativeOwner
+CHECK_DEADLOCK FALSE
diff --git a/tests/models/tla/lifecycle-controller/positive.cfg b/tests/models/tla/lifecycle-controller/positive.cfg
new file mode 100644
index 000000000..29c68afda
--- /dev/null
+++ b/tests/models/tla/lifecycle-controller/positive.cfg
@@ -0,0 +1,4 @@
+SPECIFICATION Spec
+CONSTANTS SkipEpochFence = FALSE SkipPendingFence = FALSE
+INVARIANTS TypeOK NoAdoptedStop DurableBeforeLaunch LateFailurePublished StoppedPreventsAdd StableNativeOwner
+CHECK_DEADLOCK FALSE