From c98e435402f91245daac716ca3ce12750401c72f Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:23:47 +0000 Subject: [PATCH 1/5] fix(native): keep status reads from contending with startup --- docs/guides/native-candidate.md | 58 +++- docs/reference/native-authored-commands.md | 8 +- docs/reference/v5-model-coverage.md | 2 + packages/runtime-core/src/graph_cli.rs | 11 + packages/runtime-core/src/graph_cli/native.rs | 3 + packages/runtime-core/src/main.rs | 4 +- packages/runtime-core/src/provider/engine.rs | 115 +++---- .../src/provider/engine/inspection.rs | 51 ++++ .../src/provider/graph/foreground.rs | 24 +- .../provider/graph/foreground/publishers.rs | 57 ++++ .../src/provider/graph/host_relay.rs | 23 ++ .../src/provider/graph/live_owner_cleanup.rs | 75 +++-- .../runtime-core/src/provider/graph/mod.rs | 16 +- .../src/provider/graph/native/foreground.rs | 25 +- .../provider/graph/native/foreground/tests.rs | 23 ++ .../src/provider/graph/native/runtime.rs | 99 ++++-- .../provider/graph/native/runtime/tests.rs | 41 +++ .../runtime-core/src/provider/graph/probes.rs | 6 +- .../src/provider/graph/startup/runtime.rs | 281 +++++++++++++----- .../graph/startup/runtime/rebind/tests.rs | 7 + .../src/provider/graph/startup_failure.rs | 65 ++++ .../runtime-core/src/provider/graph/status.rs | 195 ++++++++++++ .../runtime-core/src/provider/lifecycle.rs | 18 +- packages/runtime-core/src/provider/state.rs | 117 +++++++- scripts/lib/tla-runtime-models.ts | 13 + .../native-authored-graph-protocol.ts | 6 +- .../native-authored-project-observe.ts | 15 +- src/backends/native-project-down.ts | 272 +++++++++-------- src/backends/native-project-observe.ts | 60 +++- src/backends/native-runtime-client.ts | 3 +- src/lib/native-authored-command.ts | 2 + .../tla/native-status-start/COVERAGE.md | 56 ++++ .../tla/native-status-start/StatusStart.tla | 73 +++++ tests/models/tla/native-status-start/flow.mmd | 12 + tests/models/tla/native-status-start/flow.svg | 66 ++++ .../tla/native-status-start/negative.cfg | 4 + .../tla/native-status-start/positive.cfg | 4 + tests/native-authored-graph-protocol.test.ts | 17 ++ tests/native-authored-project-observe.test.ts | 44 ++- tests/native-project-down.test.ts | 73 ++++- tests/native-project-observe.test.ts | 51 +++- 41 files changed, 1691 insertions(+), 404 deletions(-) create mode 100644 packages/runtime-core/src/provider/engine/inspection.rs create mode 100644 packages/runtime-core/src/provider/graph/status.rs create mode 100644 tests/models/tla/native-status-start/COVERAGE.md create mode 100644 tests/models/tla/native-status-start/StatusStart.tla create mode 100644 tests/models/tla/native-status-start/flow.mmd create mode 100644 tests/models/tla/native-status-start/flow.svg create mode 100644 tests/models/tla/native-status-start/negative.cfg create mode 100644 tests/models/tla/native-status-start/positive.cfg diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 67d63f56b..983ff26f8 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -599,11 +599,11 @@ HTTPS frontend it says that routes are not served on this host. Ctrl-C prints th stop sequence and, once owned cleanup is confirmed, one stopped line that says whether named volumes and the stopped run mapping were kept (`Native project stopped.` when no run mapping was published). Until the ready line, -startup holds the runtime's exclusive provider lease, so graph admission cannot -interleave with another operation. A `hack ps` from another terminal during that -window waits briefly, then reports that the runtime is busy (`runtime_busy`, -exit 0) instead of failing. After the ready line, `hack ps` and `hack logs` run -alongside the foreground command. +startup uses the runtime's exclusive provider lease for its effect stages. +`hack ps` observes without that lease, including when another project is starting +in the same pool. If the journal changes during its read, it reports +`runtime_busy` (exit 0); retry for a current snapshot. After the ready line, +`hack ps` and `hack logs` run alongside the foreground command. While the project runs, foreground `up` reports each service container that stops, as attached Compose `up` does (`SERVICE exited with code N`), including one-shot @@ -1881,9 +1881,11 @@ owner could not be authenticated, while `runtime_degraded` means its runtime che failed. A stopped candidate runtime is an ordinary state for `ps`: with a mapping present it exits 0 and reports `runtime_not_running` (no phase, no service rows), and text output prints "Native runtime is not running; no services are running." -A provider lease held for the whole bounded read, for example by a starting -foreground `up`, is also an ordinary state: `ps` exits 0 and reports -`runtime_busy` (no phase, no service rows). Reads retry brief lease contention. +Status reads take no provider lease and never reap publishers or write journals. +A journal change during a read is an ordinary transitional state: `ps` exits 0 +and reports `runtime_busy` (no phase, no service rows). Status is presentation, +not a storage-witness or source-continuity proof; mutating commands keep their +locked verification. `hack ps --json` prints the standard `{ok: true, data}` envelope; `data` holds `backend: "native"`, `status` (`not_started`, `pending`, `runtime_not_running`, `runtime_busy`, `observed`, `owner_unconfirmed` or `runtime_degraded`), `run`, @@ -3699,7 +3701,7 @@ owner as dead or bypass its cleanup checks. `hack up` and `hack restart` refuse a run whose foreground publication is still present and name the next step: a running owner must be stopped with Ctrl-C or `hack down`; an owner that exited while its services were running is stopped by -`hack down --recover`. That command selects the run's exact receipt with the +plain `hack down` (or `hack down --recover`). That command selects the run's exact receipt with the read-only `graph foreground-owner --run-id RUN --json` and then uses the same-boot `graph recover-live-owner` described below, retaining data and retiring the stale publication. If its recorded frontend also exited before @@ -3727,11 +3729,39 @@ and data is retained. Then `hack up` restores the run. An owner that exited after its restore was admitted is still reported without a supported retirement. -Read-only `hack ps` and `hack logs` hold the provider lease only briefly. Native -startup, retained-start eligibility, restore admission, inspections inside -mutating commands and owned foreground cleanup wait out such a holder within a -bound instead of failing, so polling status during `up` or `restart` does not -make them fail or strand a foreground publication. +`hack ps` uses lock-free reads for both adopted and authored projects. Finite +`hack logs` and inspections used by mutating commands retain their existing +lease behavior. Startup's initial bounded wait preserves its deadline and +names the recorded holder kind when it expires. That advisory record grants no +authority. A second concurrent mutation still refuses under the existing rules. +Authored `up` readiness still uses locked `status`; only authored `ps` uses the +presentation-only `observe` action. Contention between two starts is unchanged. + +For adopted projects, plain `down` also selects exact-proof recovery of a +`failed-retained` start when every startup binding is settled (`released` or +`completed`), initializers are resolved, and the journal has no pending effect. +Recovery rechecks owner death, boot, incarnation, selected receipt and resource +inventories under its lease. Unsettled bindings, uncertain owners, corrupt or +partial journals and changed identities still refuse by name; they grant no +cleanup authority. Authored pre-Ready recovery is unchanged. + +Failure evidence is retained by startup and effectful verification, and by owned +cleanup for listeners that exited before cleanup stops them. The first recorded +failure wins. Cleanup captures in memory and includes it in the required cleanup +journal commit, so an extra diagnostic write cannot obstruct stopping. Settling +an exited listener may add up to ten seconds per inspected listener to cleanup. +Incomplete-rebind cleanup does not capture new listener-exit evidence; restore +cleanup admits no listener children. Status reports live classifications but +never writes them. If an owner crashes after a post-ready listener exit and +before cleanup, dead-owner recovery can observe container state but cannot +reconstruct that host child's exit. This was also unguaranteed when recording +depended on an incidental `ps` call. + +HACK-2110 live acceptance remains a lead-owned Mac check: poll `ps` every 0.5 s +across ten starts, require all starts and polls to succeed, and verify plain +`down` after an injected post-service startup failure. Also poll project A while +project B starts in the same pool. Portable tests and the interleaving model do +not qualify that VM behavior. For a fully ready graph whose foreground owner died while its pool remains on the same boot, `graph recover-live-owner --run-id RUN --expect-receipt SHA256` selects diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 6115bd2f3..4b23c2d80 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -192,7 +192,7 @@ retained attempt is inspected. ## Saved-run `ps` -`ps` and `ps --json` issue one authenticated current status request for the exact +`ps` and `ps --json` issue one authenticated `observe` request for the exact saved project and explicit branch selection. Text output lists service, container, current state and current health. JSON uses the ordinary result envelope with `data.backend: "native"`, `data.status`, `data.run`, historical `data.phase`, and @@ -200,6 +200,12 @@ current state and current health. JSON uses the ordinary result envelope with Only fresh runtime observations supply state and health. Stored readiness and phase do not prove that a service is still live. +Observation takes no provider lease, executes no source checks or storage-witness +helpers, and writes no runtime journal. If the journal changes during the read, +`ps` reports `runtime_busy` with no phase or service rows; retry for a fresh view. +The owner's separate `status` action retains full locked verification for `up` +readiness. An `observed` presentation reply cannot satisfy a `status` request. + A project with no saved attempt reports `not_started`; a retained startup intent without a Ready mapping reports `pending` with no confirmed service rows. `pending` covers a start that is still in progress and a failed start whose reserved intent was diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 1d7843b46..5780d5ce4 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,6 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | +| Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. | | CI cancellation must not lose merged-change coverage; only exact-head full evidence may qualify a prerelease | No maintained GitHub scheduler model. Finite fixture interleavings cover predecessor success/failure/cancellation, two competing full/scoped runs and two rerun attempts | `ci.yml` serializes next pushes in one non-cancelling group (GitHub cancels only the replaced pending run), cancels stale PR runs, and requires a successful before run for selection. `selectPrereleaseCiRun` reads current attempts; `requireBranchChecks` binds suite and job check IDs. `ci-trusted-policy`, `prerelease-plan` and `release-prepare-gate` regressions reject cancelled predecessors, green subsets, stale checks, skipped hosts and changed attempts | API snapshots and GitHub group/event semantics are assumptions; races after the last read remain possible. Publication rechecks the frozen protected head and full evidence but does not transactionally lock GitHub. Hosted dispatch/cancellation checks qualify wiring; no scheduler liveness or formal refinement claim. | | Native log follower versus lifecycle mutation, frontend loss, stopped container and transport loss | No new state model: the follower observes without a mutation lease and grants no cleanup authority | `provider/graph/log_follow.rs` and its native counterpart pin Ready admission, container and start generation; `provider/engine/log_follow.rs` bounds framing and distinguishes real EOF from stopped-service idle cutoff. `lifecycle_log_follow_tests.rs` keeps the production transport on a fake Engine socket active while acquiring startup and operation leases, with a same-lock contention control and output while each lease is held. `native-log-follow-source.test.ts` guards the observer-only route; process tests cover cancellation, frontend loss, real output-pipe closure while idle, replacement-container restart (including retained adopted run/owner IDs) with one marker, NDJSON admission failures, truncated/partial output and slow idle-end delivery. `native-log-restart.test.ts` covers bounded re-admission, unchanged-container refusal and prompt down; `native-log-selection.test.ts` covers strict finite admission responses and owned-child cancellation/timeouts; advisory markers grant no authority | 4 KiB payload envelopes, 8 KiB wire lines, 64 Ki-character retained payload prefixes excluding Engine timestamps, 30-second restart wait, 1000-line new-container backlog. The portable lease test fakes identity/state checks; it does not exercise host process/disk auditing, graph admission or the VM relay. Real relay EOF on stop/down/restart, VM loss and concurrent lifecycle latency require Apple Silicon acceptance. No performance or whole-product parity claim. | | Foreground group interrupt; publisher exit before authenticated stop; dead owner's pending cleanup enrollment and bridge release | native-publisher-stop | [Process adapter](../../src/backends/native-project-process.ts) isolates Compose owners regardless of hooks; Rust `process::publisher_command` isolates publishers; `legacy_cleanup_enrollment` binds the old coordinator and inventories before a distinct recovery intent. [Group regression](../../tests/native-foreground-group-interrupt.test.ts) signals the whole group for fresh/restore owners; Rust real-publisher process, frontend-loss, exact child-reaping and enrollment tests check grouping, lifetime wakeups and stale selectors. | One graph, one publisher, one crash, one identity replacement and one audited successor boot; 168 states. Publisher exit observation is separate from lease-bound reaping/proof commitment; `NoUncommittedExit` rejects owner cleanup while proof is pending. A busy lease stutters without delaying stop control. SIGHUP requests retaining cleanup; a reaped publisher enters an observable publisher-down state until explicit stop. Native process tests and frontend framing/ps tests cover signal delivery and notices. Dependency release is a separate post-completion step; `NoStrandedReservation` rejects success with a held reservation. Real reservation-file regressions check retry, capacity reuse and refusal fences. Separate stopped-receipt and intent-completion commits are collapsed; TypeScript dispatch and Rust completion-digest regressions cover that interruption window. Publication-registry sync/rename is abstracted and covered by portable commit-interruption tests. Kernel identity, socket/inode proofs, guest observations, durable storage and existing next.13 home recovery remain implementation/Apple Silicon VM gates. No eventual recovery is asserted. | @@ -76,6 +77,7 @@ liveness/reachability controls have their own documented assumptions. | Model | Positive distinct states | Configurations | | --- | ---: | ---: | +| [native-status-start](../../tests/models/tla/native-status-start/StatusStart.tla) | 110 | 2 | | [native-publisher-stop](../../tests/models/tla/native-publisher-stop/PublisherStop.tla) | 168 | 7 | | [shutdown-rebase](../../tests/models/tla/shutdown-rebase/ShutdownRebase.tla) | 43 | 3 | | [native-plan-reconcile](../../tests/models/tla/native-plan-reconcile/Reconcile.tla) | 747,656 | 4 | diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index e86f4e0a9..09e80f400 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -689,6 +689,7 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result Result graph::reconcile_export(candidate, run), "inspect" => serde_json::to_value(graph::inspect(candidate, run)?) .map_err(|_| CandidateError::new("graph_output", "Cannot encode graph snapshot.")), + "status" => { + #[cfg(target_os = "macos")] + { + serde_json::to_value(graph::status::inspect(candidate, run)?).map_err(|_| invalid()) + } + #[cfg(not(target_os = "macos"))] + { + Err(invalid()) + } + } "observe" => { #[cfg(target_os = "macos")] { diff --git a/packages/runtime-core/src/graph_cli/native.rs b/packages/runtime-core/src/graph_cli/native.rs index f4b9ea98a..0e802da71 100644 --- a/packages/runtime-core/src/graph_cli/native.rs +++ b/packages/runtime-core/src/graph_cli/native.rs @@ -160,6 +160,7 @@ pub(super) fn command(candidate: &Candidate, args: &[&str]) -> Result "status", + Some("observe") => "observe", Some("cleanup") => "cleanup", _ => return Err(refused()), }; @@ -172,6 +173,8 @@ pub(super) fn command(candidate: &Candidate, args: &[&str]) -> Result --file --expect-plan [--profile ] [--reconcile] [--json] hack-local project verify-source --project --file --expect-plan [--profile ] [--json] hack-local graph run|restart|restore --project --file --expect-plan --run-id <32-hex> --ready ... [--source-revision ] [--live-source] [--profile ] [--timeout-seconds ] [--json] - hack-local graph inspect|observe|reconcile|archive|export|reconcile-export|prune --run-id <32-hex> [--json] + hack-local graph inspect|status|observe|reconcile|archive|export|reconcile-export|prune --run-id <32-hex> [--json] hack-local graph cleanup --run-id <32-hex> [--remove-data] [--json] hack-local graph native plan --source-file [--json] hack-local graph native run|serve --source-file --expect-review <64-hex> [--environment-stdin] [--timeout-seconds <1..300>] [--json] hack-local graph native inspect|cleanup --run-id <32-hex> [--json] - hack-local graph native control --run-id <32-hex> --action status|cleanup [--json] + hack-local graph native control --run-id <32-hex> --action status|observe|cleanup [--json] hack-local graph inspect-shutdown-rebase --run-id <32-hex> [--json] hack-local graph recover-shutdown-rebase --run-id <32-hex> --expect-selection <64-hex> [--json] hack-local graph recover-cleanup --run-id <32-hex> --expect-receipt [--json] diff --git a/packages/runtime-core/src/provider/engine.rs b/packages/runtime-core/src/provider/engine.rs index 1a5376d5e..63c4b0447 100644 --- a/packages/runtime-core/src/provider/engine.rs +++ b/packages/runtime-core/src/provider/engine.rs @@ -12,6 +12,7 @@ use std::io::Read; use std::path::Path; use std::time::{Duration, Instant}; +pub(in crate::provider) mod inspection; #[cfg(any(target_os = "macos", test))] pub(in crate::provider) mod log_follow; #[cfg(any(all(target_os = "macos", feature = "environment-launcher"), test))] @@ -384,53 +385,7 @@ impl<'a> Engine<'a> { Ok(events) } pub(super) fn probe_status(&self, id: &str) -> Result, CandidateError> { - if id.len() != 64 || !id.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(failure("Invalid probe container identity.")); - } - self.guest.verify()?; - let bytes = match self.transport.request_bytes( - Method::GET, - &format!("/v1.53/containers/{id}/archive?path=/run/hack-http-probe-state/status"), - None, - ) { - Ok(bytes) => bytes, - Err(e) if e.code == "engine_not_found" => { - self.guest.verify()?; - return Ok(None); - } - Err(e) => return Err(e), - }; - if bytes.len() > 32_768 { - return Err(failure("Probe archive exceeds its budget.")); - } - let mut archive = tar::Archive::new(bytes.as_slice()); - let mut entries = archive - .entries() - .map_err(|_| failure("Invalid probe archive."))?; - let mut entry = entries - .next() - .ok_or_else(|| failure("Empty probe archive."))? - .map_err(|_| failure("Invalid probe archive."))?; - if entry - .path() - .map_err(|_| failure("Invalid probe path."))? - .as_ref() - != Path::new("status") - || !entry.header().entry_type().is_file() - || entry.size() > 256 - { - return Err(failure("Unexpected probe archive entry.")); - } - let mut raw = String::new(); - entry - .read_to_string(&mut raw) - .map_err(|_| failure("Invalid probe status."))?; - drop(entry); - if entries.next().is_some() { - return Err(failure("Multiple probe archive entries.")); - } - self.guest.verify()?; - Ok(Some(raw)) + read_probe_status(&self.transport, id, || self.guest.verify()) } #[cfg(target_os = "macos")] pub(in crate::provider) fn job_logs( @@ -539,6 +494,7 @@ impl<'a> Engine<'a> { deadline: std::time::Instant, cancelled: impl Fn() -> bool, ) -> Result { + let mut holder = None; loop { if cancelled() { return Err(CandidateError::new( @@ -548,13 +504,16 @@ impl<'a> Engine<'a> { } let remaining = deadline.saturating_duration_since(std::time::Instant::now()); if remaining.is_zero() { - return Err(CandidateError::new( - "provider_busy", - "Provider lease acquisition expired before admission.", - )); + return Err(holder.unwrap_or_else(|| { + CandidateError::new( + "provider_busy", + "Provider lease acquisition expired before admission.", + ) + })); } match Self::connect(candidate) { Err(error) if error.code == "provider_busy" => { + holder = Some(error); std::thread::sleep(remaining.min(Duration::from_millis(50))) } result => return result, @@ -1009,6 +968,60 @@ pub fn info(candidate: &Candidate) -> Result { Observer::connect(candidate)?.info() } +fn read_probe_status( + transport: &Transport, + id: &str, + verify: impl Fn() -> Result<(), CandidateError>, +) -> Result, CandidateError> { + if id.len() != 64 || !id.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(failure("Invalid probe container identity.")); + } + verify()?; + let bytes = match transport.request_bytes( + Method::GET, + &format!("/v1.53/containers/{id}/archive?path=/run/hack-http-probe-state/status"), + None, + ) { + Ok(bytes) => bytes, + Err(e) if e.code == "engine_not_found" => { + verify()?; + return Ok(None); + } + Err(e) => return Err(e), + }; + if bytes.len() > 32_768 { + return Err(failure("Probe archive exceeds its budget.")); + } + let mut archive = tar::Archive::new(bytes.as_slice()); + let mut entries = archive + .entries() + .map_err(|_| failure("Invalid probe archive."))?; + let mut entry = entries + .next() + .ok_or_else(|| failure("Empty probe archive."))? + .map_err(|_| failure("Invalid probe archive."))?; + if entry + .path() + .map_err(|_| failure("Invalid probe path."))? + .as_ref() + != Path::new("status") + || !entry.header().entry_type().is_file() + || entry.size() > 256 + { + return Err(failure("Unexpected probe archive entry.")); + } + let mut raw = String::new(); + entry + .read_to_string(&mut raw) + .map_err(|_| failure("Invalid probe status."))?; + drop(entry); + if entries.next().is_some() { + return Err(failure("Multiple probe archive entries.")); + } + verify()?; + Ok(Some(raw)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/runtime-core/src/provider/engine/inspection.rs b/packages/runtime-core/src/provider/engine/inspection.rs new file mode 100644 index 000000000..e12c085b1 --- /dev/null +++ b/packages/runtime-core/src/provider/engine/inspection.rs @@ -0,0 +1,51 @@ +//! Read capability shared by locked inspection and lease-free status. It cannot +//! dispatch effects, execute guest programs or grant cleanup authority. +use super::*; + +pub(in crate::provider) trait Inspection { + fn incarnation(&self) -> &str; + fn boot_id(&self) -> Result<&str, CandidateError>; + #[cfg(any(target_os = "macos", test))] + fn verify(&self) -> Result<(), CandidateError>; + fn get(&self, path: &str) -> Result; + fn probe_status(&self, id: &str) -> Result, CandidateError>; +} + +impl Inspection for Engine<'_> { + fn incarnation(&self) -> &str { + self.guest.incarnation() + } + fn boot_id(&self) -> Result<&str, CandidateError> { + Ok(self.guest.boot_id()) + } + #[cfg(any(target_os = "macos", test))] + fn verify(&self) -> Result<(), CandidateError> { + self.guest.verify() + } + fn get(&self, path: &str) -> Result { + self.request(Method::GET, path, None) + } + fn probe_status(&self, id: &str) -> Result, CandidateError> { + self.probe_status(id) + } +} + +#[cfg(target_os = "macos")] +impl Inspection for Observer<'_> { + fn incarnation(&self) -> &str { + self.guest.incarnation() + } + fn boot_id(&self) -> Result<&str, CandidateError> { + self.guest.boot_id() + } + #[cfg(any(target_os = "macos", test))] + fn verify(&self) -> Result<(), CandidateError> { + self.guest.verify() + } + fn get(&self, path: &str) -> Result { + self.get(path) + } + fn probe_status(&self, id: &str) -> Result, CandidateError> { + read_probe_status(&self.transport, id, || self.guest.verify()) + } +} diff --git a/packages/runtime-core/src/provider/graph/foreground.rs b/packages/runtime-core/src/provider/graph/foreground.rs index 24f092d97..ec97a3e44 100644 --- a/packages/runtime-core/src/provider/graph/foreground.rs +++ b/packages/runtime-core/src/provider/graph/foreground.rs @@ -538,22 +538,20 @@ fn serve_input<'a>( json!({"ok":false,"run":receipt.run,"code":code,"foreground_alive":true,"recovery_required":true}), ); } - // inspect releases its Engine lease before the fresh runtime check. - let publishers_live = publishers.observe_live(candidate)?; - if publishers.has_pending_reap() { - // Host child observations do not need a guest lease. Keep - // pending ports visible even when guest inspection is busy; - // this response makes no readiness or guest-health claim. + let publisher_error = publishers.observe_readonly(candidate)?; + let publishers_live = publisher_error.is_none(); + if let Some(code) = publisher_error { return Ok( - json!({"ok":true,"run":receipt.run,"phase":receipt.phase,"plan":receipt.plan_id,"generation":redelivery::generation(&receipt)?,"foreground_alive":true,"runtime_verified":false,"runtime_error":"publisher_exit_pending","publisher_ports":publishers.port_status()}), + json!({"ok":true,"run":receipt.run,"phase":receipt.phase,"plan":receipt.plan_id,"generation":redelivery::generation(&receipt)?,"foreground_alive":true,"runtime_verified":false,"runtime_error":code,"publisher_ports":publishers.port_status()}), ); } - let mut snapshot = super::inspect(candidate, &receipt.run)?; - let engine = Engine::connect(candidate)?; - let (current, root) = super::load(candidate, &engine, &receipt.run)?; - snapshot.receipt = current; - let checked = - startup::Driver::verify(&mut runtime, &engine, &mut snapshot.receipt, &root); + let observer = crate::provider::engine::Observer::connect(candidate)?; + let root = super::directory(candidate, &receipt.run)?; + let (snapshot, checked) = super::status::stable(&root, || { + let snapshot = super::inspect_using(candidate, &observer, &receipt.run)?; + let checked = runtime.check_readonly(&observer, &snapshot.receipt, &root); + Ok((snapshot, checked)) + })?; Ok::<_, CandidateError>( json!({"ok":true,"run":snapshot.receipt.run,"phase":snapshot.receipt.phase,"plan":snapshot.receipt.plan_id,"generation":redelivery::generation(&snapshot.receipt)?,"foreground_alive":true,"runtime_verified":publishers_live && checked.is_ok(),"runtime_error":if !publishers_live {Some("publisher_exited")} else {checked.err().map(|error|error.code)},"publisher_ports":publishers.port_status()}), ) diff --git a/packages/runtime-core/src/provider/graph/foreground/publishers.rs b/packages/runtime-core/src/provider/graph/foreground/publishers.rs index e7539d006..db158c5e8 100644 --- a/packages/runtime-core/src/provider/graph/foreground/publishers.rs +++ b/packages/runtime-core/src/provider/graph/foreground/publishers.rs @@ -345,6 +345,24 @@ impl Publishers { } Ok(()) } + /// Status sees an exit without consuming the waitable child or committing + /// its exit proof. The owner's child-event path retains responsibility for reaping. + pub(super) fn observe_readonly( + &self, + candidate: &Candidate, + ) -> Result, CandidateError> { + for owned in &self.children { + if owned.exited.is_some() || owned.exit_proof.is_some() { + return Ok(Some("publisher_exited")); + } + if owned.exit_pending || publication::child_exit_pending(&owned.child)? { + return Ok(Some("publisher_exit_pending")); + } + published(candidate, owned)?; + } + Ok(None) + } + pub(super) fn observe_live(&mut self, candidate: &Candidate) -> Result { let exited = self.reap_exited(candidate)?; for owned in &mut self.children { @@ -689,6 +707,45 @@ fn matches_publication( #[cfg(test)] mod tests { + #[test] + fn status_distinguishes_waitable_and_reaped_publishers_without_reaping() { + let fixture = graph::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let child = Command::new("sh").args(["-c", "exit 7"]).spawn().unwrap(); + let mut publishers = Publishers::default(); + publishers.children.push(Owned { + child, + run: "a".repeat(32), + reservation: "b".repeat(32), + service: "web".into(), + hostnames: Vec::new(), + port: None, + target: None, + exited: None, + exit_proof: None, + exit_pending: false, + }); + let deadline = Instant::now() + Duration::from_secs(5); + while !publication::child_exit_pending(&publishers.children[0].child).unwrap() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + } + let _lease = + crate::provider::state::Lock::acquire(&candidate.state_root.join("run/smolvm")) + .unwrap(); + assert_eq!( + publishers.observe_readonly(&candidate).unwrap(), + Some("publisher_exit_pending") + ); + assert!(publication::child_exit_pending(&publishers.children[0].child).unwrap()); + let status = publishers.children[0].child.wait().unwrap(); + publishers.children[0].exited = Some(status); + assert_eq!( + publishers.observe_readonly(&candidate).unwrap(), + Some("publisher_exited") + ); + } + use super::*; #[test] fn publication_must_match_every_owned_identity_field() { diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 0786c8464..9fb915a8d 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -56,6 +56,24 @@ pub(super) fn cleanup_with_relay_expected( remove_data: bool, endpoint: &crate::provider::relay_owner::publication::PinnedEndpoint, expected_generation: Option<(&str, std::time::Instant)>, +) -> Result { + cleanup_with_relay_captured( + candidate, + run, + remove_data, + endpoint, + expected_generation, + None, + ) +} + +pub(super) fn cleanup_with_relay_captured( + candidate: &Candidate, + run: &str, + remove_data: bool, + endpoint: &crate::provider::relay_owner::publication::PinnedEndpoint, + expected_generation: Option<(&str, std::time::Instant)>, + failure: Option, ) -> Result { use crate::provider::relay_owner::lifecycle_intent::Coordinator; let engine = Engine::connect_cleanup_wait(candidate)?; @@ -66,6 +84,11 @@ pub(super) fn cleanup_with_relay_expected( super::foreground::redelivery::check_generation(&receipt, expected)?; super::check_environment_deadline(deadline)?; } + // Fold pre-stop evidence into cleanup's required journal commit. A separate + // diagnostic write must not leave state.pending and prevent the owned stop. + if failure.is_some_and(|failure| !startup_failure::capture_for_cleanup(&mut receipt, failure)) { + startup_failure::unrecorded(); + } cleanup_preflight(&engine, &receipt, &root, remove_data)?; if receipt .relay_startup diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index 9aa605555..c2430f089 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -101,7 +101,7 @@ pub(super) fn removal_selection( fn refused() -> CandidateError { error( "graph_live_owner_recovery", - "Same-boot recovery requires an unchanged fully ready graph, confirmed dead foreground/relay owner, held publication locks, and exact cleanup inventories; evidence retained.", + "Same-boot recovery requires an unchanged graph with settled startup bindings, confirmed dead foreground/relay owner, held publication locks, and exact cleanup inventories; evidence retained.", ) } fn digest(value: &impl Serialize) -> Result { @@ -134,16 +134,27 @@ fn no_pending(root: &Path, receipt: &Receipt) -> Result<(), CandidateError> { startup::require_dependency_rebind_complete(root, receipt)?; dead_owner_cleanup::require_historical_recovery(root, receipt) } -fn ready(receipt: &Receipt) -> Result<(), CandidateError> { - ready_startup(receipt)?; +/// Advisory selection only. Recovery repeats the settled binding and journal +/// checks under the provider lease and exact dead-publication witnesses. +pub(super) fn failed_start_recoverable(root: &Path, receipt: &Receipt) -> bool { + receipt.phase == "failed-retained" + && recoverable_settled(receipt).is_ok() + && no_pending(root, receipt).is_ok() +} + +fn recoverable_settled(receipt: &Receipt) -> Result<(), CandidateError> { + settled_startup(receipt)?; if receipt.relay_cleanup.is_some() { return Err(refused()); } Ok(()) } -fn ready_startup(receipt: &Receipt) -> Result<(), CandidateError> { +// This module only admits exact-dead-owner recovery. Its callers validate new +// recovery intents, resume those same intents, and archive completed prior +// recoveries; this predicate is never a readiness check for up or restore. +fn settled_startup(receipt: &Receipt) -> Result<(), CandidateError> { let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; - if receipt.phase != "ready-observed" + if !["ready-observed", "failed-retained"].contains(&receipt.phase.as_str()) || !startup.valid(receipt) || startup.services.values().any(|s| { !matches!( @@ -175,10 +186,10 @@ fn validate( boot: &str, ) -> Result<(), CandidateError> { if let Some(enrollment) = &intent.enrollment { - ready_startup(&intent.original)?; + settled_startup(&intent.original)?; enrollment.validate(&intent.original)?; } else { - ready(&intent.original)?; + recoverable_settled(&intent.original)?; } intent.relay.validate()?; if intent.version @@ -261,7 +272,7 @@ fn archive_prior(root: &Path, current: &Receipt, expected: &str) -> Result<(), C if prior.original_sha256 == expected { return Ok(()); } - ready_startup(current)?; + settled_startup(current)?; if digest(current)? != expected || exists(&root.join("live-owner-cleanup.pending"))? { return Err(refused()); } @@ -371,7 +382,7 @@ pub fn recover_live_owner( let mut intent = if exists(&root.join(FILE))? { state::read::(&root.join(FILE))? } else { - ready_startup(&receipt)?; + settled_startup(&receipt)?; if digest(&receipt)? != expected { return Err(refused()); } @@ -792,21 +803,45 @@ mod tests { } #[test] - fn only_fully_ready_receipts_are_admitted() { + fn ready_and_failed_receipts_require_settled_startup_bindings() { + let fixture = super::super::tests::Fixture::new(); let mut value = receipt(); - ready(&value).unwrap(); - for phase in [ - "preparing", - "failed-retained", - "cleanup-intent", - "stopped-data-retained", - ] { + recoverable_settled(&value).unwrap(); + assert!(!failed_start_recoverable(&fixture.0, &value)); + value.phase = "failed-retained".into(); + recoverable_settled(&value).unwrap(); + assert!(failed_start_recoverable(&fixture.0, &value)); + fs::write(fixture.0.join("state.pending"), b"uncertain effect").unwrap(); + assert!(!failed_start_recoverable(&fixture.0, &value)); + fs::remove_file(fixture.0.join("state.pending")).unwrap(); + let mut unsettled = value.clone(); + unsettled.resources.insert( + "container:web".into(), + serde_json::from_value(json!({ + "kind":"container","key":"web","name":"owned-web","id":"7".repeat(64), + "image":format!("sha256:{}", "8".repeat(64)),"phase":"started" + })) + .unwrap(), + ); + let startup = unsettled.relay_startup.as_mut().unwrap(); + startup.control_only = false; + startup.services.insert( + "web".into(), + serde_json::from_value(json!({ + "generation":"6".repeat(32), "phase":"prepared", "started_at":null, + "bindings":{"content":{"slot":0,"port":8080,"aliases":[],"process":null}} + })) + .unwrap(), + ); + assert!(unsettled.relay_startup.as_ref().unwrap().valid(&unsettled)); + assert!(!failed_start_recoverable(&fixture.0, &unsettled)); + for phase in ["preparing", "cleanup-intent", "stopped-data-retained"] { value.phase = phase.into(); - assert!(ready(&value).is_err()); + assert!(recoverable_settled(&value).is_err()); } value = receipt(); value.relay_startup = None; - assert!(ready(&value).is_err()); + assert!(recoverable_settled(&value).is_err()); } #[test] fn enrolled_recovery_versions_require_the_unchanged_legacy_pending_shape() { @@ -818,7 +853,7 @@ mod tests { })) .unwrap(), ); - assert!(ready(&original).is_err()); + assert!(recoverable_settled(&original).is_err()); let mut selected = completed(original.clone()); selected.enrollment = Some( serde_json::from_value( diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 180f4f777..1779910f5 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1,5 +1,8 @@ //! Fresh owned graph attempts. Recovery only observes or cleans recorded resources; never replay. pub mod one_off; +#[cfg(any(target_os = "macos", test))] +pub mod status; +use super::engine::inspection::Inspection; mod service_exec; pub use service_exec::{ ServiceExecOptions, ServiceExecResult, service_exec, service_exec_generation, @@ -359,7 +362,7 @@ fn directory(candidate: &Candidate, run: &str) -> Result, + engine: &impl Inspection, receipt: &Receipt, resource: &Resource, ) -> Result, CandidateError> { @@ -373,7 +376,7 @@ fn inspect_resource( "" } ); - let value = match engine.request(Method::GET, &path, None) { + let value = match engine.get(&path) { Ok(v) => v, Err(e) if e.code == "engine_not_found" => return Ok(None), Err(e) => return Err(e), @@ -1737,6 +1740,7 @@ pub fn foreground_owner(candidate: &Candidate, run: &str) -> Result, + engine: &impl Inspection, run: &str, ) -> Result { - let (receipt, root) = load(candidate, engine, run)?; + let (receipt, root) = load_at(directory(candidate, run)?, run, engine.incarnation())?; let mut observations = BTreeMap::new(); let journal_incomplete = root.join("state.pending").exists() || root.join("state.pending").is_symlink() @@ -1997,7 +2001,7 @@ fn inspect_using( probe.config.port, &endpoints::EndpointScope { receipt: &receipt, - boot: engine.guest().boot_id(), + boot: engine.boot_id()?, service: &resource.key, }, )?, @@ -2024,7 +2028,7 @@ fn inspect_using( port.target, &endpoints::EndpointScope { receipt: &receipt, - boot: engine.guest().boot_id(), + boot: engine.boot_id()?, service: &resource.key, }, )?, diff --git a/packages/runtime-core/src/provider/graph/native/foreground.rs b/packages/runtime-core/src/provider/graph/native/foreground.rs index 4b9462592..055cc67f2 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground.rs @@ -29,6 +29,7 @@ pub(super) fn require_unpublished(candidate: &Candidate, run: &str) -> Result<() #[serde(rename_all = "kebab-case")] pub enum Action { Status, + Observe, Cleanup, Logs, Exec, @@ -80,7 +81,7 @@ impl Request { (Action::Logs, Some(logs), None, None) => !logs.valid(), (Action::Exec, None, Some(exec), None) => !exec.valid(), (Action::Run, None, None, Some(one_off)) => !one_off.valid(), - (Action::Status | Action::Cleanup, None, None, None) => false, + (Action::Status | Action::Observe | Action::Cleanup, None, None, None) => false, _ => true, } { @@ -119,6 +120,10 @@ enum Outcome { #[serde(deserialize_with = "decode_snapshot")] snapshot: Snapshot, }, + Observed { + #[serde(deserialize_with = "decode_snapshot")] + snapshot: Snapshot, + }, Cleaned { receipt: Receipt, }, @@ -201,12 +206,15 @@ impl Reply { } &logs.receipt } - Outcome::Status { snapshot } => { - if action != Action::Status - || snapshot - .observations - .keys() - .ne(snapshot.receipt.readiness.keys()) + Outcome::Status { snapshot } | Outcome::Observed { snapshot } => { + if !matches!( + (&self.result, action), + (Outcome::Status { .. }, Action::Status) + | (Outcome::Observed { .. }, Action::Observe) + ) || snapshot + .observations + .keys() + .ne(snapshot.receipt.readiness.keys()) { return Err(refused()); } @@ -361,6 +369,9 @@ pub fn serve_with_storage_tool( publication.verify()?; let result = match request.action { Action::Status => inspect(candidate, &run).map(|snapshot| Outcome::Status { snapshot }), + Action::Observe => { + runtime::status(candidate, &run).map(|snapshot| Outcome::Observed { snapshot }) + } Action::Cleanup => clean().map(|receipt| Outcome::Cleaned { receipt }), Action::Exec => { let selected = request.exec.as_ref().ok_or_else(refused)?; diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index c4baa6efd..effcb5d62 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -1373,3 +1373,26 @@ fn finite_exec_request_closes_argv_workdir_and_action_without_reusing_logs_autho .is_err() ); } + +#[test] +fn presentation_observation_never_satisfies_readiness_status() { + let fixture = Fixture::new(); + let receipt = fixture.receipt(); + let verified = status(receipt.clone()); + assert!( + verified + .validate(&receipt, Action::Observe, None, None, None) + .is_err() + ); + let mut encoded = serde_json::to_value(verified).unwrap(); + encoded["result"]["outcome"] = json!("observed"); + let observed: Reply = serde_json::from_value(encoded).unwrap(); + observed + .validate(&receipt, Action::Observe, None, None, None) + .unwrap(); + assert!( + observed + .validate(&receipt, Action::Status, None, None, None) + .is_err() + ); +} diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index 47db7c101..99781b161 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -23,6 +23,16 @@ pub(super) use run::execute as service_run; #[cfg(any(target_os = "macos", test))] type OneOffCompletion = (String, i32, Vec, Vec, bool); +/// Status has no mutation, guest-program or storage-witness capability. +trait ReadBackend { + fn get(&self, path: &str) -> Result; +} +impl ReadBackend for B { + fn get(&self, path: &str) -> Result { + self.request(Method::GET, path, None) + } +} + trait Backend { #[cfg(any(target_os = "macos", test))] fn exec( @@ -491,25 +501,21 @@ pub(super) fn ownership( Ok(()) } -fn inspected( +fn inspected( backend: &B, receipt: &Receipt, resource: &Resource, ) -> Result, CandidateError> { - let value = match backend.request( - Method::GET, - &format!( - "/v1.53/{}/{}{}", - resource.kind.collection(), - resource.id.as_deref().unwrap_or(&resource.name), - if resource.kind == Kind::Container { - "/json" - } else { - "" - } - ), - None, - ) { + let value = match backend.get(&format!( + "/v1.53/{}/{}{}", + resource.kind.collection(), + resource.id.as_deref().unwrap_or(&resource.name), + if resource.kind == Kind::Container { + "/json" + } else { + "" + } + )) { Ok(value) => value, Err(error) if error.code == "engine_not_found" => return Ok(None), Err(error) => return Err(error), @@ -647,7 +653,7 @@ fn verify_data_mounts( // A network receipt grants no authority over an unknown endpoint. Recheck all // members before container effects and require a running container's exact endpoint. -fn project_network( +fn project_network( backend: &B, receipt: &Receipt, logical: &str, @@ -741,7 +747,7 @@ fn network_keys(receipt: &Receipt) -> Vec { |topology| topology.networks.keys().cloned().collect(), ) } -fn project_networks( +fn project_networks( backend: &B, receipt: &Receipt, container: Option<&Value>, @@ -1345,12 +1351,25 @@ fn snapshot_until( verify_source()?; let data_deadline = deadline.unwrap_or_else(|| Instant::now() + Duration::from_secs(40)); backend.verify_data(&receipt, data_deadline, &|| Ok(()))?; + let observations = service_observations(backend, &receipt)?; + verify_source()?; + backend.verify_data(&receipt, data_deadline, &|| Ok(()))?; + Ok(Snapshot { + receipt, + observations, + }) +} + +fn service_observations( + backend: &impl ReadBackend, + receipt: &Receipt, +) -> Result>, CandidateError> { let mut observations = BTreeMap::new(); if receipt.phase == Phase::Removed { - for logical in network_keys(&receipt) { + for logical in network_keys(receipt) { if inspected( backend, - &receipt, + receipt, &receipt.resources[&format!("network:{logical}")], )? .is_some() @@ -1359,28 +1378,52 @@ fn snapshot_until( } } } else { - project_networks(backend, &receipt, None)?; + project_networks(backend, receipt, None)?; } for resource in receipt .resources .values() .filter(|r| r.kind == Kind::Container) { - let observed = inspected(backend, &receipt, resource)?; + let observed = inspected(backend, receipt, resource)?; if let Some(value) = &observed { - verify_attachment(&receipt, &resource.key, value)?; - project_networks(backend, &receipt, Some(value))?; + verify_attachment(receipt, &resource.key, value)?; + project_networks(backend, receipt, Some(value))?; } observations.insert( resource.key.clone(), observed.as_ref().map(observation).transpose()?, ); } - verify_source()?; - backend.verify_data(&receipt, data_deadline, &|| Ok(()))?; - Ok(Snapshot { - receipt, - observations, + Ok(observations) +} + +/// Point-in-time service status. Source and storage continuity remain checked +/// by locked inspection and mutation, never inferred from service health here. +#[cfg(target_os = "macos")] +pub(super) fn status(candidate: &Candidate, run: &str) -> Result { + use crate::provider::engine::inspection::Inspection; + struct StatusBackend<'a>(crate::provider::engine::Observer<'a>); + impl ReadBackend for StatusBackend<'_> { + fn get(&self, path: &str) -> Result { + self.0.get(path) + } + } + let backend = StatusBackend(crate::provider::engine::Observer::connect(candidate)?); + let root = journal::directory(candidate, run)?; + super::super::status::stable(&root, || { + let (receipt, _) = journal::load( + candidate, + run, + backend.0.incarnation(), + backend.0.boot_id()?, + )?; + let observations = service_observations(&backend, &receipt)?; + backend.0.verify()?; + Ok(Snapshot { + receipt, + observations, + }) }) } diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index fd8033c48..72c687762 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -3139,3 +3139,44 @@ fn finite_native_logs_share_one_deadline_with_all_source_and_data_checks() { ); assert_eq!(session.backend.state.borrow().log_reads.len(), 2); } + +#[test] +fn project_start_waits_for_a_legacy_status_holder_then_runs_without_replaying_admission() { + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + let pool = fixture.candidate.state_root.join("run/smolvm"); + let reader = state::Lock::acquire(&pool).unwrap(); + fs::write(pool.join("operation.lock"), b"v1 123 inspection\n").unwrap(); + assert!(matches!(state::Lock::acquire(&pool), Err(e) if e.code == "provider_busy")); + let release = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(150)); + drop(reader); + }); + let _start = crate::provider::lifecycle::startup_lease(&pool, Duration::from_secs(2)).unwrap(); + release.join().unwrap(); + execution::run(&graph, &mut session, RUN_BUDGET).unwrap(); + assert_eq!(session.receipt.phase, Phase::ReadyObserved); + assert_eq!(session.backend.state.borrow().containers.len(), 1); + // The new lease is still exclusive: another mutation is never admitted. + assert!(matches!(state::Lock::acquire(&pool), Err(e) if e.code == "provider_busy")); +} + +#[test] +fn lasting_status_holder_is_named_at_the_acquisition_deadline_without_starting_services() { + let fixture = Fixture::new(basic()); + let (_, session) = fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + let pool = fixture.candidate.state_root.join("run/smolvm"); + let _reader = state::Lock::acquire(&pool).unwrap(); + fs::write(pool.join("operation.lock"), b"v1 123 inspection\n").unwrap(); + let started = Instant::now(); + let error = crate::provider::lifecycle::startup_lease(&pool, Duration::from_millis(75)) + .err() + .unwrap(); + assert_eq!(error.code, "provider_busy"); + assert!(error.message.contains("holder: inspection")); + assert!(error.message_fixed); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(session.backend.state.borrow().effects.is_empty()); + assert!(session.backend.state.borrow().containers.is_empty()); +} diff --git a/packages/runtime-core/src/provider/graph/probes.rs b/packages/runtime-core/src/provider/graph/probes.rs index 65eb0da7d..436789525 100644 --- a/packages/runtime-core/src/provider/graph/probes.rs +++ b/packages/runtime-core/src/provider/graph/probes.rs @@ -177,7 +177,7 @@ fn attach(config: &mut Value, p: &Probe) { mounts.push(json!({"Type":"bind","Source":format!("{}/state",path(p)),"Target":"/run/hack-http-probe-state","ReadOnly":false})); } fn exec( - engine: &Engine<'_>, + engine: &impl super::Inspection, receipt: &Receipt, name: &str, ) -> Result, CandidateError> { @@ -185,7 +185,7 @@ fn exec( let Some(id) = &p.exec_id else { return Ok(None); }; - let value = match engine.request(Method::GET, &format!("/v1.53/exec/{id}/json"), None) { + let value = match engine.get(&format!("/v1.53/exec/{id}/json")) { Ok(v) => v, Err(e) if e.code == "engine_not_found" => return Ok(None), Err(e) => return Err(e), @@ -210,7 +210,7 @@ fn exec( Ok(Some(value)) } pub(super) fn observe( - engine: &Engine<'_>, + engine: &impl super::Inspection, receipt: &Receipt, name: &str, value: &Value, diff --git a/packages/runtime-core/src/provider/graph/startup/runtime.rs b/packages/runtime-core/src/provider/graph/startup/runtime.rs index c6bc69c09..e51de68c5 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime.rs @@ -182,6 +182,35 @@ fn verify_exited_listener( }; Err(stage_refused(code)) } + +// One classifier for live status and effectful verification. Only the caller's +// callback may retain evidence; a presentation observation cannot write it. +fn classify_exited_listener( + receipt: &mut Receipt, + name: &str, + observed: &Value, + observation: Observation, + mut record: impl FnMut( + &mut Receipt, + &str, + Observation, + Option, + ) -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + if observation.failed() { + record(receipt, name, observation, None)?; + return Err(application_failed(name, observation)); + } + if let Err(error) = verify_exited_listener(receipt.readiness.get(name), observed) { + if error.code == "graph_startup_listener_unexpected_exit" + && observation == (Observation::Exited { code: 0 }) + { + return record(receipt, name, observation, Some(error)); + } + return Err(error); + } + Ok(()) +} fn verify_endpoint_generations( dependencies: &BTreeMap<(String, String), Dependency>, startup: &Startup, @@ -720,6 +749,29 @@ impl HostRelayRuntime { } let engine = Engine::connect_cleanup_wait(candidate)?; let (before, root) = crate::provider::graph::load(candidate, &engine, &run)?; + // Capture before stopping any listener. The next required cleanup + // journal commit retains the evidence; there is no advisory disk write + // that could strand state.pending and prevent stopping. + // Keep the persisted generation unchanged for rebind archival. + let mut captured = before.clone(); + let classified = if captured.startup_failure.is_some() { + Ok(()) + } else { + self.verify_listeners( + &engine, + &mut captured, + None, + |receipt, name, observation, failure| { + if receipt.startup_failure.is_none() { + startup_failure::record(receipt, name, observation)?; + } + failure.map_or(Ok(()), Err) + }, + ) + }; + if classified.is_err() && captured.startup_failure.is_none() { + startup_failure::unrecorded(); + } let boot = engine.guest().boot_id().to_owned(); for name in self.children.keys().cloned().collect::>() { let child = self.children.get_mut(&name).ok_or_else(refused)?; @@ -729,8 +781,14 @@ impl HostRelayRuntime { self.children.remove(&name); } drop(engine); - let receipt = - host_relay::cleanup_with_relay(candidate, &run, remove_data, &self.endpoint())?; + let receipt = host_relay::cleanup_with_relay_captured( + candidate, + &run, + remove_data, + &self.endpoint(), + None, + captured.startup_failure, + )?; self.children.clear(); let cleanup_engine = Engine::connect_cleanup_wait(candidate)?; startup::verify_cleanup(&cleanup_engine, &receipt)?; @@ -747,53 +805,32 @@ impl HostRelayRuntime { } Ok(()) } - fn check(&self, engine: &Engine<'_>, receipt: &Receipt) -> Result<(), CandidateError> { + fn check(&self, engine: &impl Inspection, receipt: &Receipt) -> Result<(), CandidateError> { self.managed.verify_alive()?; - let context = host_relay::context(engine.guest().incarnation(), engine.guest().boot_id())?; + let context = host_relay::context(engine.incarnation(), engine.boot_id()?)?; if context.runtime != self.context.runtime || context.boot != self.context.boot - || receipt.owner != engine.guest().incarnation() + || receipt.owner != engine.incarnation() || !matches_selected_run(self.selected_run.as_deref(), &receipt.run) || self.run.as_deref() != Some(&receipt.run) { return Err(refused()); } - engine.guest().verify() + engine.verify() } -} -impl Driver for HostRelayRuntime { - fn admission_started(&self) -> bool { - self.run.is_some() - } - - fn process_services(&self) -> std::collections::BTreeSet { - self.dependencies - .keys() - .map(|(service, _)| service.clone()) - .collect() - } - - fn check_cancelled(&self) -> Result<(), CandidateError> { - if self.startup_cancelled.is_some_and(|check| check()) { - return Err(error( - "graph_cancelled", - "Foreground startup was cancelled; owned cleanup is required.", - )); - } - Ok(()) - } - - fn validate_inputs( - &self, - inputs: &project::inputs::ExecutionInputs, - ) -> Result<(), CandidateError> { - validate_routes(self.dependencies.values(), inputs) - } - fn verify( + /// The status caller supplies no persistence callback. Effects retain the + /// original failure-recording boundary under their exclusive Engine lease. + fn verify_with( &mut self, - engine: &Engine<'_>, + engine: &impl Inspection, receipt: &mut Receipt, root: &Path, + record: impl FnMut( + &mut Receipt, + &str, + Observation, + Option, + ) -> Result<(), CandidateError>, ) -> Result<(), CandidateError> { self.check(engine, receipt)?; if matches!( @@ -824,17 +861,27 @@ impl Driver for HostRelayRuntime { .ok_or_else(refused)?; let observed = inspect_resource(engine, receipt, resource)?.ok_or_else(refused)?; - completed_generation( - receipt, - engine.guest().boot_id(), - resource, - &observed, - service, - )?; + completed_generation(receipt, engine.boot_id()?, resource, &observed, service)?; } } } - let cancelled = self.startup_cancelled; + self.verify_listeners(engine, receipt, self.startup_cancelled, record) + } + + // Called before cleanup stops any child. Cleanup passes no cancellation + // callback: cancellation requests stopping, not loss of exit evidence. + fn verify_listeners( + &mut self, + engine: &impl Inspection, + receipt: &mut Receipt, + cancelled: Option bool>, + mut record: impl FnMut( + &mut Receipt, + &str, + Observation, + Option, + ) -> Result<(), CandidateError>, + ) -> Result<(), CandidateError> { for ((name, _), child) in &mut self.children { if child.poll_exit()?.is_some() { let resource = receipt @@ -857,36 +904,77 @@ impl Driver for HostRelayRuntime { }, )?; let observation = probes::observe(engine, receipt, name, &observed)?; - if observation.failed() { - // The relay often exits with its application. Preserve the owned - // service's value-free failure before automatic cleanup removes it. - startup_failure::record(receipt, name, observation)?; - state::write(&root.join("state.json"), receipt)?; - // Name the failed service, as a graph without dependency - // bindings does, rather than a dependency validation failure. - return Err(application_failed(name, observation)); - } - if let Err(error) = verify_exited_listener(receipt.readiness.get(name), &observed) { - if error.code == "graph_startup_listener_unexpected_exit" - && observation == (Observation::Exited { code: 0 }) - { - // A successful process exit can still lose a required - // listener. Keep its identity before cleanup removes it. - return Err(startup_failure::preserve_listener_error( - root, - receipt, - name, - observation, - error, - )); - } - return Err(error); - } + classify_exited_listener(receipt, name, &observed, observation, &mut record)?; } } Ok(()) } + pub(in crate::provider::graph) fn check_readonly( + &mut self, + observer: &impl Inspection, + receipt: &Receipt, + root: &Path, + ) -> Result<(), CandidateError> { + self.verify_with(observer, &mut receipt.clone(), root, |_, _, _, failure| { + failure.map_or(Ok(()), Err) + }) + } +} +impl Driver for HostRelayRuntime { + fn admission_started(&self) -> bool { + self.run.is_some() + } + + fn process_services(&self) -> std::collections::BTreeSet { + self.dependencies + .keys() + .map(|(service, _)| service.clone()) + .collect() + } + + fn check_cancelled(&self) -> Result<(), CandidateError> { + if self.startup_cancelled.is_some_and(|check| check()) { + return Err(error( + "graph_cancelled", + "Foreground startup was cancelled; owned cleanup is required.", + )); + } + Ok(()) + } + + fn validate_inputs( + &self, + inputs: &project::inputs::ExecutionInputs, + ) -> Result<(), CandidateError> { + validate_routes(self.dependencies.values(), inputs) + } + fn verify( + &mut self, + engine: &Engine<'_>, + receipt: &mut Receipt, + root: &Path, + ) -> Result<(), CandidateError> { + self.verify_with( + engine, + receipt, + root, + |receipt, name, observation, failure| { + if let Some(error) = failure { + return Err(startup_failure::preserve_listener_error( + root, + receipt, + name, + observation, + error, + )); + } + startup_failure::record(receipt, name, observation)?; + state::write(&root.join("state.json"), receipt) + }, + ) + } + fn prepare( &mut self, engine: &Engine<'_>, @@ -1215,6 +1303,59 @@ mod route_tests { use super::*; use std::os::unix::fs::MetadataExt; #[test] + fn status_and_cleanup_classify_the_same_exit_without_a_status_write() { + let fixture = super::super::super::tests::Fixture::new(); + let root = &fixture.0; + let original: Receipt = serde_json::from_value(json!({"version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64),"plan_id":"d".repeat(64),"phase":"ready-observed","readiness":{"web":"started"},"resources":{"container:web":{"key":"web","kind":"container","name":format!("hkg-{}-container-0", "a".repeat(32)),"id":"e".repeat(64),"phase":"started","image":format!("sha256:{}", "f".repeat(64))}}})).unwrap(); + state::write(&root.join("state.json"), &original).unwrap(); + let bytes = fs::read(root.join("state.json")).unwrap(); + for code in [0, 7] { + let observed = json!({"State":{"Running":false,"ExitCode":code,"OOMKilled":false}}); + let observation = Observation::Exited { code }; + let mut status = original.clone(); + let live_error = classify_exited_listener( + &mut status, + "web", + &observed, + observation, + |_, _, _, error| error.map_or(Ok(()), Err), + ) + .unwrap_err(); + assert!(status.startup_failure.is_none()); + assert_eq!(fs::read(root.join("state.json")).unwrap(), bytes); + let mut cleanup = original.clone(); + let captured_error = classify_exited_listener( + &mut cleanup, + "web", + &observed, + observation, + |receipt, service, observation, error| { + startup_failure::record(receipt, service, observation)?; + error.map_or(Ok(()), Err) + }, + ) + .unwrap_err(); + assert_eq!(live_error.code, captured_error.code); + assert_eq!(cleanup.startup_failure.unwrap().observation, observation); + // Cleanup capture also has no standalone persistence step that + // could strand state.pending before the required cleanup commit. + assert_eq!(fs::read(root.join("state.json")).unwrap(), bytes); + assert!(!root.join("state.pending").exists()); + } + let mut completed = original; + completed + .readiness + .insert("web".into(), Condition::Completed); + classify_exited_listener( + &mut completed, + "web", + &json!({"State":{"Running":false,"ExitCode":0,"OOMKilled":false}}), + Observation::Exited { code: 0 }, + |_, _, _, _| panic!("successful completion is not failure evidence"), + ) + .unwrap(); + } + #[test] fn dependency_free_constructor_skips_artifact_but_required_artifact_uses_fixed_refusal() { let fixture = super::super::super::tests::Fixture::new(); let absent = fixture.0.join("absent-guest-artifact"); diff --git a/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs b/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs index 326010eca..c54cfd32e 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs @@ -590,7 +590,14 @@ fn exact_owned_cleanup_archives_partial_rebind_and_unblocks_fresh_restore() { ) .unwrap(); assert!(require_complete(&root, &original).is_err()); + // A newly captured failure belongs to cleanup's receipt, never to the + // persisted pre-cleanup generation that keys this rebind archive. let mut cleaned = original.clone(); + startup_failure::record(&mut cleaned, "web", Observation::Exited { code: 7 }).unwrap(); + assert_ne!( + service_exec_generation(&original).unwrap(), + service_exec_generation(&cleaned).unwrap() + ); cleaned.phase = "stopped-data-retained".into(); assert!(archive_after_cleanup(&root, &original, &cleaned, "old-boot").is_err()); cleaned.resources.get_mut("container:web").unwrap().phase = "absent".into(); diff --git a/packages/runtime-core/src/provider/graph/startup_failure.rs b/packages/runtime-core/src/provider/graph/startup_failure.rs index e8b853990..ef3744dc0 100644 --- a/packages/runtime-core/src/provider/graph/startup_failure.rs +++ b/packages/runtime-core/src/provider/graph/startup_failure.rs @@ -56,6 +56,38 @@ pub(super) fn record( receipt.startup_failure = Some(failure); Ok(()) } +/// Fold in pre-stop evidence only for the same container, keeping the first +/// recorded failure. False is diagnostic only; it must not obstruct cleanup. +#[cfg(any(target_os = "macos", test))] +pub(super) fn capture_for_cleanup(receipt: &mut Receipt, failure: Failure) -> bool { + if receipt.startup_failure.is_some() { + return true; + } + if !failure.valid(receipt) + || receipt + .resources + .get(&format!("container:{}", failure.service)) + .and_then(|resource| resource.id.as_deref()) + != Some(failure.container.as_str()) + { + return false; + } + receipt.startup_failure = Some(failure); + true +} + +/// Diagnostics must not abort cleanup when the frontend's stderr pipe is gone. +#[cfg(target_os = "macos")] +pub(super) fn unrecorded() { + write_unrecorded(&mut std::io::stderr()); +} +#[cfg(any(target_os = "macos", test))] +fn write_unrecorded(output: &mut impl std::io::Write) { + let _ = writeln!( + output, + "startup_failure_unrecorded: listener failure evidence could not be retained before cleanup." + ); +} #[cfg(any(target_os = "macos", test))] pub(super) fn preserve_listener_error( root: &Path, @@ -87,6 +119,39 @@ mod tests { serde_json::from_value(json!({"version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64),"plan_id":"d".repeat(64),"phase":"preparing","readiness":{"redis":"started"},"resources":{"container:redis":{"key":"redis","kind":"container","name":format!("hkg-{}-container-0", "a".repeat(32)),"id":"e".repeat(64),"phase":"started","image":format!("sha256:{}", "f".repeat(64))}}})).unwrap() } #[test] + fn lost_diagnostic_pipe_does_not_abort_cleanup_evidence_handling() { + use std::os::unix::net::UnixStream; + let (mut writer, reader) = UnixStream::pair().unwrap(); + drop(reader); + write_unrecorded(&mut writer); + let mut cleaned = receipt(); + record(&mut cleaned, "redis", Observation::Exited { code: 7 }).unwrap(); + cleaned.phase = "stopped-data-retained".into(); + cleaned.resources.get_mut("container:redis").unwrap().id = None; + let root = super::super::tests::Fixture::new(); + crate::provider::state::write(&root.0.join("state.json"), &cleaned).unwrap(); + assert!(super::super::load_at(root.0.clone(), &cleaned.run, &cleaned.owner).is_ok()); + } + #[test] + fn cleanup_capture_keeps_first_failure_and_refuses_replaced_container_without_writing() { + let mut original = receipt(); + record(&mut original, "redis", Observation::Exited { code: 7 }).unwrap(); + let first = original.startup_failure.clone().unwrap(); + let mut later = first.clone(); + later.observation = Observation::Exited { code: 8 }; + assert!(capture_for_cleanup(&mut original, later)); + assert_eq!(original.startup_failure, Some(first.clone())); + let mut replaced = receipt(); + replaced.resources.get_mut("container:redis").unwrap().id = Some("9".repeat(64)); + let bytes = serde_json::to_vec(&replaced).unwrap(); + assert!(!capture_for_cleanup(&mut replaced, first.clone())); + assert_eq!(serde_json::to_vec(&replaced).unwrap(), bytes); + let mut current = receipt(); + assert!(capture_for_cleanup(&mut current, first.clone())); + assert!(capture_for_cleanup(&mut current, first.clone())); + assert_eq!(current.startup_failure, Some(first)); + } + #[test] fn failure_survives_cleanup_and_legacy_receipts_remain_readable() { let mut receipt = receipt(); assert!(receipt.startup_failure.is_none()); diff --git a/packages/runtime-core/src/provider/graph/status.rs b/packages/runtime-core/src/provider/graph/status.rs new file mode 100644 index 000000000..a4ad71529 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/status.rs @@ -0,0 +1,195 @@ +//! Lease-free presentation. A stable journal fences a point-in-time observation; +//! it never authorizes an effect or substitutes for locked recovery inspection. +use super::*; +use std::os::unix::fs::MetadataExt; + +#[derive(PartialEq, Eq)] +struct Stamp { + dev: u64, + ino: u64, + len: u64, + modified: (i64, i64), + changed: (i64, i64), +} + +fn stamp(path: &std::path::Path) -> Result, CandidateError> { + match fs::symlink_metadata(path) { + Ok(value) => Ok(Some(Stamp { + dev: value.dev(), + ino: value.ino(), + len: value.len(), + modified: (value.mtime(), value.mtime_nsec()), + changed: (value.ctime(), value.ctime_nsec()), + })), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(state::io(error)), + } +} + +fn changed() -> CandidateError { + error( + "graph_status_changed", + "The graph changed during status observation; no consistent snapshot was observed.", + ) +} + +/// Atomic journal replacement and pending-file publication change the directory +/// stamp, including a pending file created and removed during the observation. +/// Also fence in-place writes to each existing direct journal member. No lock, +/// journal creation, repair or retry of an effect participates in this read. +pub(in crate::provider::graph) fn stable( + root: &std::path::Path, + read: impl FnOnce() -> Result, +) -> Result { + let capture = || -> Result<_, CandidateError> { + let directory = stamp(root)?; + let mut members = BTreeMap::new(); + if directory.is_some() { + for entry in fs::read_dir(root).map_err(state::io)? { + let entry = entry.map_err(state::io)?; + if members.len() >= 1024 { + return Err(changed()); + } + members.insert(entry.file_name(), stamp(&entry.path())?); + } + } + Ok((directory, members)) + }; + let before = capture()?; + let result = read(); + if capture()? != before { + return Err(changed()); + } + result +} + +#[cfg(target_os = "macos")] +pub fn inspect(candidate: &Candidate, run: &str) -> Result { + let observer = super::super::engine::Observer::connect(candidate)?; + inspect_with(candidate, run, &observer) +} + +fn inspect_with( + candidate: &Candidate, + run: &str, + observer: &impl Inspection, +) -> Result { + stable(&directory(candidate, run)?, || { + observer.verify()?; + let snapshot = inspect_using(candidate, observer, run)?; + observer.verify()?; + if snapshot.journal_incomplete { + return Err(changed()); + } + Ok(snapshot) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + struct Read<'a> { + receipt: &'a Receipt, + during: &'a dyn Fn(), + } + impl Inspection for Read<'_> { + fn incarnation(&self) -> &str { + &self.receipt.owner + } + fn boot_id(&self) -> Result<&str, CandidateError> { + Ok("fixture-boot") + } + fn verify(&self) -> Result<(), CandidateError> { + Ok(()) + } + fn get(&self, _: &str) -> Result { + (self.during)(); + let resource = &self.receipt.resources["container:web"]; + Ok( + json!({"Id":resource.id,"Name":resource.name,"Image":resource.image, + "Config":{"Labels":expected_labels(self.receipt, resource)}, + "State":{"Status":"running","Running":true}}), + ) + } + fn probe_status(&self, _: &str) -> Result, CandidateError> { + panic!("fixture has no probe") + } + } + + fn fixture() -> (super::super::tests::Fixture, Candidate, Receipt, PathBuf) { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let receipt: Receipt = serde_json::from_value(json!({"version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64),"plan_id":"d".repeat(64),"phase":"ready-observed","readiness":{"web":"started"},"resources":{"container:web":{"key":"web","kind":"container","name":format!("hkg-{}-container-0", "a".repeat(32)),"id":"e".repeat(64),"phase":"started","image":format!("sha256:{}", "f".repeat(64))}}})).unwrap(); + let root = directory(&candidate, &receipt.run).unwrap(); + state::private_directory(&root).unwrap(); + state::write(&root.join("state.json"), &receipt).unwrap(); + (fixture, candidate, receipt, root) + } + + #[test] + fn status_and_mutation_overlap_without_taking_each_others_lease() { + let (_fixture, candidate, receipt, root) = fixture(); + let pool = candidate.state_root.join("run/smolvm"); + let before = fs::read(root.join("state.json")).unwrap(); + { + let held = state::Lock::acquire(&pool).unwrap(); + let calls = Cell::new(0); + let read = Read { + receipt: &receipt, + during: &|| { + calls.set(calls.get() + 1); + // A second mutation still refuses while the first holds the lease. + assert!( + matches!(state::Lock::acquire(&pool), Err(e) if e.code == "provider_busy") + ); + }, + }; + let snapshot = inspect_with(&candidate, &receipt.run, &read).unwrap(); + assert_eq!(snapshot.observations["container:web"]["state"], "running"); + assert_eq!(calls.get(), 1); + drop(held); + // A mutation arriving in the middle of a status GET can acquire immediately. + let read = Read { + receipt: &receipt, + during: &|| { + let _start = state::Lock::acquire(&pool).unwrap(); + }, + }; + inspect_with(&candidate, &receipt.run, &read).unwrap(); + } + assert_eq!(fs::read(root.join("state.json")).unwrap(), before); + } + + #[test] + fn journal_change_during_status_never_returns_a_mixed_snapshot() { + for mode in ["replace", "pending", "rewrite"] { + let (_fixture, candidate, receipt, root) = fixture(); + let read = Read { + receipt: &receipt, + during: &|| match mode { + "replace" => { + let mut changed = receipt.clone(); + changed.phase = "cleanup-intent".into(); + state::write(&root.join("state.json"), &changed).unwrap(); + } + "pending" => { + fs::write(root.join("state.pending"), b"intent").unwrap(); + fs::remove_file(root.join("state.pending")).unwrap(); + } + _ => { + fs::write(root.join("state.json"), b"interrupted").unwrap(); + } + }, + }; + assert_eq!( + inspect_with(&candidate, &receipt.run, &read) + .err() + .unwrap() + .code, + "graph_status_changed" + ); + } + } +} diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 073953d10..2ac3f3ff5 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -266,16 +266,20 @@ fn operation_lease( deadline: Option, verify: impl FnOnce() -> Result, ) -> Result<(state::Lock, T), CandidateError> { + let mut holder = None; let lock = loop { if deadline.is_some_and(|limit| Instant::now() >= limit) { - return Err(CandidateError::new( - "provider_busy", - "Provider lease acquisition deadline expired; no operation was admitted.", - )); + return Err(holder.unwrap_or_else(|| { + CandidateError::new( + "provider_busy", + "Provider lease acquisition deadline expired; no operation was admitted.", + ) + })); } match state::Lock::acquire(root) { Ok(lock) => break lock, Err(error) if error.code == "provider_busy" && deadline.is_some() => { + holder = Some(error); let remaining = deadline .expect("bounded acquisition") .saturating_duration_since(Instant::now()); @@ -289,14 +293,14 @@ fn operation_lease( } /// How long owned cleanup and recovery wait for a provider lease that another operation -/// holds, such as the `graph inspect` behind a polling `hack ps`. Acquisition admits nothing, +/// holds, such as a finite `graph inspect`. Acquisition admits nothing, /// and a foreground owner that gives up its cleanup exits with its publication and services /// still bound, so this waits longer than startup does. #[cfg(target_os = "macos")] pub(super) const OWNED_CLEANUP_LEASE_WAIT: Duration = Duration::from_secs(30); /// How long ordinary startup waits for a provider lease that another operation holds, such as -/// the `graph inspect` behind `hack ps`. +/// a finite `graph inspect` (presentation `ps` takes no provider lease). pub(super) const STARTUP_LEASE_WAIT: Duration = Duration::from_secs(5); /// Startup's provider lease. Everything `start_pool` does before taking it is read-only @@ -327,7 +331,7 @@ pub(super) struct ObservedGuest<'a> { } impl<'a> ObservedGuest<'a> { - #[cfg(all(target_os = "macos", feature = "native-config-plan"))] + #[cfg(target_os = "macos")] pub(super) fn boot_id(&self) -> Result<&str, CandidateError> { self.owner.guest_boot_id.as_deref().ok_or_else(|| { CandidateError::new("runtime_changed", "Observed guest boot is unavailable.") diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index 07a28b1f7..399b6f2a2 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; use std::os::fd::AsRawFd; -use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}; +use std::os::unix::fs::{DirBuilderExt, FileExt, MetadataExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; pub fn io(error: std::io::Error) -> CandidateError { @@ -45,8 +45,27 @@ pub fn check_private_directory(path: &Path) -> Result<(), CandidateError> { Ok(()) } -pub struct Lock(File); +pub struct Lock(File, bool); impl Lock { + /// Advisory diagnostics only, on the provider pool's lock inode. Never + /// interpreted as ownership or recovery authority; crashes may leave it stale. + fn describe_holder(mut self, root: &Path) -> Result { + if provider_lock(root) { + // A diagnostic must never turn an acquired stop/start lease into a + // refusal. Partial or stale bytes are ignored by the closed reader. + let label = operation_label(); + self.1 = self + .0 + .set_len(0) + .and_then(|()| { + self.0 + .write_all(format!("v1 {} {label}\n", std::process::id()).as_bytes()) + }) + .is_ok(); + } + Ok(self) + } + /// Verify that the retained cooperative lock still names this exact private inode. /// Holding an unlinked predecessor cannot serialize a replacement pathname. pub(crate) fn verify_path(&self, root: &Path) -> Result<(), CandidateError> { @@ -96,7 +115,7 @@ impl Lock { .custom_flags(libc::O_NOFOLLOW) .open(root.join("operation.lock")) .map_err(io)?; - Self::from_file(file) + Self::from_file(file)?.describe_holder(root) } /// Observe existing state without initializing a directory or lock file. @@ -104,10 +123,11 @@ impl Lock { check_private_directory(root)?; let file = OpenOptions::new() .read(true) + .write(provider_lock(root)) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) .open(root.join("operation.lock")) .map_err(io)?; - Self::from_file(file) + Self::from_file(file)?.describe_holder(root) } #[cfg(target_os = "macos")] @@ -152,22 +172,79 @@ impl Lock { } // SAFETY: file is a live owned descriptor; flock does not retain pointers. if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 { - return Err(CandidateError::new( - "provider_busy", - "Another candidate operation holds the provider lock.", - )); + return Err(CandidateError::new("provider_busy", holder_message(&file))); } - Ok(Self(file)) + Ok(Self(file, false)) } } impl Drop for Lock { fn drop(&mut self) { + if self.1 { + let _ = self.0.set_len(0); + } unsafe { libc::flock(self.0.as_raw_fd(), libc::LOCK_UN); } } } +fn provider_lock(root: &Path) -> bool { + root.ends_with("run/smolvm") +} + +fn operation_label() -> &'static str { + // Only closed command words enter diagnostics, never user arguments or paths. + for argument in std::env::args_os().skip(1) { + match argument.to_str() { + Some("cleanup" | "down" | "stop") => return "cleanup", + Some("recover" | "recover-live-owner" | "recover-cleanup") => return "recovery", + Some("run" | "serve" | "start" | "restore" | "restart") => return "start", + Some("inspect" | "status") => return "inspection", + _ => {} + } + } + "operation" +} + +fn holder_message(file: &File) -> &'static str { + let mut bytes = [0; 64]; + let kind = file.read_at(&mut bytes, 0).ok().and_then(|count| { + let text = std::str::from_utf8(&bytes[..count]).ok()?; + let mut fields = text.split_whitespace(); + if fields.next()? != "v1" { + return None; + } + fields.next()?.parse::().ok().filter(|pid| *pid > 0)?; + let kind = fields.next()?; + if fields.next().is_some() { + return None; + } + Some(kind) + }); + // Closed literal messages survive the frontend's value-free diagnostic + // boundary. The record is advisory; no process identity is inferred from it. + match kind { + Some("cleanup") => { + "Provider lease acquisition refused; recorded holder: cleanup; no operation was admitted." + } + Some("recovery") => { + "Provider lease acquisition refused; recorded holder: recovery; no operation was admitted." + } + Some("start") => { + "Provider lease acquisition refused; recorded holder: start; no operation was admitted." + } + Some("inspection") => { + "Provider lease acquisition refused; recorded holder: inspection; no operation was admitted." + } + Some("operation") => { + "Provider lease acquisition refused; recorded holder: native operation; no operation was admitted." + } + _ => { + "Another candidate operation holds the provider lock; holder identity is unavailable; no operation was admitted." + } + } +} + pub fn read(path: &Path) -> Result { read_bounded(path, 1024 * 1024) } @@ -793,6 +870,28 @@ impl Owner { #[cfg(test)] mod tests { + #[test] + fn advisory_write_failure_does_not_refuse_the_acquired_provider_lease() { + let fixture = fixture(); + let root = fixture.join("run/smolvm"); + super::private_directory(&root).unwrap(); + let path = root.join("operation.lock"); + std::fs::write(&path, b"diagnostic-canary").unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + // An actual read-only descriptor injects EBADF in the advisory write, + // while flock itself can still acquire exclusive ownership. + let lock = super::Lock::from_file(std::fs::File::open(&path).unwrap()) + .unwrap() + .describe_holder(&root) + .unwrap(); + assert!(matches!(super::Lock::acquire(&root), Err(error) if error.code == "provider_busy")); + drop(lock); + assert_eq!(std::fs::read(&path).unwrap(), b"diagnostic-canary"); + drop(super::Lock::acquire(&root).unwrap()); + std::fs::remove_dir_all(fixture).unwrap(); + } + use super::*; use std::os::unix::fs::PermissionsExt; diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index 1bee5962b..dab5b1fd4 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -955,6 +955,19 @@ const contracts: readonly ModelContract[] = [ }, ], }, + { + name: "native-status-start", + module: "StatusStart", + states: 110, + invariant: "ReaderNeverFailsStart", + action: "ReaderRefusal", + fields: [ + "failedByReader = TRUE", + "services = TRUE", + "reader = TRUE", + 'lease = "reader"', + ], + }, { name: "native-publisher-stop", module: "PublisherStop", diff --git a/src/backends/native-authored-graph-protocol.ts b/src/backends/native-authored-graph-protocol.ts index 7d0d1676d..cc6aafd34 100644 --- a/src/backends/native-authored-graph-protocol.ts +++ b/src/backends/native-authored-graph-protocol.ts @@ -988,7 +988,7 @@ export function parseNativeAuthoredReady( export function parseNativeAuthoredControl( value: unknown, expected: NativeAuthoredReceipt, - action: "status" | "cleanup" + action: "status" | "observe" | "cleanup" ): { readonly receipt: NativeAuthoredReceipt; readonly observations?: Readonly>; @@ -1019,9 +1019,9 @@ export function parseNativeAuthoredControl( return { receipt }; } if ( - action !== "status" || + (action !== "status" && action !== "observe") || !fields(result, ["outcome", "snapshot"]) || - result.outcome !== "status" + result.outcome !== (action === "observe" ? "observed" : "status") ) { return refused(); } diff --git a/src/backends/native-authored-project-observe.ts b/src/backends/native-authored-project-observe.ts index 036132a32..ce0ecb9d1 100644 --- a/src/backends/native-authored-project-observe.ts +++ b/src/backends/native-authored-project-observe.ts @@ -8,6 +8,7 @@ import { withPinnedRuntimeBinary } from "./native-authored-runtime-binary.ts"; import { invokeNativeRuntime, isNativeRuntimeNotRunning, + NativeRuntimeRequestError, type NativeRuntimeSelection, } from "./native-runtime-client.ts"; @@ -37,6 +38,7 @@ export type NativeAuthoredProjectStatus = { readonly status: | "not_started" | "pending" + | "runtime_busy" | "runtime_not_running" | "observed"; readonly run: string | null; @@ -104,7 +106,7 @@ export async function nativeAuthoredProjectPs(input: { "--run-id", receipt.review.provenance.run, "--action", - "status", + "observe", "--json", ], timeoutMs: 15_000, @@ -112,7 +114,12 @@ export async function nativeAuthoredProjectPs(input: { boundNativeStatusDrain: true, }); } catch (error: unknown) { - if (!isNativeRuntimeNotRunning(error)) { + const changed = + error instanceof NativeRuntimeRequestError && + (error.nativeCode === "graph_status_changed" || + (error.nativeCode === "native_graph_foreground" && + error.nativeCauseCode === "graph_status_changed")); + if (!(changed || isNativeRuntimeNotRunning(error))) { throw error; } // A stopped runtime is a status, not a refusal; the same saved @@ -120,14 +127,14 @@ export async function nativeAuthoredProjectPs(input: { await check(); return { backend: "native", - status: "runtime_not_running", + status: changed ? "runtime_busy" : "runtime_not_running", run: receipt.review.provenance.run, phase: null, items: [], }; } await check(); - const current = parseNativeAuthoredControl(value, receipt, "status"); + const current = parseNativeAuthoredControl(value, receipt, "observe"); const items: Item[] = Object.entries(current.receipt.readiness).map( ([service]) => { const observation = current.observations?.[service]; diff --git a/src/backends/native-project-down.ts b/src/backends/native-project-down.ts index acd2500a3..b13e39801 100644 --- a/src/backends/native-project-down.ts +++ b/src/backends/native-project-down.ts @@ -113,6 +113,7 @@ type ForegroundOwnerSelection = { /** Guest completion commit. Retirement still finishes and verifies reservation release. */ readonly deadOwnerRecoveryComplete: boolean; readonly deadOwnerRecoveryError: string | null; + readonly failedStartRecoverable: boolean; }; function ownerRecoveryRefused(): Error { @@ -136,6 +137,7 @@ async function selectForegroundOwner(opts: { if ( !isRecord(value) || value.version !== 1 || + !optionalBoolean(value.failed_start_recoverable) || value.run !== opts.run.run || value.owner !== opts.run.owner || typeof value.phase !== "string" || @@ -173,6 +175,7 @@ async function selectForegroundOwner(opts: { } return { phase: value.phase, + failedStartRecoverable: value.failed_start_recoverable === true, receiptSha256: value.receipt_sha256, foreground: value.foreground as ForegroundOwnerSelection["foreground"], liveOwnerRecovery: @@ -196,6 +199,10 @@ async function selectForegroundOwner(opts: { }; } +function optionalBoolean(value: unknown): boolean { + return value === undefined || typeof value === "boolean"; +} + /** * After selected predecessor recovery is handled, dispatch a live/absent owner * to ordinary authenticated cleanup, or a dead ready owner to same-boot @@ -205,6 +212,7 @@ async function selectForegroundOwner(opts: { async function requestOwnedStop(opts: { readonly runtime: NativeRuntimeSelection; readonly projectRoot: string; + readonly scope: NativeProjectRunScope; readonly run: NativeProjectRun; readonly invoke: typeof invokeNativeRuntime; readonly selection: ForegroundOwnerSelection | undefined; @@ -228,9 +236,19 @@ async function requestOwnedStop(opts: { error instanceof NativeRuntimeRequestError && error.nativeCode === "graph_owner_recovery" ) { - throw new ClosedCliError( - "Native down cannot reach this run's foreground owner. If the hack up or hack restart that owned it has exited, run hack down --recover to stop its services with data retained. Nothing was stopped." - ); + const observed = await selectForegroundOwner(opts); + requireRecoverableOwner(observed); + if (await resumeSelectedOwner({ ...opts, selection: observed })) { + return; + } + if (observed.foreground !== "exited") { + throw ownerRecoveryRefused(); + } + await recoverLiveOwner({ + ...opts, + expectReceipt: observed.receiptSha256, + }); + return; } throw error; } @@ -383,7 +401,8 @@ function requireRecoverableOwner(selection: ForegroundOwnerSelection): void { } if ( selection.foreground === "exited" && - selection.phase !== "ready-observed" + selection.phase !== "ready-observed" && + !(selection.phase === "failed-retained" && selection.failedStartRecoverable) ) { throw new ClosedCliError( "Native down --recover found a foreground owner that exited before its start was confirmed. Nothing was changed. Run hack ps --json to inspect this run's retained state." @@ -391,6 +410,82 @@ function requireRecoverableOwner(selection: ForegroundOwnerSelection): void { } } +/** Resume only an existing exact recovery selection. Shared by explicit + * recovery and plain down after an authenticated dead-owner refusal. */ +async function resumeSelectedOwner(opts: { + readonly runtime: NativeRuntimeSelection; + readonly scope: NativeProjectRunScope; + readonly run: NativeProjectRun; + readonly invoke: typeof invokeNativeRuntime; + readonly selection: ForegroundOwnerSelection; +}): Promise { + const { run, invoke, selection } = opts; + if (selection.deadOwnerRecoveryError) { + throw new ClosedCliError( + `Native predecessor recovery selection failed (${selection.deadOwnerRecoveryError}). ${nativeRecoveryGuidance(selection.deadOwnerRecoveryError)}` + ); + } + requireRecoverableOwner(selection); + if (selection.deadOwnerRecovery && selection.foreground !== "running") { + if (!selection.deadOwnerRecoveryComplete) { + const result = await invokeOwnerRecovery({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + invoke, + args: [ + "graph", + "recover-cleanup", + "--run-id", + run.run, + "--expect-receipt", + selection.deadOwnerRecovery, + "--json", + ], + }); + if ( + !isRecord(result) || + result.run !== run.run || + result.recovered !== true || + result.data_retained !== true + ) { + throw ownerRecoveryRefused(); + } + } + await retireNativeRecoveredPublisher({ + runtime: opts.runtime, + scope: opts.scope, + run, + invoke, + }); + return true; + } + if (selection.liveOwnerRecovery && selection.foreground !== "running") { + // Finish the interrupted recovery first; the stopped path below then retires + // host processes and runs only journaled owed hooks, never replaying a stop. + await recoverLiveOwner({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + run, + invoke, + expectReceipt: selection.liveOwnerRecovery, + }); + return true; + } + if (unadmittedRetainedOwner(selection)) { + // Compute is already stopped; only the dead attempt's publications remain. + // The stopped path below then retires host processes and owed hooks. + await retireUnadmittedOwner({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + run, + invoke, + expectReceipt: selection.receiptSha256, + }); + return true; + } + return false; +} + /** Retaining cleanup only. Success follows confirmed compute absence and the * original frontend's finalizers, including a previously recovered stop. */ @@ -452,92 +547,37 @@ export async function nativeProjectDown(opts: { }); let initial = await inspect(); verify(initial, run, false); - const selection = opts.recover - ? await selectForegroundOwner({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - run, - invoke, - }) - : undefined; - if (selection) { - if (selection.deadOwnerRecoveryError) { - throw new ClosedCliError( - `Native predecessor recovery selection failed (${selection.deadOwnerRecoveryError}). ${nativeRecoveryGuidance(selection.deadOwnerRecoveryError)}` - ); - } - requireRecoverableOwner(selection); - } - if (selection?.deadOwnerRecovery && selection.foreground !== "running") { - if (!selection.deadOwnerRecoveryComplete) { - const result = await invokeOwnerRecovery({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - invoke, - args: [ - "graph", - "recover-cleanup", - "--run-id", - run.run, - "--expect-receipt", - selection.deadOwnerRecovery, - "--json", - ], - }); - if ( - !isRecord(result) || - result.run !== run.run || - result.recovered !== true || - result.data_retained !== true - ) { - throw ownerRecoveryRefused(); - } - } - await retireNativeRecoveredPublisher({ + const finalization = opts.deferFinalization + ? undefined + : await captureFinalization({ scope: opts.scope, run }); + const frontendGone = + finalization !== undefined && + finalization.version !== 1 && + (await nativeFrontendGone(finalization)); + // A failed foreground start must remain stoppable by plain down. Selection + // grants no authority: the runtime rechecks exact death, boot and receipt. + const selection = + opts.recover || frontendGone + ? await selectForegroundOwner({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + run, + invoke, + }) + : undefined; + if ( + selection && + (await resumeSelectedOwner({ runtime: opts.runtime, scope: opts.scope, run, invoke, - }); - initial = await inspect(); - verify(initial, run, false); - } else if ( - selection?.liveOwnerRecovery && - selection.foreground !== "running" + selection, + })) ) { - // Finish the interrupted recovery first; the stopped path below then retires - // host processes and runs only journaled owed hooks, never replaying a stop. - await recoverLiveOwner({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - run, - invoke, - expectReceipt: selection.liveOwnerRecovery, - }); - initial = await inspect(); - verify(initial, run, false); - } else if (selection && unadmittedRetainedOwner(selection)) { - // Compute is already stopped; only the dead attempt's publications remain. - // The stopped path below then retires host processes and owed hooks. - await retireUnadmittedOwner({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - run, - invoke, - expectReceipt: selection.receiptSha256, - }); initial = await inspect(); verify(initial, run, false); } - const finalization = opts.deferFinalization - ? undefined - : await captureFinalization({ scope: opts.scope, run }); - // The shared identity predicate also distinguishes a reused macOS PID. A - // frontend can exit during cleanup or the wait, so recheck before recovery. - const frontendGone = - finalization !== undefined && - finalization.version !== 1 && - (await nativeFrontendGone(finalization)); const wait = async () => { if (finalization === undefined) { return; @@ -556,43 +596,36 @@ export async function nativeProjectDown(opts: { ) { throw finalizationRefused(); } - if (opts.recover === true) { - // Exact captured attempt, dead frontend, stopped graph and all frontend - // effects are rechecked under the existing finalization lock. Recovery - // remains a separate fact from that frontend completing its own cleanup. - await recoverNativeProjectFinalization({ - scope: opts.scope, - run, - token: finalization, - expectAttempt: finalization.attempt, - verifyEffects: () => - verifyNativeFrontendRecovery({ - runtime: opts.runtime, - scope: opts.scope, - run, - httpsPort: finalization.httpsPort, - httpsLease: - finalization.version === 3 - ? finalization.httpsLease - : undefined, - legacy: false, - invoke, - cleanupLifecycle: async () => { - await opts.retireHostProcesses?.(run); - }, - }), - }); - await waitNativeProjectFinalization({ - scope: opts.scope, - run, - token: finalization, - timeoutMs: 1, - }); - return; - } - throw new ClosedCliError( - "Native down frontend finalization is unconfirmed: this run's services are stopped with data retained, but its frontend exited before confirming cleanup. Run hack down --recover to finish retaining cleanup." - ); + // Exact captured attempt, dead frontend, stopped graph and all frontend + // effects are rechecked under the existing finalization lock. Recovery + // remains a separate fact from that frontend completing its own cleanup. + await recoverNativeProjectFinalization({ + scope: opts.scope, + run, + token: finalization, + expectAttempt: finalization.attempt, + verifyEffects: () => + verifyNativeFrontendRecovery({ + runtime: opts.runtime, + scope: opts.scope, + run, + httpsPort: finalization.httpsPort, + httpsLease: + finalization.version === 3 ? finalization.httpsLease : undefined, + legacy: false, + invoke, + cleanupLifecycle: async () => { + await opts.retireHostProcesses?.(run); + }, + }), + }); + await waitNativeProjectFinalization({ + scope: opts.scope, + run, + token: finalization, + timeoutMs: 1, + }); + return; } }; const hooks = opts.hooks; @@ -680,6 +713,7 @@ export async function nativeProjectDown(opts: { await requestOwnedStop({ runtime: opts.runtime, projectRoot: opts.scope.projectRoot, + scope: opts.scope, run, invoke, selection, diff --git a/src/backends/native-project-observe.ts b/src/backends/native-project-observe.ts index 474e4c0fe..9b12b94f7 100644 --- a/src/backends/native-project-observe.ts +++ b/src/backends/native-project-observe.ts @@ -67,9 +67,9 @@ function runtimeNotRunning(): HackCliError { * Native ps result. `pending` reports frontend startup intent before a run mapping * exists; it grants no readiness or live-process authority. `runtime_not_running` means the project has an owned run * mapping but the candidate runtime is stopped: nothing was observed, so - * `phase` is null and `items` is empty. `runtime_busy` means another operation - * held the exclusive provider lease for the whole bounded read (a starting - * foreground `up` holds it until its graph is ready): nothing was observed either. + * `phase` is null and `items` is empty. `runtime_busy` means the journal changed + * during presentation, so no consistent snapshot is returned. Presentation + * takes no provider lease; legacy lease-busy responses remain transitional. */ export type NativeProjectPsStatus = | "not_started" @@ -178,16 +178,27 @@ function containerItem( exitCode: typeof observation.code === "number" ? observation.code : null, }; } -/** Read-only inspection retries brief provider lease contention, never effects. */ -async function inspect(opts: Options, run: NativeProjectRun) { +/** Presentation is lease-free; finite log inspection keeps its bounded lease retry. */ +async function inspect( + opts: Options, + run: NativeProjectRun, + statusOnly = false +) { return inspected( - await inspectNativeProjectGraph({ - runtime: opts.runtime, - projectRoot: opts.scope.projectRoot, - run: run.run, - invoke: opts.invoke, - busyWaitMs: OBSERVER_BUSY_WAIT_MS, - }), + statusOnly + ? await (opts.invoke ?? invokeNativeRuntime)({ + runtime: opts.runtime, + cwd: opts.scope.projectRoot, + args: ["graph", "status", "--run-id", run.run, "--json"], + timeoutMs: 30_000, + }) + : await inspectNativeProjectGraph({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + run: run.run, + invoke: opts.invoke, + busyWaitMs: OBSERVER_BUSY_WAIT_MS, + }), run, opts.includeReadiness ); @@ -205,7 +216,11 @@ function unobservedStatus( if (isNativeRuntimeNotRunning(error)) { return "runtime_not_running"; } - if (isNativeProviderBusy(error)) { + if ( + isNativeProviderBusy(error) || + (error instanceof NativeRuntimeRequestError && + error.nativeCode === "graph_status_changed") + ) { return "runtime_busy"; } throw error; @@ -270,6 +285,14 @@ async function unobservedProject( * A stopped candidate runtime is an ordinary state for this read: it reports * `runtime_not_running` instead of failing. Every other refusal still throws. */ +function ownerJournalChanged(owner: unknown): boolean { + return ( + isRecord(owner) && + owner.ok === false && + owner.code === "graph_status_changed" + ); +} + export async function nativeProjectPs( opts: Options ): Promise { @@ -287,7 +310,7 @@ export async function nativeProjectPs( } let snapshot: Awaited>; try { - snapshot = await inspect(opts, run); + snapshot = await inspect(opts, run, true); } catch (error: unknown) { return await unobservedProject(opts, run, error); } @@ -302,6 +325,15 @@ export async function nativeProjectPs( args: ["graph", "owner-status", "--run-id", run.run, "--json"], timeoutMs: 15_000, }); + if (ownerJournalChanged(owner)) { + return { + backend: "native", + status: "runtime_busy", + run: run.run, + phase: null, + items: [], + }; + } if ( !isRecord(owner) || owner.ok !== true || diff --git a/src/backends/native-runtime-client.ts b/src/backends/native-runtime-client.ts index 959b0ab2e..9d0cdf28c 100644 --- a/src/backends/native-runtime-client.ts +++ b/src/backends/native-runtime-client.ts @@ -790,7 +790,8 @@ function nativeFailureCause(input: { input.code ) || (input.code === "native_graph_foreground" && - input.cause === NATIVE_RUNTIME_NOT_RUNNING) + (input.cause === NATIVE_RUNTIME_NOT_RUNNING || + input.cause === "graph_status_changed")) ? input.cause : undefined; } diff --git a/src/lib/native-authored-command.ts b/src/lib/native-authored-command.ts index 935245189..cc7d5fa28 100644 --- a/src/lib/native-authored-command.ts +++ b/src/lib/native-authored-command.ts @@ -406,6 +406,8 @@ const STATUS_NOTES: Readonly< "Native project startup is pending: a start reserved its intent but has not reached Ready (it is still starting, or it failed and its startup evidence is retained). No current service status is confirmed. If no startup is running, `hack down --recover` retires an attempt that never reserved runtime state.", runtime_not_running: "Native runtime is not running; no services are running.", + runtime_busy: + "Native project changed during status observation; retry `hack ps` for a current snapshot.", }; async function showStatus(options: { diff --git a/tests/models/tla/native-status-start/COVERAGE.md b/tests/models/tla/native-status-start/COVERAGE.md new file mode 100644 index 000000000..fc91e7354 --- /dev/null +++ b/tests/models/tla/native-status-start/COVERAGE.md @@ -0,0 +1,56 @@ +# Status during native startup (HACK-2110) + +Question: can a presentation read make a start fail at a provider lease acquisition, +including after its service containers are running? + +![Status observation and startup ownership](flow.svg) + +[Diagram source](flow.mmd); the decision diagram is an implementation overview, +not an additional checked model. The cleanup branch describes adopted recovery. + +`StatusStart.tla` enumerates one start, one overlapping reader and seven acquisition +stages: pool, relay, graph, probes, reservation, bridge and publisher. The first +three stages already wait for acquisition; the later stages refuse immediately. +The positive instance makes reads lease-free. The negative instance restores the +exclusive read lease and violates `ReaderNeverFailsStart` at `ReaderRefusal`, with +services already running and the reader holding the lease. + +## Source and regression mapping + +| Transition/property | Implementation | Regression | +| --- | --- | --- | +| Read / ReadDone never own the provider lease | `graph/status.rs`, `engine/inspection.rs`; adopted owner-status uses `Observer`; authored `Action::Observe` uses `runtime::status` | `status_and_mutation_overlap_without_taking_each_others_lease`, journal replacement/pending ABA/in-place mutation tests; frontend observe/ps fixtures | +| Initial bounded acquisition; holder named on expiry | `lifecycle::startup_lease`, `operation_lease`, `state::Lock` | `project_start_waits_for_a_legacy_status_holder_then_runs_without_replaying_admission`, `lasting_status_holder_is_named_at_the_acquisition_deadline_without_starting_services` | +| Later acquisition refuses a held lease | publisher startup, probe supervision and bridge publication | Negative control reproduces the old interleaving; actual lock test confirms second-mutator refusal | +| Down requires exact dead-owner proof | `native-project-down.ts`, `live_owner_cleanup.rs` | Plain-down frontend tests, settled-binding and missing-proof Rust tests; recovery repeats proof checks under its lease | + +Authored `Action::Status` remains the full locked readiness check. Presentation +`Observed` replies cannot satisfy it. Status never records failure or reaps a +publisher. Failure recording remains at startup/effectful verification and owned +cleanup; first failure wins. Cleanup captures before stopping listeners and folds +evidence into its required journal commit. Incomplete-rebind cleanup does not +capture new listener exits. An owner crash before cleanup cannot reconstruct its +host child's exit. No background recording tick or new control-loop lease exists. + +## Bounds and proof limits + +Positive: **110 distinct states**. Negative: the named invariant fails with +`failedByReader = TRUE`, `services = TRUE`, `reader = TRUE`, `lease = "reader"`. +Both use the repository checksum-pinned TLC 1.7.4 distribution (banner 2.19), +Temurin 17.0.20+101, two workers and a 512 MiB heap. The registry verifies the count +and same-state negative witness. Apalache is not run. + +Each stage's effects are atomic in this model. There is no fairness or liveness +claim. The model does not implement filesystem durability, guest identity, socket +transport, bounded wall-clock waits or recovery journals. `SecondMutation` is an +assumption preserving existing refusal, not an independent proof of admission. +`Down` abstracts complete exact-death/identity proof and assumes no uncertain +pending effect; it does not prove arbitrary failed-start recoverability. Unknown +identity never authorizes cleanup. Authored pre-Ready recovery is unchanged. + +The held-lock test uses real flock and a fake native backend; it proves bounded +acquisition plus graph execution, not macOS provider behavior. Mac-only cleanup +and publisher tests require hosted macOS execution; cross-Clippy only compiles +them. Lead acceptance polls `ps` every 0.5 seconds through ten real VM starts, +including cross-project observation in the same pool, and checks plain `down` +after a post-service failure. Portable green checks do not replace that gate. diff --git a/tests/models/tla/native-status-start/StatusStart.tla b/tests/models/tla/native-status-start/StatusStart.tla new file mode 100644 index 000000000..d8e0197e7 --- /dev/null +++ b/tests/models/tla/native-status-start/StatusStart.tla @@ -0,0 +1,73 @@ +--------------------------- MODULE StatusStart --------------------------- +EXTENDS Naturals, TLC +CONSTANT ReaderTakesLease +VARIABLES stage, lease, reader, failedByReader, failed, services, dead, + sameIdentity, stopped, secondAdmitted +vars == <> + +\* Seven separate acquisitions: pool, relay, graph, probes, reservation, +\* bridge, publisher. The first three already retry acquisition in production. +Init == /\ stage = 1 /\ lease = "free" /\ reader = FALSE + /\ failedByReader = FALSE /\ failed = FALSE /\ services = FALSE + /\ dead = FALSE /\ sameIdentity = TRUE /\ stopped = FALSE + /\ secondAdmitted = FALSE + +Read == /\ ~reader /\ (~ReaderTakesLease \/ lease = "free") + /\ reader' = TRUE + /\ lease' = IF ReaderTakesLease THEN "reader" ELSE lease + /\ UNCHANGED <> +ReadDone == /\ reader /\ reader' = FALSE + /\ lease' = IF ReaderTakesLease THEN "free" ELSE lease + /\ UNCHANGED <> + +Acquire == /\ stage <= 7 /\ ~failed /\ lease = "free" + /\ lease' = "start" + /\ UNCHANGED <> +FinishStage == /\ lease = "start" /\ stage <= 7 /\ ~failed + /\ stage' = stage + 1 /\ lease' = "free" + /\ services' = (services \/ stage = 3) + /\ UNCHANGED <> +\* These later acquisitions have no retry. This is the baseline counterexample. +ReaderRefusal == /\ stage \in 4..7 /\ ~failed /\ lease = "reader" + /\ failed' = TRUE /\ failedByReader' = TRUE + /\ UNCHANGED <> +OtherFailure == /\ ~failed /\ lease = "free" /\ services + /\ failed' = TRUE + /\ UNCHANGED <> +OwnerExit == /\ failed /\ ~dead /\ dead' = TRUE + /\ UNCHANGED <> +IdentityChanges == /\ ~stopped /\ sameIdentity /\ sameIdentity' = FALSE + /\ UNCHANGED <> +\* A new explicit plain down invokes the existing exact-proof recovery. +\* Unknown identity never grants cleanup. No crash/uncertain effect replay modeled. +Down == /\ failed /\ services /\ dead /\ sameIdentity /\ lease = "free" + /\ services' = FALSE /\ stopped' = TRUE + /\ UNCHANGED <> +\* The run's existing foreground/admission claim rejects a second mutation; +\* neither status nor this change grants it a capability. +SecondMutation == /\ ~stopped /\ secondAdmitted' = FALSE + /\ UNCHANGED <> +Next == Read \/ ReadDone \/ Acquire \/ FinishStage \/ ReaderRefusal + \/ OtherFailure \/ OwnerExit \/ IdentityChanges \/ Down \/ SecondMutation +Spec == Init /\ [][Next]_vars +TypeOK == /\ stage \in 1..8 /\ lease \in {"free", "reader", "start"} + /\ reader \in BOOLEAN /\ failedByReader \in BOOLEAN + /\ failed \in BOOLEAN /\ services \in BOOLEAN /\ dead \in BOOLEAN + /\ sameIdentity \in BOOLEAN /\ stopped \in BOOLEAN + /\ secondAdmitted \in BOOLEAN +ReaderNeverFailsStart == ~failedByReader +\* Assumed unchanged admission policy, not independently checked by this model. +NoSecondMutation == ~secondAdmitted +StopRequiresProof == stopped => dead /\ sameIdentity /\ ~services +============================================================================= diff --git a/tests/models/tla/native-status-start/flow.mmd b/tests/models/tla/native-status-start/flow.mmd new file mode 100644 index 000000000..39b9dcc45 --- /dev/null +++ b/tests/models/tla/native-status-start/flow.mmd @@ -0,0 +1,12 @@ +flowchart TD + Ps["hack ps"] --> Observer["Read-only Observer"] + Observer --> Fence{"Journal unchanged?"} + Fence -->|yes| Status["Service state / health"] + Fence -->|no| Busy["runtime_busy: retry read"] + Up["hack up"] --> Lease["Exclusive provider lease"] + Lease --> Effects["Startup effect stages"] + Effects --> Ready["Verified readiness"] + Effects -->|failure| Down["plain hack down"] + Down --> Proof["Existing exact-owner proofs"] + Proof -->|proved| Cleanup["Owned cleanup"] + Proof -->|uncertain| Refuse["Named refusal"] diff --git a/tests/models/tla/native-status-start/flow.svg b/tests/models/tla/native-status-start/flow.svg new file mode 100644 index 000000000..e7a9fbdaf --- /dev/null +++ b/tests/models/tla/native-status-start/flow.svg @@ -0,0 +1,66 @@ +yesnofailureproveduncertainhack psRead-only ObserverJournal unchanged?Service state / healthruntime_busy: retry readhack upExclusive provider leaseStartup effect stagesVerified readinessplain hack downExisting exact-ownerproofsOwned cleanupNamed refusal \ No newline at end of file diff --git a/tests/models/tla/native-status-start/negative.cfg b/tests/models/tla/native-status-start/negative.cfg new file mode 100644 index 000000000..b45930d85 --- /dev/null +++ b/tests/models/tla/native-status-start/negative.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANT ReaderTakesLease = TRUE +INVARIANTS TypeOK ReaderNeverFailsStart NoSecondMutation StopRequiresProof +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-status-start/positive.cfg b/tests/models/tla/native-status-start/positive.cfg new file mode 100644 index 000000000..def063784 --- /dev/null +++ b/tests/models/tla/native-status-start/positive.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANT ReaderTakesLease = FALSE +INVARIANTS TypeOK ReaderNeverFailsStart NoSecondMutation StopRequiresProof +CHECK_DEADLOCK FALSE diff --git a/tests/native-authored-graph-protocol.test.ts b/tests/native-authored-graph-protocol.test.ts index e9a1a0ed0..d4defa2ba 100644 --- a/tests/native-authored-graph-protocol.test.ts +++ b/tests/native-authored-graph-protocol.test.ts @@ -899,6 +899,23 @@ test("native ready and cleanup carry the same admitted binding through mutable p ).toThrow("invalid"); }); +test("presentation observation cannot satisfy startup readiness status", () => { + const expected = parseNativeAuthoredReceipt(receipt()); + const observed = status(); + observed.result.outcome = "observed"; + expect( + parseNativeAuthoredControl(observed, expected, "observe").observations?.[ + "a.peer" + ] + ).toEqual({ state: "running", health: "healthy" }); + expect(() => + parseNativeAuthoredControl(observed, expected, "status") + ).toThrow("invalid"); + expect(() => + parseNativeAuthoredControl(status(), expected, "observe") + ).toThrow("invalid"); +}); + test("self-valid foreign images IDs readiness owner and boot refuse against original membership", () => { const expected = parseNativeAuthoredReceipt(receipt()); for (const mutate of [ diff --git a/tests/native-authored-project-observe.test.ts b/tests/native-authored-project-observe.test.ts index 0270d8118..3b1926991 100644 --- a/tests/native-authored-project-observe.test.ts +++ b/tests/native-authored-project-observe.test.ts @@ -128,7 +128,7 @@ async function fixture( run: provenance.run, review: review.review_id, result: { - outcome: "status", + outcome: "observed", snapshot: { receipt, observations: { web: { state: "running", health: "healthy" } }, @@ -139,7 +139,7 @@ async function fixture( const attempts = join(nativeHome, "status-attempts"); await Bun.write( runtime.binary, - `#!${process.execPath}\nimport { appendFileSync } from 'node:fs';\nconst expected=${JSON.stringify(["--candidate-root", nativeHome, "graph", "native", "control", "--run-id", provenance.run, "--action", "status", "--json"])};if(JSON.stringify(process.argv.slice(2))!==JSON.stringify(expected))process.exit(97);appendFileSync(${JSON.stringify(attempts)},'status\\n');${nativeCode === undefined ? `console.log(${JSON.stringify(JSON.stringify(reply))});` : `process.stderr.write(${JSON.stringify(JSON.stringify({ code: nativeCode, ...(nativeCode === "native_graph_foreground" ? { cause_code: "runtime_not_running" } : {}) }))});process.exit(1);`}\n` + `#!${process.execPath}\nimport { appendFileSync } from 'node:fs';\nconst expected=${JSON.stringify(["--candidate-root", nativeHome, "graph", "native", "control", "--run-id", provenance.run, "--action", "observe", "--json"])};if(JSON.stringify(process.argv.slice(2))!==JSON.stringify(expected))process.exit(97);appendFileSync(${JSON.stringify(attempts)},'status\\n');${nativeCode === undefined ? `console.log(${JSON.stringify(JSON.stringify(reply))});` : `process.stderr.write(${JSON.stringify(JSON.stringify({ code: nativeCode, ...(nativeCode === "native_graph_foreground" ? { cause_code: "runtime_not_running" } : {}) }))});process.exit(1);`}\n` ); await chmod(runtime.binary, 0o700); const mapping = join( @@ -223,7 +223,7 @@ test.each([ "--run-id", f.receipt.review.provenance.run, "--action", - "status", + "observe", "--json", ]); expect(request.boundNativeStatusDrain).toBe(true); @@ -231,7 +231,7 @@ test.each([ return { ...f.reply, result: { - outcome: "status", + outcome: "observed", snapshot: { receipt: f.receipt, observations: { web: observation }, @@ -266,7 +266,7 @@ test.each([ owner: { ...f.reply, result: { - outcome: "status", + outcome: "observed", snapshot: { ...snapshot, receipt: { ...f.receipt, owner: "4".repeat(32) }, @@ -276,7 +276,7 @@ test.each([ member: { ...f.reply, result: { - outcome: "status", + outcome: "observed", snapshot: { ...snapshot, observations: { foreign: { state: "dead" } }, @@ -286,21 +286,21 @@ test.each([ source: { ...f.reply, result: { - outcome: "status", + outcome: "observed", snapshot: { ...snapshot, receipt: { ...f.receipt, source: {} } }, }, }, storage: { ...f.reply, result: { - outcome: "status", + outcome: "observed", snapshot: { ...snapshot, receipt: { ...f.receipt, data: {} } }, }, }, malformed: { ...f.reply, result: { - outcome: "status", + outcome: "observed", snapshot: { ...snapshot, observations: { @@ -534,6 +534,32 @@ test("a stopped runtime is a status for a ready generation; other refusals still ).rejects.toThrow(); })); +test.each([ + "graph_status_changed", + "native_graph_foreground", +])("journal transition is a presentation status (%s)", (nativeCode) => + owned(async () => { + const f = await fixture(); + const status = await nativeAuthoredProjectPs({ + ...f, + invoke: async ({ args }) => { + expect(args).toContain("observe"); + expect(args).not.toContain("status"); + throw new NativeRuntimeRequestError({ + message: "changed", + nativeCode, + nativeCauseCode: + nativeCode === "native_graph_foreground" + ? "graph_status_changed" + : undefined, + }); + }, + }); + expect(status.status).toBe("runtime_busy"); + expect(status.phase).toBeNull(); + expect(status.items).toEqual([]); + })); + test.each( ["rebirth", "content", "binary", "recovery"].flatMap((fault) => ["runtime_not_running", "native_graph_foreground"].map((nativeCode) => ({ diff --git a/tests/native-project-down.test.ts b/tests/native-project-down.test.ts index d8860a6a5..e5ce86af9 100644 --- a/tests/native-project-down.test.ts +++ b/tests/native-project-down.test.ts @@ -316,13 +316,19 @@ test("already-stopped unacknowledged frontend refuses without replaying compute" ...opts, invoke: async (request) => { calls.push(request.args[1]!); + if (request.args[1] === "foreground-owner") { + return ownerSelection("uncertain", "stopped-data-retained"); + } return snapshot(true); }, }); - await expectClosedRefusal(result, "frontend finalization is unconfirmed"); - await expectClosedRefusal(result, "hack down --recover"); + await expectClosedRefusal(result, "foreground owner"); expect(performance.now() - started).toBeLessThan(10_000); - expect(calls).toEqual(["inspect"]); + expect(calls).toEqual([ + "inspect", + "foreground-owner", + "retire-unadmitted-owner", + ]); expect(await loadNativeProjectRun(opts.scope)).toEqual(run); }); @@ -1070,7 +1076,7 @@ function ownerSelection(foreground: string, phase = "ready-observed") { }; } -test("down without --recover names the recovery when the foreground owner is gone", async () => { +test("plain down refuses missing owner recovery proof without replaying cleanup", async () => { const opts = await fixture(); const calls: string[] = []; await expectClosedRefusal( @@ -1087,9 +1093,9 @@ test("down without --recover names the recovery when the foreground owner is gon return snapshot(); }, }), - "run hack down --recover" + "cannot confirm this run's foreground owner state" ); - expect(calls).toEqual(["inspect", "inspect", "cleanup"]); + expect(calls).toEqual(["inspect", "inspect", "cleanup", "foreground-owner"]); expect(await loadNativeProjectRun(opts.scope)).toEqual(run); }); @@ -1364,7 +1370,10 @@ test("down --recover names a refused unadmitted owner retirement without hooks o expect(await loadNativeProjectRun(opts.scope)).toEqual(run); }); -test("down --recover retires the dead frontend in one command and admits a fresh plain start", async () => { +test.each([ + false, + true, +])("down retires the dead frontend in one command and admits a fresh plain start (recover=%s)", async (recover) => { const opts = await fixture(); // A frontend that began its finalization and exited without completing it. const module = join( @@ -1389,7 +1398,7 @@ test("down --recover retires the dead frontend in one command and admits a fresh const started = performance.now(); const result = await nativeProjectDown({ ...opts, - recover: true, + recover, invoke: async (request) => { if (request.args[1] === "foreground-owner") { return ownerSelection("exited"); @@ -1815,3 +1824,51 @@ for (const predecessor of [false, true]) { await next.complete(); }); } + +test.each([ + { phase: "ready-observed", original: null }, + { phase: "failed-retained", original: null }, + { phase: "cleanup-intent", original: "1".repeat(64) }, +])("plain down resumes exact dead-owner cleanup after $phase without replaying the refused stop", async ({ + phase, + original, +}) => { + const opts = await fixture(); + const calls: string[] = []; + let recovered = false; + const result = await nativeProjectDown({ + ...opts, + invoke: async (request) => { + const command = request.args[1]!; + calls.push(command); + if (command === "cleanup") { + throw new NativeRuntimeRequestError({ + nativeCode: "graph_owner_recovery", + message: "owner unavailable", + }); + } + if (command === "foreground-owner") { + return { + ...ownerSelection("exited", phase), + live_owner_recovery: original, + failed_start_recoverable: phase === "failed-retained", + }; + } + if (command === "recover-live-owner") { + expect(request.args).toContain(original ?? receiptSha); + recovered = true; + return { + run: run.run, + recovered: true, + data_retained: true, + publisher_retired: true, + }; + } + return snapshot(recovered); + }, + }); + expect(result.status).toBe("stopped"); + expect(calls.filter((call) => call === "cleanup")).toHaveLength(1); + expect(calls.filter((call) => call === "recover-live-owner")).toHaveLength(1); + expect(await loadNativeProjectRun(opts.scope)).toEqual(run); +}); diff --git a/tests/native-project-observe.test.ts b/tests/native-project-observe.test.ts index 859c0e3b0..d82720d4e 100644 --- a/tests/native-project-observe.test.ts +++ b/tests/native-project-observe.test.ts @@ -130,7 +130,7 @@ test("a lease-busy ps still reports an authenticated pending publisher exit with let response: unknown = owner; const invoke: typeof import("../src/backends/native-runtime-client.ts").invokeNativeRuntime = async (request) => { - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { throw new NativeRuntimeRequestError({ nativeCode: "provider_busy", message: "busy", @@ -183,6 +183,27 @@ test("ps returns current native observations and refuses each ownership mismatch nativeProjectPs({ ...opts, invoke: async () => value }) ).rejects.toThrow(); }); +test("an owner journal transition returns busy without stale service rows", async () => { + const opts = await fixture(); + const result = await nativeProjectPs({ + ...opts, + invoke: async ({ args }) => + args[1] === "status" + ? snapshot(true) + : { + ok: false, + code: "graph_status_changed", + }, + }); + expect(result).toEqual({ + backend: "native", + status: "runtime_busy", + run: run.run, + phase: null, + items: [], + }); +}); + test("ps distinguishes a live foreground owner from historical ready containers", async () => { const opts = await fixture(); const calls: string[] = []; @@ -196,17 +217,17 @@ test("ps distinguishes a live foreground owner from historical ready containers" }; const invoke = async (request: { args: readonly string[] }) => { calls.push(request.args[1]!); - return request.args[1] === "inspect" ? snapshot(true) : owner; + return request.args[1] === "status" ? snapshot(true) : owner; }; expect(await nativeProjectPs({ ...opts, invoke })).toMatchObject({ status: "observed", phase: "ready-observed", }); - expect(calls).toEqual(["inspect", "owner-status"]); + expect(calls).toEqual(["status", "owner-status"]); const failed = await nativeProjectPs({ ...opts, invoke: async (request) => { - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(true); } throw new Error("graph_owner_recovery"); @@ -221,7 +242,7 @@ test("ps distinguishes a live foreground owner from historical ready containers" await nativeProjectPs({ ...opts, invoke: async (request) => - request.args[1] === "inspect" + request.args[1] === "status" ? snapshot(true) : { ...owner, runtime_verified: false }, }) @@ -230,7 +251,7 @@ test("ps distinguishes a live foreground owner from historical ready containers" await nativeProjectPs({ ...opts, invoke: async (request) => - request.args[1] === "inspect" + request.args[1] === "status" ? snapshot(true) : { ...owner, run: "foreign" }, }) @@ -257,7 +278,7 @@ test("ps keeps current service rows and marks the reaped publisher's port down", ...opts, includeReadiness: true, invoke: async (request) => - request.args[1] === "inspect" + request.args[1] === "status" ? value : { ok: true, @@ -507,7 +528,7 @@ test("plain native ps warns when the foreground owner has died despite running c await writeFile( binary, `#!/bin/sh -if [ "$3" = graph ] && [ "$4" = inspect ]; then +if [ "$3" = graph ] && { [ "$4" = inspect ] || [ "$4" = status ]; }; then cat <<'JSON' ${JSON.stringify(snapshot(true))} JSON @@ -666,7 +687,7 @@ test("ps reports a stopped runtime for an owned mapping instead of failing", asy items: [], }); // One inspect; no owner-status or other follow-up request. - expect(calls).toEqual([`graph inspect --run-id ${run.run} --json`]); + expect(calls).toEqual([`graph status --run-id ${run.run} --json`]); }); test("ps still fails on every other native refusal", async () => { const opts = await fixture(); @@ -711,7 +732,7 @@ test("logs refuse a stopped runtime with a closed, coded error", async () => { }) ).rejects.toThrow("(engine_protocol)"); }); -test("ps reports a busy runtime after bounded lease retries instead of failing", async () => { +test("ps reports a changing snapshot without taking or retrying a lease", async () => { const opts = await fixture(); let calls = 0; const started = performance.now(); @@ -720,7 +741,7 @@ test("ps reports a busy runtime after bounded lease retries instead of failing", ...opts, invoke: () => { calls++; - return Promise.reject(nativeRefusal("provider_busy")); + return Promise.reject(nativeRefusal("graph_status_changed")); }, }) ).toEqual({ @@ -730,8 +751,8 @@ test("ps reports a busy runtime after bounded lease retries instead of failing", phase: null, items: [], }); - // Read-only inspection retried lease contention for its bounded budget only. - expect(calls).toBeGreaterThan(1); + // One status read, then the existing pending-publisher observation. + expect(calls).toBe(2); expect(performance.now() - started).toBeLessThan(10_000); }); test("ps and logs observe after brief lease contention without failing", async () => { @@ -741,7 +762,7 @@ test("ps and logs observe after brief lease contention without failing", async ( if (request.args[1] === "inspect" && busy-- > 0) { throw nativeRefusal("provider_busy"); } - if (request.args[1] === "inspect") { + if (["inspect", "status"].includes(request.args[1] ?? "")) { return snapshot(); } return { container: id, stdout: "out\n", stderr: "", truncated: false }; @@ -787,7 +808,7 @@ if [ -e '${marker}' ]; then echo '{"code":"provider_busy","message":"Another candidate operation holds the provider lock."}' >&2 exit 2 fi -if [ "$3" = graph ] && [ "$4" = inspect ]; then +if [ "$3" = graph ] && { [ "$4" = inspect ] || [ "$4" = status ]; }; then cat <<'JSON' ${JSON.stringify(exited)} JSON From 2c4e988dc43d6562393ddc6ca380882d09849926 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:23:06 +0000 Subject: [PATCH 2/5] fix(native): recover settled authored starts before frontend Ready --- docs/guides/native-candidate.md | 17 +- docs/reference/native-authored-commands.md | 57 ++- docs/reference/v5-model-coverage.md | 2 +- .../graph/native/foreground/recovery.rs | 31 +- .../native/foreground/recovery/cleanup.rs | 4 +- .../graph/native/foreground/recovery/tests.rs | 72 +++ .../src/provider/graph/native/journal.rs | 44 +- .../src/provider/graph/native/tests.rs | 50 ++ scripts/lib/tla-runtime-models.ts | 37 +- src/backends/native-authored-hook-journal.ts | 12 +- src/backends/native-authored-hook-stop.ts | 18 +- .../native-authored-host-processes.ts | 7 + .../native-authored-project-recovery.ts | 484 +++++++++++++----- src/backends/native-authored-project-run.ts | 60 +++ .../native-authored-recovery-protocol.ts | 56 +- src/backends/native-authored-stop-errors.ts | 9 + .../native-authored-unreserved-start.ts | 15 +- src/lib/native-authored-command.ts | 74 ++- tests/models/tla/README.md | 5 + .../tla/native-frontend-recovery/COVERAGE.md | 41 ++ .../tla/native-frontend-recovery/PreReady.tla | 82 +++ .../tla/native-frontend-recovery/flow.mmd | 15 + .../tla/native-frontend-recovery/flow.svg | 66 +++ .../pre-ready-negative.cfg | 4 + .../pre-ready-positive.cfg | 4 + .../pre-ready-release.cfg | 4 + .../pre-ready-replay.cfg | 4 + tests/native-authored-command.test.ts | 79 +++ tests/native-authored-hook-stop.test.ts | 23 +- tests/native-authored-host-processes.test.ts | 59 +++ .../native-authored-project-recovery.test.ts | 192 ++++++- .../native-authored-recovery-protocol.test.ts | 71 +++ 32 files changed, 1485 insertions(+), 213 deletions(-) create mode 100644 src/backends/native-authored-stop-errors.ts create mode 100644 tests/models/tla/native-frontend-recovery/COVERAGE.md create mode 100644 tests/models/tla/native-frontend-recovery/PreReady.tla create mode 100644 tests/models/tla/native-frontend-recovery/flow.mmd create mode 100644 tests/models/tla/native-frontend-recovery/flow.svg create mode 100644 tests/models/tla/native-frontend-recovery/pre-ready-negative.cfg create mode 100644 tests/models/tla/native-frontend-recovery/pre-ready-positive.cfg create mode 100644 tests/models/tla/native-frontend-recovery/pre-ready-release.cfg create mode 100644 tests/models/tla/native-frontend-recovery/pre-ready-replay.cfg diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 87b23a786..c3f4f1bf1 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -3796,7 +3796,22 @@ For adopted projects, plain `down` also selects exact-proof recovery of a Recovery rechecks owner death, boot, incarnation, selected receipt and resource inventories under its lease. Unsettled bindings, uncertain owners, corrupt or partial journals and changed identities still refuse by name; they grant no -cleanup authority. Authored pre-Ready recovery is unchanged. +cleanup authority. + +For authored projects, plain `down` and `down --recover` use the same recovery +proof when the frontend never saved Ready. A saved start and source envelope, +an absent Ready mapping, an exact dead admission owner (or a freshly held +admission), and a settled native receipt are required. Runtime Ready, failed, +and cleanup phases may be selected; `preparing`, pending writes and uncertain +resource effects refuse. Never-created resources must have no ID and must be +proved absent by exact name and labels before cleanup marks them removed. +Existing host-process and hook records require proof of settlement; recovery +does not signal saved PIDs. The original receipt and resource identities stay +fixed, cleanup progress only advances, and files retire only after authenticated +Removed with null observations. A version 3 frontend intent keeps incomplete +recovery blocking startup, including the gap between admission release and +completion. Graph2 and fully enrolled Graph4 receipts are supported; source and +topology recovery remain outside this proof. Nothing recompiles or replays startup. Failure evidence is retained by startup and effectful verification, and by owned cleanup for listeners that exited before cleanup stops them. The first recorded diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 4b23c2d80..98484df7d 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -237,8 +237,10 @@ stop; no host declaration, hook permit, new controller or stored-PID signal is created. Success requires authenticated Removed, the original child and detached group's settlement, and exact endpoint/start/Ready/source retirement. Compute shutdown retains persistent data; this command adds no volume deletion authority. -A missing, changed or dead endpoint refuses. Partial publication or retirement -keeps startup evidence blocked. Legacy no-host version 1 records remain ineligible for dead-owner recovery. +A missing, changed or dead endpoint cannot authorize an owner-mediated stop. +Plain `down` then tries the same independent recovery proof as `down --recover`. +Partial publication or retirement keeps startup evidence blocked unless that +exact proof permits monotonic completion. Legacy no-host version 1 records remain ineligible for dead-owner recovery. New version 2 records capture the original native child while its foreground owns it: PID, birth, executable identities, UID, host boot session, process group and session. Recovery uses two bounded complete process-metadata observations. It refuses @@ -276,12 +278,12 @@ kill/recover/up or retained SQL behavior. Portable private-filesystem and stand-in lifecycle controls cover this boundary; installed Source4 SQL retention through public down/up remains a separate runtime gate. -`down --recover` explicitly retires a complete stored native generation whose +`down --recover` explicitly selects recovery of a stored native generation whose foreground publisher and frontend admission owner are both dead on the same host boot. New version 4 native publications use the kernel boot-session UUID, which -remains independent of calendar clock correction. Recovery requires the exact stored Ready, +remains independent of calendar clock correction. Ready recovery requires the exact stored Ready, startup intent and private source envelope, and the original guest incarnation. -Live owners, older version 2 dead publications, partial startup, pending writes, +The absent-Ready case below has a separate selection. Live owners, older version 2 dead publications, unsettled startup effects, pending writes, changed files and rebooted guests refuse. Env/profile changes, service subsets and `--json` are unsupported for this operation. It neither recompiles authored input nor acquires managed values or starts a provider. @@ -295,7 +297,7 @@ pending writes, retained owner records and changed history still select recovery or refuse; they do not qualify for the no-op. When the selection has a retained startup intent but no Ready mapping and no -recovery intent, `down --recover` takes a different path. It handles only an +recovery intent, both `down` and `down --recover` first try retirement of an incomplete startup that never reserved runtime state. One example is an attempt retained by an earlier CLI before this retirement existed. The command first takes a fresh startup admission. A live or interrupted foreground CLI still holds @@ -305,7 +307,48 @@ the exact run. It retires the start intent and the validated source envelope onl when the native runtime refuses `native_graph_unreserved`. It removes the source first, so an interrupted retirement keeps the start intent blocking. Any receipt, any other refusal, or a successful inspection retains every file. This path does -not compile input, acquire values, serve a run or touch guest resources. +not compile input, acquire values, serve a run or touch guest resources. If it +refuses because runtime state was reserved, the command independently attempts +the settled recovery proof below; that refusal grants no authority. + +When Ready was never saved, recovery requires the exact start and private source +envelope, repeatedly proves Ready absent, and records a distinct version 3 +frontend intent. A live admission refuses. An interrupted admission must name +the exact dead owner; if none exists, recovery holds a fresh admission before +taking recovery and mutation leases and saves its full identity. Every intent +reader blocks new startup until recovery completes. Before releasing a fresh +admission, recovery commits `admission_releasing`; a second pass requires its +absence before completing. A crash retains the exact admission for dead-owner +recovery. A changed lock or unrecorded absence refuses. +If recovery refuses after committing its intent but before recording release, +the exact admission remains held on disk. `up` stays blocked until `down` +finishes that recovery. Older bundles refuse the new runtime v2 and frontend v3 +intent versions rather than interpreting them as Ready recovery. + +The native original may be `ready-observed`, `failed-retained`, `stop-intent`, +`stopped`, `removal-intent` or `removed`. Containers must be `created`, `started`, +`stopped` or `removed` with their saved IDs, or never-created `reserved` without +an ID. Networks admit `created`, `remove-intent`, `removed`, or never-created +`reserved`. `preparing`, per-container intent/uncertain states and pending files +refuse. A never-created resource is marked removed only after exact name/label +absence checks; recovery never creates or starts it. Original bytes, phase, +boot, incarnation and resource identities remain fixed, and cleanup ranks only +advance. Runtime recovery intents use version 2; legacy version 1 keeps its +Ready-only admission. Frontend versions 1 and 2 likewise keep their Ready proof. +Older readers refuse unknown intent versions. + +Pre-Ready host processes and hooks must already have settlement proof and no +live or uncertain owned groups; recovery only observes them before native +cleanup and retires their records after Removed/null observations. A live-stop +record in this absent-Ready path refuses. It never signals a stored PID or +replays startup. Graph2 and fully enrolled Graph4 retain the existing version +restrictions; source and topology recovery remain unsupported. Named refusals +retain evidence and direct the operator to `hack ps`. + +A crash after taking admission but before creating any native journal still +refuses: unreserved retirement cannot take that stale admission, and settled +recovery has no native receipt to prove. No services were admitted in this case; +the stale admission and start record remain retained for inspection with `hack ps`. The closed version 3 publication and version 1 recovery selector remain supported with their original calendar boot qualifier; clock drift conservatively refuses diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index f215c9dd3..6530f052c 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -85,7 +85,7 @@ liveness/reachability controls have their own documented assumptions. | [native-cache-reconcile](../../tests/models/tla/native-cache-reconcile/CacheReconcile.tla) | 2,208 | 5 | | [pre-ready-retirement](../../tests/models/tla/pre-ready-retirement/Retirement.tla) | 10,965 | 8 | | [native-persistent-enrollment](../../tests/models/tla/native-persistent-enrollment/Enrollment.tla) | 38 | 6 | -| [native-frontend-recovery](../../tests/models/tla/native-frontend-recovery/Recovery.tla) | 1752 | 12 | +| [native-frontend-recovery](../../tests/models/tla/native-frontend-recovery/COVERAGE.md) | 1752 + 160 pre-Ready | 16 | | [native-storage-witness](../../tests/models/tla/native-storage-witness/Witness.tla) | 352 | 19 | | [native-file-material](../../tests/models/tla/native-file-material/Material.tla) | 8,388 | 12 | | [native-foreground-down-hooks](../../tests/models/tla/native-foreground-down-hooks/DownHooks.tla) | 1,246 | 6 | diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs index f5a691d27..22840267e 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -15,7 +15,7 @@ const LIMIT: usize = 64 * 1024; fn refused() -> CandidateError { error( "native_graph_live_owner_recovery", - "Native same-boot recovery requires the exact complete Ready selection, dead boot-qualified publisher and unchanged durable progress; evidence retained.", + "Native same-boot recovery requires an exact settled resource selection, dead boot-qualified publisher and unchanged durable progress; pending effects and evidence are retained.", ) } fn digest(bytes: &[u8]) -> String { @@ -96,7 +96,7 @@ struct Intent { } fn receipt_progress(receipt: &Receipt) -> Result { match receipt.phase { - Phase::ReadyObserved => Ok(0), + Phase::ReadyObserved | Phase::FailedRetained => Ok(0), Phase::StopIntent => Ok(1), Phase::Stopped => Ok(2), Phase::RemovalIntent => Ok(3), @@ -110,8 +110,9 @@ fn resource_progress(receipt: &Receipt) -> Result, Candidat .iter() .map(|(key, resource)| { let rank = match (&resource.kind, resource.phase.as_str()) { - (Kind::Container, "started") | (Kind::Network, "created") => 0, - (Kind::Network, "remove-intent") => 1, + (Kind::Container, "created" | "started") | (Kind::Network, "created") => 0, + (Kind::Container | Kind::Network, "reserved") if resource.id.is_none() => 0, + (Kind::Container, "stopped") | (Kind::Network, "remove-intent") => 1, (Kind::Container | Kind::Network, "removed") => 2, _ => return Err(refused()), }; @@ -121,7 +122,7 @@ fn resource_progress(receipt: &Receipt) -> Result, Candidat } impl Intent { fn original(&self, run: &str) -> Result { - if self.version != 1 + if !matches!(self.version, 1 | 2) || self.run != run || self.original.len() > 64 * 1024 || !super::super::super::hex(&self.receipt_sha256, 64) @@ -131,11 +132,19 @@ impl Intent { return Err(refused()); } let receipt: Receipt = serde_json::from_str(&self.original).map_err(|_| refused())?; - receipt.require_recovery_ready().map_err(|_| refused())?; + if self.version == 1 { + receipt.require_recovery_ready().map_err(|_| refused())?; + } else { + receipt.require_recovery_settled().map_err(|_| refused())?; + } if receipt.review.scope().run != run + || self.receipt_progress < receipt_progress(&receipt)? || self.resource_progress.keys().ne(receipt.resources.keys()) || self.resource_progress.iter().any(|(key, rank)| { - *rank > 2 || (*rank == 1 && receipt.resources[key].kind != Kind::Network) + *rank > 2 + || (self.version == 1 + && *rank == 1 + && receipt.resources[key].kind != Kind::Network) }) { return Err(refused()); @@ -153,12 +162,16 @@ impl Intent { let rank = receipt_progress(current)?; let resources = resource_progress(current)?; if id(root)? != self.journal_parent + || rank < receipt_progress(original)? || rank < self.receipt_progress || (rank == 0 && digest(bytes.as_bytes()) != self.receipt_sha256) || (self.progress != Progress::Cleanup && rank != 4) || resources .iter() .any(|(key, rank)| *rank < self.resource_progress[key]) + || resources.iter().any(|(key, rank)| { + *rank == 0 && current.resources[key].phase != original.resources[key].phase + }) { return Err(refused()); } @@ -225,7 +238,7 @@ impl Admitted<'_> { }) } } -/// Select only a dead boot-qualified complete Ready publication, or its unchanged +/// Select only a dead boot-qualified settled publication, or its unchanged /// committed recovery intent. No file is created or repaired, and no provider /// operation, private acquisition, signal or cleanup is performed. pub fn select(candidate: &Candidate, run: &str) -> Result { @@ -271,7 +284,7 @@ fn admit<'a>(candidate: &'a Candidate, run: &'a str) -> Result, Can (original, intent.receipt_sha256.clone()) } None => { - current.require_recovery_ready().map_err(|_| refused())?; + current.require_recovery_settled().map_err(|_| refused())?; Progress::Cleanup.verify(&lease)?; (current, digest(bytes.as_bytes())) } diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs index 0c33345c6..cb43815b2 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs @@ -20,7 +20,7 @@ impl<'a> Context<'a> { let intent = match admitted.intent.take() { Some(intent) => intent, None => Intent { - version: 1, + version: 2, kind: IntentKind::NativeGraphLiveOwnerRecovery, run: admitted.run.into(), journal_parent: admitted.root_identity, @@ -28,7 +28,7 @@ impl<'a> Context<'a> { receipt_sha256: admitted.receipt_sha256.clone(), publication: admitted.lease.selected().clone(), progress: Progress::Cleanup, - receipt_progress: 0, + receipt_progress: receipt_progress(&admitted.original)?, resource_progress: resource_progress(&admitted.original)?, environment: crate::provider::native_environment::Inventory::capture( admitted.candidate, diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index a9d564155..5a57d91eb 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -648,6 +648,78 @@ fn committed_intent_keeps_original_hash_and_inventory_through_cleanup_progress() fixture.assert_refused_unchanged(); } +#[test] +fn pre_ready_origins_resume_monotonic_cleanup_with_exact_original_and_never_created_members() { + for phase in [ + Phase::ReadyObserved, + Phase::FailedRetained, + Phase::StopIntent, + Phase::Stopped, + Phase::RemovalIntent, + Phase::Removed, + ] { + let fixture = Fixture::new(); + fixture.dead(); + let mut original = fixture.receipt(); + original.phase = phase.clone(); + if phase != Phase::ReadyObserved { + let resource = original.resources.get_mut("container:web").unwrap(); + resource.id = None; + resource.phase = "reserved".into(); + } + if phase == Phase::Removed { + for resource in original.resources.values_mut() { + resource.phase = "removed".into(); + } + } + journal::save(&fixture.journal_root(), &original).unwrap(); + let selected = select(&fixture.candidate, RUN).unwrap(); + cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, _| { + let intent: Intent = + serde_json::from_slice(&fs::read(fixture.journal_root().join(FILE)).unwrap()) + .unwrap(); + assert_eq!(intent.version, 2); + assert_eq!(intent.receipt_progress, receipt_progress(&original)?); + assert_eq!(intent.original(RUN)?.phase, phase); + if original.resources["container:web"].phase == "reserved" { + let mut created = original.clone(); + created.phase = Phase::StopIntent; + created.resources.get_mut("container:web").unwrap().phase = "created".into(); + let bytes = serde_json::to_string(&created).unwrap(); + assert!( + intent + .current(&fixture.journal_root(), &original, &created, &bytes) + .is_err() + ); + } + for next in [Phase::StopIntent, Phase::Stopped, Phase::RemovalIntent] { + let mut current = fixture.receipt(); + current.phase = next; + if receipt_progress(¤t)? < receipt_progress(&original)? { + continue; + } + journal::save(&fixture.journal_root(), ¤t)?; + context.guard()?; + } + let receipt = removed(&fixture); + context.finish_using(&absent_snapshot(receipt.clone()), &|_| Ok(()))?; + Ok(receipt) + }, + ) + .unwrap(); + let again = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(again.receipt.phase, phase); + assert_eq!(again.receipt_sha256, selected.receipt_sha256); + assert_eq!( + again.receipt.resources["container:web"].id, + original.resources["container:web"].id + ); + } +} + #[test] fn changed_inventory_and_regressed_resource_progress_refuse_under_original_intent() { for case in ["id", "owner", "boot", "resource-regression"] { diff --git a/packages/runtime-core/src/provider/graph/native/journal.rs b/packages/runtime-core/src/provider/graph/native/journal.rs index a7f803907..b9fdcd2b4 100644 --- a/packages/runtime-core/src/provider/graph/native/journal.rs +++ b/packages/runtime-core/src/provider/graph/native/journal.rs @@ -203,6 +203,23 @@ impl Receipt { } #[cfg(target_os = "macos")] pub(super) fn require_recovery_ready(&self) -> Result<(), CandidateError> { + self.require_recovery_settled()?; + if self.phase != Phase::ReadyObserved + || self.failure.is_some() + || self + .resources + .values() + .any(|resource| resource.id.is_none()) + { + return Err(refused()); + } + Ok(()) + } + /// Cleanup admission, never readiness evidence. A missing ID is settled only + /// for a never-created or already-removed resource; every pending effect + /// remains a refusal. The cleanup backend rechecks exact ownership/absence. + #[cfg(any(target_os = "macos", test))] + pub(super) fn require_recovery_settled(&self) -> Result<(), CandidateError> { self.validate(self.review.scope().run, &self.owner)?; // Source/topology recovery remains separate. Storage recovery may stop // compute only with complete saved verifier and independently enrolled @@ -214,12 +231,27 @@ impl Receipt { .is_some_and(|tool| tool.root.is_some() && tool.helper.is_some()) && self.data.values().all(|reference| reference.enrolled()); if !(self.version == 2 || storage) - || self.phase != Phase::ReadyObserved - || self.failure.is_some() - || self - .resources - .values() - .any(|resource| resource.id.is_none()) + || self.phase == Phase::Preparing + || (self.phase == Phase::ReadyObserved && self.failure.is_some()) + || self.resources.values().any(|resource| { + if self.phase == Phase::Removed && resource.phase != "removed" { + return true; + } + if resource.id.is_none() { + return !matches!(resource.phase.as_str(), "reserved" | "removed"); + } + match resource.kind { + Kind::Container => !matches!( + resource.phase.as_str(), + "created" | "started" | "stopped" | "removed" + ), + Kind::Network => !matches!( + resource.phase.as_str(), + "created" | "remove-intent" | "removed" + ), + Kind::Volume => true, + } + }) { return Err(refused()); } diff --git a/packages/runtime-core/src/provider/graph/native/tests.rs b/packages/runtime-core/src/provider/graph/native/tests.rs index c61467a7b..66c8677b1 100644 --- a/packages/runtime-core/src/provider/graph/native/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/tests.rs @@ -38,6 +38,56 @@ fn image() -> String { fn basic() -> Value { json!({"schema_version":1,"name":"fixture","services":{"web":{"image":image()}}}) } + +#[test] +fn recovery_settled_selection_keeps_never_created_members_but_refuses_pending_effects() { + let prepared = prepare(basic(), json!({"web":{}}), &ManagedValues::new()); + let config = configuration(&prepared, OWNER).unwrap(); + let original = + Receipt::preparing(&config, OWNER, "12345678-abcd-abcd-abcd-123456789abc").unwrap(); + assert!(original.require_recovery_settled().is_err()); + for phase in [ + Phase::FailedRetained, + Phase::StopIntent, + Phase::Stopped, + Phase::RemovalIntent, + ] { + let mut receipt = original.clone(); + receipt.phase = phase; + receipt.require_recovery_settled().unwrap(); + for settled in ["created", "started", "stopped", "removed"] { + let mut next = receipt.clone(); + let resource = next.resources.get_mut("container:web").unwrap(); + resource.id = Some("1".repeat(64)); + resource.phase = settled.into(); + next.require_recovery_settled().unwrap(); + } + for pending in [ + "create-intent", + "start-intent", + "uncertain", + "stop-intent", + "remove-intent", + ] { + let mut next = receipt.clone(); + let resource = next.resources.get_mut("container:web").unwrap(); + resource.id = Some("1".repeat(64)); + resource.phase = pending.into(); + assert!(next.require_recovery_settled().is_err(), "{pending}"); + } + let mut wire = serde_json::to_value(&receipt).unwrap(); + wire["failure"] = json!({"service":"web","observation":{"state":"exited","code":7}}); + let failed: Receipt = serde_json::from_value(wire).unwrap(); + failed.require_recovery_settled().unwrap(); + } + let mut removed = original; + removed.phase = Phase::Removed; + assert!(removed.require_recovery_settled().is_err()); + for resource in removed.resources.values_mut() { + resource.phase = "removed".into(); + } + removed.require_recovery_settled().unwrap(); +} fn two_bridges() -> Value { json!({"schema_version":1,"name":"fixture","networks":{"outbound":{"internal":false},"inside":{"internal":true}},"services":{ "db":{"image":image(),"networks":{"inside":{"aliases":["db-reader"]}}}, diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index dab5b1fd4..61f47515d 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -20,7 +20,7 @@ type ModelContract = { readonly alternativeWitnesses?: readonly ModelWitness[]; readonly additionalControls?: readonly { readonly name: string; - readonly module?: "TerminalReuse" | "MetadataAliasing"; + readonly module?: "TerminalReuse" | "MetadataAliasing" | "PreReady"; readonly negative: boolean; readonly states?: number; readonly invariant?: string; @@ -310,6 +310,41 @@ const contracts: readonly ModelContract[] = [ "pending = 0", ], additionalControls: [ + { + name: "pre-ready-positive", + module: "PreReady", + negative: false, + states: 160, + }, + { + name: "pre-ready-negative", + module: "PreReady", + negative: true, + invariant: "OnlySettledCleanup", + action: "Cleanup", + fields: ["unsafeCleanup = TRUE", "intent = TRUE", "removed = TRUE"], + }, + { + name: "pre-ready-release", + module: "PreReady", + negative: true, + invariant: "ReleaseRecorded", + action: "Release", + fields: ["unsafeRelease = TRUE", "release = FALSE", 'owner = "none"'], + }, + { + name: "pre-ready-replay", + module: "PreReady", + negative: true, + invariant: "NoReplay", + action: "StartAnother", + fields: [ + "starts = 2", + "intent = TRUE", + "complete = FALSE", + "files = FALSE", + ], + }, { name: "live-launcher-recovery", negative: true, diff --git a/src/backends/native-authored-hook-journal.ts b/src/backends/native-authored-hook-journal.ts index 08433bb10..a5432f636 100644 --- a/src/backends/native-authored-hook-journal.ts +++ b/src/backends/native-authored-hook-journal.ts @@ -638,13 +638,18 @@ async function recoverPhase( */ export async function retireNativeAuthoredRecoveredHooks( store: Store, - run: string + run: string, + /** Caller has already proved retained host processes stopped. Observe only; + * the normal retirement still requires their owner file to be retired first. */ + checkOnly = false ): Promise { const saved = await optionalCapture(ownerPath(store)); if (!saved) { return; } - await absent(join(store.root.path, `${run}.host-process-owner.json`)); + if (!checkOnly) { + await absent(join(store.root.path, `${run}.host-process-owner.json`)); + } const owner = parseOwner(JSON.parse(saved.text)); if ( owner.run !== run || @@ -666,6 +671,9 @@ export async function retireNativeAuthoredRecoveredHooks( for (const group of groups) { requireAbsentGroup(group); } + if (checkOnly) { + return; + } await unlink(ownerPath(store)); await synchronizeDirectories(store.held); } diff --git a/src/backends/native-authored-hook-stop.ts b/src/backends/native-authored-hook-stop.ts index e469311a5..d5de07b30 100644 --- a/src/backends/native-authored-hook-stop.ts +++ b/src/backends/native-authored-hook-stop.ts @@ -1,6 +1,7 @@ import { randomBytes, timingSafeEqual } from "node:crypto"; import { connect, createServer, type Socket } from "node:net"; import { isRecord } from "../lib/guards.ts"; +import { NativeAuthoredStopUnavailableError } from "./native-authored-stop-errors.ts"; const SHA = /^[a-f0-9]{64}$/; const RUN = /^[a-f0-9]{32}$/; @@ -165,15 +166,17 @@ export async function requestNativeHookStop(input: { }; opts.signal?.addEventListener("abort", fail, { once: true }); let text = ""; - socket.on("connect", () => + let connected = false; + socket.on("connect", () => { + connected = true; socket.end( `${JSON.stringify({ kind: "native-authored-stop", run: opts.run, token: opts.token, })}\n` - ) - ); + ); + }); socket.on("data", (bytes) => { try { text = appendAscii(text, bytes); @@ -181,7 +184,14 @@ export async function requestNativeHookStop(input: { fail(); } }); - socket.on("error", fail); + socket.on("error", (error) => { + if (!connected && "code" in error && error.code === "ECONNREFUSED") { + socket.destroy(); + reject(new NativeAuthoredStopUnavailableError()); + } else { + fail(); + } + }); socket.on("end", () => { try { const value: unknown = JSON.parse(text); diff --git a/src/backends/native-authored-host-processes.ts b/src/backends/native-authored-host-processes.ts index f67a5ae30..29ed25456 100644 --- a/src/backends/native-authored-host-processes.ts +++ b/src/backends/native-authored-host-processes.ts @@ -388,6 +388,8 @@ export async function retireNativeAuthoredHostProcesses(opts: { readonly scope: NativeAuthoredProjectRunScope; readonly run: string; readonly assertFresh: () => Promise; + /** Pre-Ready recovery may only observe an already stopped, absent group. */ + readonly checkOnly?: boolean; }): Promise { if (!RUN.test(opts.run)) { return refuse(); @@ -481,10 +483,15 @@ export async function retireNativeAuthoredHostProcesses(opts: { for (const process of savedEntry?.processes ?? []) { absentGroup(process.processGroupId ?? refuse()); } + } else if (opts.checkOnly) { + return refuse(); } else { await stopKnown({ scope: opts.scope, owner, savedEntry, check }); } await check(); + if (opts.checkOnly) { + return; + } await unlink(path); const held = await holdDirectory(root, false); try { diff --git a/src/backends/native-authored-project-recovery.ts b/src/backends/native-authored-project-recovery.ts index a5da66df0..d8e308ca7 100644 --- a/src/backends/native-authored-project-recovery.ts +++ b/src/backends/native-authored-project-recovery.ts @@ -66,7 +66,7 @@ type Identity = NativeAuthoredProjectRunSelection["identity"]; type IntentFields = { readonly kind: "native-authored-project-recovery"; readonly phase: Phase; - readonly ready: NativeAuthoredProjectRunSelection; + readonly ready: NativeAuthoredProjectRunSelection | null; readonly start: NativeAuthoredProjectStartSelection; readonly source: Identity; readonly admission: NativeComposeInterruptedLockSelection; @@ -78,28 +78,54 @@ type IntentFields = { readonly next_lease: NativeComposeInterruptedLockSelection | null; readonly next_mutation: NativeComposeInterruptedLockSelection | null; }; -type Intent = IntentFields & - ( - | { - readonly version: 1; - readonly live_stop?: never; - readonly live_stop_retiring?: never; - readonly live_stop_retired?: never; - } - | { - readonly version: 2; - readonly live_stop: NativeAuthoredLiveStopRecovery; - readonly live_stop_retiring: boolean; - readonly live_stop_retired: boolean; - } - ); +type IntentKind = + | { + readonly version: 1; + readonly live_stop?: never; + readonly live_stop_retiring?: never; + readonly live_stop_retired?: never; + } + | { + readonly version: 2; + readonly live_stop: NativeAuthoredLiveStopRecovery; + readonly live_stop_retiring: boolean; + readonly live_stop_retired: boolean; + } + | { + readonly version: 3; + readonly admission_held: boolean; + readonly admission_releasing: boolean; + }; +type Intent = IntentFields & IntentKind; +function selectedIntentKind(opts: { + readonly ready: NativeAuthoredProjectRunSelection | null; + readonly liveStop: NativeAuthoredLiveStopRecovery | null; + readonly held?: NativeComposeMutationLease; +}): IntentKind { + if (!opts.ready) { + return { + version: 3, + admission_held: opts.held !== undefined, + admission_releasing: false, + }; + } + if (opts.liveStop) { + return { + version: 2, + live_stop: opts.liveStop, + live_stop_retiring: false, + live_stop_retired: false, + }; + } + return { version: 1 }; +} type Store = Parameters< Parameters[1] >[0]; type Saved = { readonly record: Intent; readonly identity: Identity }; function refused(): never { throw new Error( - "Native frontend recovery is retained or changed; values omitted. No startup was replayed." + "Native frontend recovery is retained or changed; run hack ps to inspect the selected attempt. Values omitted. No startup was replayed." ); } function hash(text: string): string { @@ -130,18 +156,69 @@ function freeze(value: T): T { } return value; } +function intentKeys(version: unknown): string { + if (version === 3) { + return "admission,admission_held,admission_releasing,kind,lease,lease_releasing,mutation,mutation_releasing,native,next_lease,next_mutation,phase,ready,source,start,version"; + } + if (version === 2) { + return "admission,kind,lease,lease_releasing,live_stop,live_stop_retired,live_stop_retiring,mutation,mutation_releasing,native,next_lease,next_mutation,phase,ready,source,start,version"; + } + return "admission,kind,lease,lease_releasing,mutation,mutation_releasing,native,next_lease,next_mutation,phase,ready,source,start,version"; +} +function parseIntentKind( + value: Record, + admission: NativeComposeInterruptedLockSelection +): IntentKind { + if (value.version === 2) { + const stop = value.live_stop; + if ( + !(isRecord(stop) && keys(stop, "identity,original,settled")) || + typeof value.live_stop_retiring !== "boolean" || + typeof value.live_stop_retired !== "boolean" || + (value.live_stop_retired && !value.live_stop_retiring) + ) { + return refused(); + } + const original = parseNativeAuthoredProcessIncarnation(stop.original); + if ( + original.boot !== admission.owner.bootId || + original.parent !== admission.owner.pid || + original.uid !== admission.owner.uid + ) { + return refused(); + } + return { + version: 2 as const, + live_stop: { + identity: copyIdentity(stop.identity), + original, + settled: stop.settled === null ? null : copyIdentity(stop.settled), + }, + live_stop_retiring: value.live_stop_retiring, + live_stop_retired: value.live_stop_retired, + }; + } + if (value.version === 3) { + if ( + typeof value.admission_held !== "boolean" || + typeof value.admission_releasing !== "boolean" || + (value.admission_releasing && + (!value.admission_held || PHASES.indexOf(value.phase as Phase) < 8)) + ) { + return refused(); + } + return { + version: 3 as const, + admission_held: value.admission_held, + admission_releasing: value.admission_releasing, + }; + } + return { version: 1 }; +} function parseIntent(value: unknown): Intent { if ( - !( - isRecord(value) && - keys( - value, - value.version === 2 - ? "admission,kind,lease,lease_releasing,live_stop,live_stop_retired,live_stop_retiring,mutation,mutation_releasing,native,next_lease,next_mutation,phase,ready,source,start,version" - : "admission,kind,lease,lease_releasing,mutation,mutation_releasing,native,next_lease,next_mutation,phase,ready,source,start,version" - ) - ) || - (value.version !== 1 && value.version !== 2) || + !(isRecord(value) && keys(value, intentKeys(value.version))) || + (value.version !== 1 && value.version !== 2 && value.version !== 3) || value.kind !== "native-authored-project-recovery" || typeof value.phase !== "string" || !PHASES.includes(value.phase as Phase) || @@ -150,7 +227,10 @@ function parseIntent(value: unknown): Intent { ) { return refused(); } - const ready = parseNativeAuthoredProjectRunSelection(value.ready); + const ready = + value.version === 3 && value.ready === null + ? null + : parseNativeAuthoredProjectRunSelection(value.ready); const start = parseNativeAuthoredProjectStartSelection(value.start); const admission = parseNativeComposeInterruptedLockSelection(value.admission); const lease = parseNativeComposeInterruptedLockSelection(value.lease); @@ -164,8 +244,10 @@ function parseIntent(value: unknown): Intent { ? null : parseNativeComposeInterruptedLockSelection(value.next_mutation); if ( - JSON.stringify(start.record.review) !== - JSON.stringify(ready.record.receipt.review) || + (value.version === 3 && ready !== null) || + (ready && + JSON.stringify(start.record.review) !== + JSON.stringify(ready.record.receipt.review)) || lease.owner.bootId !== admission.owner.bootId || mutation.owner.bootId !== admission.owner.bootId || (nextLease !== null && nextLease.owner.bootId !== admission.owner.bootId) || @@ -174,39 +256,7 @@ function parseIntent(value: unknown): Intent { ) { return refused(); } - const extra = - value.version === 2 - ? (() => { - const stop = value.live_stop; - if ( - !(isRecord(stop) && keys(stop, "identity,original,settled")) || - typeof value.live_stop_retiring !== "boolean" || - typeof value.live_stop_retired !== "boolean" || - (value.live_stop_retired && !value.live_stop_retiring) - ) { - return refused(); - } - const original = parseNativeAuthoredProcessIncarnation(stop.original); - if ( - original.boot !== admission.owner.bootId || - original.parent !== admission.owner.pid || - original.uid !== admission.owner.uid - ) { - return refused(); - } - return { - version: 2 as const, - live_stop: { - identity: copyIdentity(stop.identity), - original, - settled: - stop.settled === null ? null : copyIdentity(stop.settled), - }, - live_stop_retiring: value.live_stop_retiring, - live_stop_retired: value.live_stop_retired, - }; - })() - : { version: 1 as const }; + const extra = parseIntentKind(value, admission); return freeze({ ...extra, kind: "native-authored-project-recovery", @@ -217,7 +267,9 @@ function parseIntent(value: unknown): Intent { admission, native: parseNativeAuthoredRecoverySelection({ value: value.native, - admitted: ready.record.receipt, + ...(ready + ? { admitted: ready.record.receipt } + : { start: start.record.review }), }), lease, mutation, @@ -489,16 +541,23 @@ async function stopBindings( async function bindings( store: Store, record: Intent, - remaining: () => number + remaining: () => number, + held?: NativeComposeMutationLease ): Promise { await stopBindings(store, record, remaining); + if (record.version === 3) { + if (!(await absent(`${store.liveStop}.settled`))) { + return refused(); + } + await store.assertPreReadyHosts(record.native.run); + } const phase = rank(record); for (const [path, expected, retirement] of [ - [store.ready, record.ready.identity, 2], + [store.ready, record.ready?.identity ?? null, 2], [store.start, record.start.identity, 4], [store.sourcePath(record.native.run), record.source, 6], ] as const) { - if (phase > retirement) { + if (phase > retirement || expected === null) { if (!(await absent(path))) { return refused(); } @@ -506,7 +565,25 @@ async function bindings( await expectedFile(path, expected, phase === retirement); } } - if (phase < 8) { + await admissionBinding(store, record, held); +} + +async function admissionBinding( + store: Store, + record: Intent, + held?: NativeComposeMutationLease +): Promise { + const phase = rank(record); + if (held) { + await held.assertHeld(); + if ( + record.version !== 3 || + !record.admission_held || + JSON.stringify(held.selection) !== JSON.stringify(record.admission) + ) { + return refused(); + } + } else if (phase < 8) { const selected = await lock( store, store.admission, @@ -515,8 +592,19 @@ async function bindings( if (JSON.stringify(selected) !== JSON.stringify(record.admission)) { return refused(); } - } else if (phase === 9 && !(await absent(store.admission))) { - return refused(); + } else if (phase >= 8) { + const missing = await absent(store.admission); + if (phase === 9 && !missing) { + return refused(); + } + if ( + missing && + record.version === 3 && + record.admission_held && + !record.admission_releasing + ) { + return refused(); + } } } @@ -533,7 +621,7 @@ async function retireFiles( ): Promise { const selected = retirement.record(); for (const [path, expected, phase] of [ - [store.ready, selected.ready.identity, 2], + [store.ready, selected.ready?.identity ?? null, 2], [store.start, selected.start.identity, 4], [store.sourcePath(selected.native.run), selected.source, 6], ] as const) { @@ -548,7 +636,7 @@ async function retireFiles( } await retirement.inspect(); await retirement.check(); - if (await expectedFile(path, expected, true)) { + if (expected !== null && (await expectedFile(path, expected, true))) { retirement.active(); await unlink(path); await synchronizeDirectories(store.held); @@ -601,7 +689,8 @@ async function retireLiveStop( async function retireAdmission( store: Store, retirement: Retirement, - lease: NativeComposeMutationLease + lease: NativeComposeMutationLease, + held?: NativeComposeMutationLease ): Promise { if (rank(retirement.record()) < 8) { await retirement.commit({ @@ -614,7 +703,38 @@ async function retireAdmission( } await retirement.inspect(); await retirement.check(); + if (held) { + const selected = retirement.record(); + if ( + selected.version !== 3 || + !selected.admission_held || + JSON.stringify(selected.admission) !== JSON.stringify(held.selection) + ) { + return refused(); + } + await held.assertHeld(); + await retirement.commit({ ...selected, admission_releasing: true }); + return; + } if (!(await absent(store.admission))) { + const selected = retirement.record(); + if ( + selected.version === 3 && + selected.admission_held && + !selected.admission_releasing + ) { + // A crashed fresh holder is now an interrupted selection. Prove that + // exact dead owner before recording permission to retire its admission. + const interrupted = await lock( + store, + store.admission, + `${store.admission}.verification` + ).selectInterruptedLock(); + if (JSON.stringify(interrupted) !== JSON.stringify(selected.admission)) { + return refused(); + } + await retirement.commit({ ...selected, admission_releasing: true }); + } await lock( store, store.admission, @@ -634,14 +754,116 @@ async function retireAdmission( /** Explicit stored-generation cleanup only. No compilation, value acquisition, * serve, allocation or automatic startup takeover exists in this owner. */ -export async function recoverNativeAuthoredProject(opts: { +type RecoveryOptions = { readonly scope: NativeAuthoredProjectRunScope; readonly runtime: NativeRuntimeSelection; readonly timeoutMs: number; readonly signal?: AbortSignal; /** Exact public native-request seam for isolated owner tests. */ readonly request?: typeof invokeNativeRuntime; -}): Promise { +}; +async function selectInitialIntent(opts: { + readonly store: Store; + readonly held?: NativeComposeMutationLease; + readonly runtime: NativeRuntimeSelection; + readonly remaining: () => number; + readonly request: typeof invokeNativeRuntime; + readonly signal?: AbortSignal; + readonly lease: NativeComposeMutationLease; + readonly mutationLease: NativeComposeMutationLease; +}): Promise { + const { + store, + held, + runtime, + remaining, + request, + signal, + lease, + mutationLease, + } = opts; + const ready = (await absent(store.ready)) ? null : await store.readReady(); + const start = await store.readStart(); + if (held && ready) { + return refused(); + } + const run = start.record.review.provenance.run; + const source = await store.readSource(run); + const admission = + held?.selection ?? + (await lock( + store, + store.admission, + `${store.admission}.verification` + ).selectInterruptedLock()); + const liveStop = ready + ? await selectLiveStop({ + store, + ready, + admission, + runtime, + remaining, + }) + : null; + if (!ready) { + if ( + !( + (await absent(store.liveStop)) && + (await absent(`${store.liveStop}.settled`)) + ) + ) { + return refused(); + } + await store.assertPreReadyHosts(run); + } + const native = parseNativeAuthoredRecoverySelection({ + value: await request({ + runtime, + cwd: store.projectRoot, + args: [ + "graph", + "native", + "recovery-selection", + "--run-id", + run, + "--json", + ], + timeoutMs: remaining(), + signal, + boundNativeAuthoredRecoveryDrain: "selection", + }), + ...(ready + ? { admitted: ready.record.receipt } + : { start: start.record.review }), + }); + remaining(); + return { + ...selectedIntentKind({ ready, liveStop, held }), + kind: "native-authored-project-recovery", + phase: "prepared", + ready, + start, + source, + admission, + native, + lease: lease.selection, + mutation: mutationLease.selection, + lease_releasing: false, + mutation_releasing: false, + next_lease: null, + next_mutation: null, + }; +} + +export async function recoverNativeAuthoredProject( + opts: RecoveryOptions +): Promise { + return await recoverUsing(opts); +} +async function recoverUsing( + opts: RecoveryOptions, + held?: NativeComposeMutationLease +): Promise { const runtime = { ...opts.runtime }; const timeoutMs = opts.timeoutMs; const signal = opts.signal; @@ -667,6 +889,42 @@ export async function recoverNativeAuthoredProject(opts: { opts.scope, async (store) => { let saved = await readIntent(store); + if ( + !(saved || held) && + (await absent(store.ready)) && + (await absent(store.admission)) + ) { + // A fresh recovery admission uses the established verification guard; + // the ordinary guard is the recovery lock we are about to hold. + const admission = lock( + store, + store.admission, + `${store.admission}.verification` + ); + await admission.withFreshRecoveryLock( + { + release: async (lease) => { + const current = await readIntent(store); + if ( + current && + (current.record.version !== 3 || + !current.record.admission_held || + !current.record.admission_releasing || + JSON.stringify(current.record.admission) !== + JSON.stringify(lease.selection)) + ) { + // Keep the exact lease if its durable release is unconfirmed. + return refused(); + } + }, + }, + async (lease) => + await recoverUsing({ ...opts, timeoutMs: remaining() }, lease) + ); + // The release marker permits absence only after the actual held lease + // has gone. Resume the same intent to commit completion under its locks. + return await recoverUsing({ ...opts, timeoutMs: remaining() }); + } if (saved?.record.version === 1 && !(await absent(store.liveStop))) { return refused(); } @@ -712,6 +970,7 @@ export async function recoverNativeAuthoredProject(opts: { remaining(); await check(); await mutationLease.assertHeld(); + await held?.assertHeld(); if (saved?.record.version === 2) { await assertNativeAuthoredProcessRuntime( saved.record.live_stop.original, @@ -741,66 +1000,17 @@ export async function recoverNativeAuthoredProject(opts: { mutationLease ); } else { - const ready = await store.readReady(); - const start = await store.readStart(); - const source = await store.readSource( - ready.record.receipt.review.provenance.run - ); - const admission = await lock( - store, - store.admission, - `${store.admission}.verification` - ).selectInterruptedLock(); - const liveStop = await selectLiveStop({ - store, - ready, - admission, - runtime, - remaining, - }); - const native = parseNativeAuthoredRecoverySelection({ - value: await request({ + await commit( + await selectInitialIntent({ + store, + held, runtime, - cwd: store.projectRoot, - args: [ - "graph", - "native", - "recovery-selection", - "--run-id", - ready.record.receipt.review.provenance.run, - "--json", - ], - timeoutMs: remaining(), + remaining, + request, signal, - boundNativeAuthoredRecoveryDrain: "selection", + lease, + mutationLease, }), - admitted: ready.record.receipt, - }); - remaining(); - await commit( - { - ...(liveStop - ? { - version: 2 as const, - live_stop: liveStop, - live_stop_retiring: false, - live_stop_retired: false, - } - : { version: 1 as const }), - kind: "native-authored-project-recovery", - phase: "prepared", - ready, - start, - source, - admission, - native, - lease: lease.selection, - mutation: mutationLease.selection, - lease_releasing: false, - mutation_releasing: false, - next_lease: null, - next_mutation: null, - }, mutationLease ); } @@ -808,7 +1018,7 @@ export async function recoverNativeAuthoredProject(opts: { return refused(); } await both(); - await bindings(store, saved.record, remaining); + await bindings(store, saved.record, remaining, held); await both(); const selected = saved.record.native; if (saved.record.phase === "prepared") { @@ -835,7 +1045,7 @@ export async function recoverNativeAuthoredProject(opts: { }), }); await both(); - await bindings(store, saved.record, remaining); + await bindings(store, saved.record, remaining, held); await both(); await commit( { ...saved.record, phase: "native-removed" }, @@ -871,7 +1081,7 @@ export async function recoverNativeAuthoredProject(opts: { return refused(); } await both(); - await bindings(store, record(), remaining); + await bindings(store, record(), remaining, held); await both(); return snapshot.receipt; }; @@ -891,9 +1101,9 @@ export async function recoverNativeAuthoredProject(opts: { await store.retireHooks(selected.run); await retireLiveStop(store, retirement, remaining); await retireFiles(store, retirement); - await retireAdmission(store, retirement, lease); + await retireAdmission(store, retirement, lease, held); const removed = await inspect(); - await bindings(store, saved.record, remaining); + await bindings(store, saved.record, remaining, held); await both(); return { version: 1, diff --git a/src/backends/native-authored-project-run.ts b/src/backends/native-authored-project-run.ts index 252b14ddb..5b71ba786 100644 --- a/src/backends/native-authored-project-run.ts +++ b/src/backends/native-authored-project-run.ts @@ -45,6 +45,7 @@ import { requestNativeAuthoredLiveStop, } from "./native-authored-live-stop.ts"; import type { NativeAuthoredProcessIncarnation } from "./native-authored-process-incarnation.ts"; +import { NativeAuthoredStopUnavailableError } from "./native-authored-stop-errors.ts"; import { NATIVE_GRAPH_UNRESERVED, NativeRuntimeRequestError, @@ -505,6 +506,7 @@ export async function withNativeAuthoredProjectRecoveryStorage( readonly readStart: () => Promise; readonly sourcePath: (run: string) => string; readonly retireHooks: (run: string) => Promise; + readonly assertPreReadyHosts: (run: string) => Promise; readonly readSource: (run: string) => Promise<{ readonly dev: number; readonly ino: number; @@ -538,6 +540,26 @@ export async function withNativeAuthoredProjectRecoveryStorage( readReady: () => read(store), readStart: () => readRecord(store, store.startFile, started), sourcePath, + assertPreReadyHosts: async (run) => { + await retireNativeAuthoredHostProcesses({ + scope: opts, + run, + assertFresh: store.check, + checkOnly: true, + }); + await retireNativeAuthoredRecoveredHooks( + { + root: store.root, + held: store.held, + key: digest(JSON.stringify(opts.branch)), + projectRoot: store.identity.projectRoot, + branch: store.identity.branch, + check: store.check, + }, + run, + true + ); + }, retireHooks: async (run) => { await retireNativeAuthoredHostProcesses({ scope: opts, @@ -749,6 +771,8 @@ export async function withNativeAuthoredProjectStatus( action: (selected: { readonly ready: NativeAuthoredProjectRunSelection | null; readonly pending: boolean; + /** Advisory routing only; actual unreserved retirement takes fresh admission. */ + readonly unreservedCandidate: boolean; readonly assertFresh: () => Promise; }) => Promise ): Promise { @@ -806,9 +830,30 @@ export async function withNativeAuthoredProjectStatus( await assertScope(); }; await assertFresh(); + let unreservedCandidate = !ready && start !== null; + if (unreservedCandidate && store) { + for (const path of [ + store.admissionPath, + liveStopPath(store), + join( + store.root.path, + `${digest(JSON.stringify(scope.branch))}.hooks.json` + ), + ]) { + try { + await lstat(path); + unreservedCandidate = false; + } catch (error) { + if (!hasCode(error, "ENOENT")) { + throw error; + } + } + } + } const result = await action({ ready, pending: !ready && start !== null, + unreservedCandidate, assertFresh, }); await assertFresh(); @@ -1247,6 +1292,17 @@ async function assertStopFiles( } } +async function requireStopPath(path: string): Promise { + try { + await lstat(path); + } catch (error) { + if (hasCode(error, "ENOENT")) { + throw new NativeAuthoredStopUnavailableError(); + } + throw error; + } +} + export async function stopNativeAuthoredProject(input: { readonly scope: NativeAuthoredProjectRunScope; readonly timeoutMs: number; @@ -1284,6 +1340,9 @@ export async function stopNativeAuthoredProject(input: { } : undefined; await withNativeAuthoredProjectRecoveryStorage(opts.scope, async (store) => { + for (const path of [store.ready, store.start]) { + await requireStopPath(path); + } const ready = await store.readReady(); const start = await store.readStart(); const run = ready.record.receipt.review.provenance.run; @@ -1331,6 +1390,7 @@ export async function stopNativeAuthoredProject(input: { return; } const path = join(store.root.path, `${run}.hook-stop.json`); + await requireStopPath(path); const current = await readPrivate(path, 8192); const value: unknown = JSON.parse(current.text); if ( diff --git a/src/backends/native-authored-recovery-protocol.ts b/src/backends/native-authored-recovery-protocol.ts index 91e1eff76..926b88bba 100644 --- a/src/backends/native-authored-recovery-protocol.ts +++ b/src/backends/native-authored-recovery-protocol.ts @@ -1,8 +1,10 @@ import { isRecord } from "../lib/guards.ts"; import { type NativeAuthoredReceipt, + type NativeAuthoredReview, nativeAuthoredReceiptBinding, parseNativeAuthoredReceipt, + parseNativeAuthoredReview, } from "./native-authored-graph-protocol.ts"; const HEX64 = /^[a-f0-9]{64}$/; @@ -45,7 +47,7 @@ function fields( function hash(value: unknown): value is string { return typeof value === "string" && HEX64.test(value); } -function ready(value: unknown): NativeAuthoredReceipt { +function settled(value: unknown): NativeAuthoredReceipt { const receipt = parseNativeAuthoredReceipt(value); if ( !( @@ -58,6 +60,29 @@ function ready(value: unknown): NativeAuthoredReceipt { (reference) => reference.state.status === "enrolled" )) ) || + receipt.phase === "preparing" || + (receipt.phase === "ready-observed" && receipt.failure !== undefined) || + Object.values(receipt.resources).some((resource) => { + if (receipt.phase === "removed" && resource.phase !== "removed") { + return true; + } + if (resource.id === null) { + return resource.phase !== "reserved" && resource.phase !== "removed"; + } + return !( + resource.kind === "container" + ? ["created", "started", "stopped", "removed"] + : ["created", "remove-intent", "removed"] + ).includes(resource.phase); + }) + ) { + return refused(); + } + return receipt; +} +function ready(value: unknown): NativeAuthoredReceipt { + const receipt = settled(value); + if ( receipt.phase !== "ready-observed" || receipt.failure !== undefined || Object.values(receipt.resources).some((resource) => resource.id === null) @@ -69,11 +94,18 @@ function ready(value: unknown): NativeAuthoredReceipt { /** Raw selectors belong to Rust's secure reader. This copied, closed envelope * binds the original durable frontend Ready membership; it grants no effects. */ -export function parseNativeAuthoredRecoverySelection(opts: { - readonly value: unknown; - readonly admitted: NativeAuthoredReceipt; -}): NativeAuthoredRecoverySelection { - const admitted = ready(opts.admitted); +export function parseNativeAuthoredRecoverySelection( + opts: { + readonly value: unknown; + } & ( + | { readonly admitted: NativeAuthoredReceipt } + | { readonly start: NativeAuthoredReview } + ) +): NativeAuthoredRecoverySelection { + const admitted = "admitted" in opts ? ready(opts.admitted) : undefined; + const review = + admitted?.review ?? + ("start" in opts ? parseNativeAuthoredReview(opts.start) : refused()); const value = opts.value; const qualifier = isRecord(value) && Object.hasOwn(value, "host_boot_uuid") @@ -91,16 +123,18 @@ export function parseNativeAuthoredRecoverySelection(opts: { ]) || (value.version !== 1 && value.version !== 2) || value.kind !== "native-graph-recovery-selection" || - value.run !== admitted.review.provenance.run || + value.run !== review.provenance.run || !hash(value.receipt_sha256) || !hash(value.owner_sha256) ) { return refused(); } - const receipt = ready(value.receipt); + const receipt = admitted ? ready(value.receipt) : settled(value.receipt); if ( - nativeAuthoredReceiptBinding(receipt) !== - nativeAuthoredReceiptBinding(admitted) + JSON.stringify(receipt.review) !== JSON.stringify(review) || + (admitted && + nativeAuthoredReceiptBinding(receipt) !== + nativeAuthoredReceiptBinding(admitted)) ) { return refused(); } @@ -140,7 +174,7 @@ export function parseNativeAuthoredRecoveryResult(opts: { }): NativeAuthoredRecoveryResult { const expected = parseNativeAuthoredRecoverySelection({ value: opts.expected, - admitted: opts.expected.receipt, + start: opts.expected.receipt.review, }); const value = opts.value; if ( diff --git a/src/backends/native-authored-stop-errors.ts b/src/backends/native-authored-stop-errors.ts new file mode 100644 index 000000000..75004b1d4 --- /dev/null +++ b/src/backends/native-authored-stop-errors.ts @@ -0,0 +1,9 @@ +/** No stop request was dispatched. This is a routing hint, never recovery proof. */ +export class NativeAuthoredStopUnavailableError extends Error { + constructor() { + super( + "Native stop owner is unavailable; values omitted. No stop request was dispatched." + ); + this.name = "NativeAuthoredStopUnavailableError"; + } +} diff --git a/src/backends/native-authored-unreserved-start.ts b/src/backends/native-authored-unreserved-start.ts index 51e71ac31..6468ff4d7 100644 --- a/src/backends/native-authored-unreserved-start.ts +++ b/src/backends/native-authored-unreserved-start.ts @@ -15,6 +15,16 @@ function refused(): never { ); } +/** A successful inspection proves a journal exists; cleanup still needs its own proof. */ +export class NativeAuthoredReservedStartError extends Error { + constructor() { + super( + "Native startup has reserved runtime state; stored-generation recovery is required. Values omitted." + ); + this.name = "NativeAuthoredReservedStartError"; + } +} + /** * Read-only selection for explicit recovery: true only when a startup intent is * retained without a Ready mapping or a frontend recovery intent. Any refusal @@ -27,7 +37,7 @@ export async function nativeAuthoredUnreservedStartSelected( try { return await withNativeAuthoredProjectStatus( { ...scope }, - async (saved) => await Promise.resolve(saved.pending) + async (saved) => await Promise.resolve(saved.unreservedCandidate) ); } catch { return false; @@ -102,7 +112,8 @@ export async function retireNativeAuthoredUnreservedStart(opts: { } if (refusal === undefined) { // A journal exists; this attempt belongs to receipt-owned recovery. - return refused(); + remaining(); + throw new NativeAuthoredReservedStartError(); } remaining(); await admission.assertHeld(); diff --git a/src/lib/native-authored-command.ts b/src/lib/native-authored-command.ts index cc7d5fa28..6e3925f47 100644 --- a/src/lib/native-authored-command.ts +++ b/src/lib/native-authored-command.ts @@ -29,7 +29,9 @@ import { serveNativeAuthoredProject, } from "../backends/native-authored-project-start.ts"; import { NativeAuthoredProjectNotStartedError } from "../backends/native-authored-runtime-binary.ts"; +import { NativeAuthoredStopUnavailableError } from "../backends/native-authored-stop-errors.ts"; import { + NativeAuthoredReservedStartError, nativeAuthoredUnreservedStartSelected, retireNativeAuthoredUnreservedStart, } from "../backends/native-authored-unreserved-start.ts"; @@ -403,7 +405,7 @@ const STATUS_NOTES: Readonly< > = { not_started: "Native project has not been started.", pending: - "Native project startup is pending: a start reserved its intent but has not reached Ready (it is still starting, or it failed and its startup evidence is retained). No current service status is confirmed. If no startup is running, `hack down --recover` retires an attempt that never reserved runtime state.", + "Native project startup is pending: a start reserved its intent but has not reached Ready (it is still starting, or it failed and its startup evidence is retained). No current service status is confirmed. Stop a live foreground start there; after it exits, `hack down` attempts exact-proof recovery of settled runtime state or retirement of an unreserved attempt.", runtime_not_running: "Native runtime is not running; no services are running.", runtime_busy: @@ -449,10 +451,15 @@ function commandFailure(options: { }); } if (options.mode === "recovery" || options.mode === "stop") { + if (options.signal.aborted) { + return options.signal.reason === 143 ? 143 : 130; + } throw new HackCliError({ code: "E_LIFECYCLE_FAILED", message: - "Native project recovery is incomplete or its ownership changed; selected state is retained. Values omitted.", + options.mode === "stop" + ? "Native project stop is unconfirmed or its ownership changed; selected state is retained. Run hack ps to inspect it. Values omitted. No request was replayed." + : "Native project recovery is incomplete or its ownership changed; selected state is retained. Run hack ps to inspect it. Values omitted.", }); } if (options.mode === "restart") { @@ -527,21 +534,30 @@ async function runRecovery(input: LifecycleInput): Promise { input.seams.unreservedSelected ?? nativeAuthoredUnreservedStartSelected )(input.scope) ) { - // A startup intent without a Ready generation: only a provably unreserved - // attempt is retired; every other retained attempt refuses unchanged. - await (input.seams.retireUnreserved ?? retireNativeAuthoredUnreservedStart)( - { + try { + await ( + input.seams.retireUnreserved ?? retireNativeAuthoredUnreservedStart + )({ runtime: input.runtime, scope: input.scope, timeoutMs: input.startupTimeoutMs, signal: input.signal, + }); + logger.info({ + message: + "Retired an incomplete native startup that never reserved runtime state; nothing else was changed.", + }); + return 0; + } catch (error) { + if ( + input.signal.aborted || + !(error instanceof NativeAuthoredReservedStartError) + ) { + throw error; } - ); - logger.info({ - message: - "Retired an incomplete native startup that never reserved runtime state; nothing else was changed.", - }); - return 0; + // A successful inspection found a journal. Only independent settled + // recovery can authorize its cleanup; other refusals keep their outcome. + } } if ( await ( @@ -569,11 +585,21 @@ async function runRecovery(input: LifecycleInput): Promise { } async function runStop(input: LifecycleInput): Promise { - await (input.seams.stop ?? stopNativeAuthoredProject)({ - scope: input.scope, - timeoutMs: input.startupTimeoutMs, - signal: input.signal, - }); + try { + await (input.seams.stop ?? stopNativeAuthoredProject)({ + scope: input.scope, + timeoutMs: input.startupTimeoutMs, + signal: input.signal, + }); + } catch (error) { + if ( + input.signal.aborted || + !(error instanceof NativeAuthoredStopUnavailableError) + ) { + throw error; + } + return await runRecovery(input); + } logger.info({ message: "Selected native project stopped; persistent data is retained.", }); @@ -737,10 +763,12 @@ async function runAuthoredLifecycle( } const controller = new AbortController(); const forceController = new AbortController(); - const cancel = () => - (controller.signal.aborted ? forceController : controller).abort(); - process.on("SIGINT", cancel); - process.on("SIGTERM", cancel); + const cancel = (code: number) => + (controller.signal.aborted ? forceController : controller).abort(code); + const interrupt = () => cancel(130); + const terminate = () => cancel(143); + process.on("SIGINT", interrupt); + process.on("SIGTERM", terminate); try { const root = await realpath(input.projectRoot); return await action({ @@ -758,8 +786,8 @@ async function runAuthoredLifecycle( } catch (error: unknown) { return commandFailure({ error, mode, signal: controller.signal }); } finally { - process.off("SIGINT", cancel); - process.off("SIGTERM", cancel); + process.off("SIGINT", interrupt); + process.off("SIGTERM", terminate); } } diff --git a/tests/models/tla/README.md b/tests/models/tla/README.md index e39d74b8a..2224ebe79 100644 --- a/tests/models/tla/README.md +++ b/tests/models/tla/README.md @@ -56,6 +56,11 @@ No credentials or running VM are needed. ## Native foreground recovery lease takeover +The companion [PreReady coverage](native-frontend-recovery/COVERAGE.md) checks +settled pre-Ready selection, durable admission release and startup exclusion +during incomplete recovery: 160 states and three intentional negative controls. +It adds no authority for pending effects or live/uncertain host processes. + `native-frontend-recovery/Recovery.tla` models the explicit saved-run frontend recovery lease protocol. The positive configuration exhausts **602 distinct states** (695 generated, maximum depth 22). It bounds one lease to two new candidates, diff --git a/tests/models/tla/native-frontend-recovery/COVERAGE.md b/tests/models/tla/native-frontend-recovery/COVERAGE.md new file mode 100644 index 000000000..4d85ee151 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/COVERAGE.md @@ -0,0 +1,41 @@ +# Authored recovery before frontend Ready + +[PreReady.tla](PreReady.tla) extends the existing frontend recovery coverage with +the absent-Ready selection and fresh admission release. It does not replace +`Recovery.tla`, which checks durable recovery-lease publication and takeover. + +The positive configuration exhausts **160 distinct states** (184 generated, +depth 10), with five original classes (Ready, failed, cleanup, Preparing and +pending), two initial admission states, two attempts, one crash and one possible +foreign-owner replacement. TLC 1.7.4, Temurin Java 17, two workers and a 512 MiB +heap are the pinned runner configuration. Three controls intentionally fail: + +| Control | First violating action and invariant | Implementation regression | +| --- | --- | --- | +| `pre-ready-negative` | `Cleanup` violates `OnlySettledCleanup` when the original is unsettled | Runtime settled selection and frontend protocol reject Preparing and pending effects | +| `pre-ready-release` | `Release` violates `ReleaseRecorded` without a durable release marker | Fresh admission crash test; missing marker refuses, failed release keeps exact directory/owner inodes | +| `pre-ready-replay` | `StartAnother` violates `NoReplay` during incomplete recovery | Admission refuses during recovery and between actual release and completion | + +| Model action / guard | Source mapping | +| --- | --- | +| `Acquire`, `Commit`, `Settled` | `recoverUsing` chooses interrupted admission or holds fresh admission before recovery/mutation. Frontend intent v3 records absent Ready, start/source and full admission selection; runtime intent v2 pins the original receipt. `require_recovery_settled` and the frontend `settled` parser reject pending effects. | +| `Cleanup` | The existing native cleanup owner rechecks exact dead publisher, boot/incarnation, inventory and binding before stopping/removing. Original phase and per-resource ranks cannot regress; reserved resources never become created. | +| `RetireFiles` | `retireFiles`/`retireHooks` run only after authenticated Removed and null observations. Pre-Ready host checks require existing settlement and absent groups; they never signal. | +| `ArmRelease`, `Release`, `Complete` | `retireAdmission` saves `admission_releasing`; `withFreshRecoveryLock` releases only the exact recorded lease. A second pass requires absence. Incomplete v3 intents continue blocking ordinary admission in the gap. | +| `Crash`, `Resume`, `ReplaceOwner` | The child-process regression crashes before/after the release marker and actual release, then resumes only the recorded dead selection; release-callback failure preserves exact inodes. A changed owner grants no authority. | + +Committed intent writes and native cleanup are atomic abstractions here. +Resource-state predicates, exact filesystem/process/guest identity, host +settlement and the nested recovery/mutation leases are assumptions checked by +source, the existing lease model and implementation tests. The diagram shows +the implementation decision flow, not an additional checked machine. +The model has no fairness or liveness claim and does not prove fsync durability, +unbounded retries, physical VM behavior, source/topology recovery, old-reader +compatibility or cleanup after an unknown pending effect. Legacy frontend v1/v2 +and runtime v1 remain covered by regression tests with their original Ready +rules. Older bundles refuse the new unknown versions. The real-Mac plain-down +run after authored pre-Ready failure remains separate acceptance evidence. + +![Authored recovery selection](flow.svg) + +[Diagram source](flow.mmd), rendered with Pele 1.9.0, links/images disabled. diff --git a/tests/models/tla/native-frontend-recovery/PreReady.tla b/tests/models/tla/native-frontend-recovery/PreReady.tla new file mode 100644 index 000000000..f1813deae --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/PreReady.tla @@ -0,0 +1,82 @@ +----------------------------- MODULE PreReady ----------------------------- +EXTENDS Naturals, TLC +CONSTANTS EnforceSettled, EnforceRelease, EnforceIntent +VARIABLES origin, owner, savedOwner, intent, release, removed, files, complete, + active, alive, attempts, crashes, unsafeCleanup, unsafeRelease, starts +publication == <> +progress == <> +process == <> +unsafe == <> +vars == <> +Settled == origin \in {"ready", "failed", "cleanup"} +Init == /\ origin \in {"ready", "failed", "cleanup", "preparing", "pending"} + /\ owner \in {"none", "original"} /\ savedOwner = "none" + /\ intent = FALSE /\ release = FALSE /\ removed = FALSE + /\ files = TRUE /\ complete = FALSE /\ active = FALSE /\ alive = FALSE + /\ attempts = 0 /\ crashes = 0 /\ unsafeCleanup = FALSE + /\ unsafeRelease = FALSE /\ starts = 1 +Acquire == + /\ ~active /\ ~alive /\ ~intent /\ attempts < 2 + /\ owner \in {"none", "original", "held"} + /\ owner' = IF owner = "none" THEN "held" ELSE owner + /\ active' = TRUE /\ alive' = TRUE /\ attempts' = attempts + 1 + /\ UNCHANGED <> +Commit == + /\ active /\ alive /\ ~intent /\ (EnforceSettled => Settled) + /\ intent' = TRUE /\ savedOwner' = owner + /\ UNCHANGED <> +Cleanup == + /\ active /\ alive /\ intent /\ ~removed /\ owner = savedOwner + /\ removed' = TRUE /\ unsafeCleanup' = (~Settled \/ unsafeCleanup) + /\ UNCHANGED <> +RetireFiles == + /\ active /\ alive /\ intent /\ removed /\ files /\ owner = savedOwner + /\ files' = FALSE + /\ UNCHANGED <> +ArmRelease == + /\ active /\ alive /\ intent /\ ~files /\ owner = savedOwner /\ ~release + /\ release' = TRUE + /\ UNCHANGED <> +Release == + /\ active /\ alive /\ intent /\ ~files /\ owner = savedOwner + /\ (EnforceRelease => release) + /\ owner' = "none" /\ active' = FALSE /\ alive' = FALSE + /\ unsafeRelease' = (~release \/ unsafeRelease) + /\ UNCHANGED <> +Crash == + /\ active /\ alive /\ crashes = 0 + /\ active' = FALSE /\ alive' = FALSE /\ crashes' = 1 + /\ UNCHANGED <> +Resume == + /\ ~active /\ ~alive /\ intent /\ ~complete /\ attempts < 2 + /\ owner = savedOwner /\ owner # "none" + /\ active' = TRUE /\ alive' = TRUE /\ attempts' = attempts + 1 + /\ UNCHANGED <> +ReplaceOwner == + /\ ~active /\ ~alive /\ intent /\ ~complete /\ owner = savedOwner + /\ owner' = "foreign" + /\ UNCHANGED <> +Complete == + /\ intent /\ removed /\ ~files /\ ~complete /\ release /\ owner = "none" + /\ complete' = TRUE + /\ UNCHANGED <> +StartAnother == + /\ ~files /\ ~complete /\ owner = "none" /\ starts = 1 + /\ (EnforceIntent => ~intent) + /\ starts' = 2 + /\ UNCHANGED <> +Next == Acquire \/ Commit \/ Cleanup \/ RetireFiles \/ ArmRelease \/ Release + \/ Crash \/ Resume \/ ReplaceOwner \/ Complete \/ StartAnother +Spec == Init /\ [][Next]_vars +TypeOK == /\ origin \in {"ready", "failed", "cleanup", "preparing", "pending"} + /\ owner \in {"none", "original", "held", "foreign"} + /\ savedOwner \in {"none", "original", "held"} + /\ intent \in BOOLEAN /\ release \in BOOLEAN /\ removed \in BOOLEAN + /\ files \in BOOLEAN /\ complete \in BOOLEAN /\ active \in BOOLEAN + /\ alive \in BOOLEAN /\ attempts \in 0..2 /\ crashes \in 0..1 + /\ unsafeCleanup \in BOOLEAN /\ unsafeRelease \in BOOLEAN /\ starts \in 1..2 +OnlySettledCleanup == ~unsafeCleanup +ReleaseRecorded == ~unsafeRelease +NoReplay == starts = 1 +RetirementHasProof == (~files => removed) /\ (complete => removed /\ release /\ owner = "none") +============================================================================= diff --git a/tests/models/tla/native-frontend-recovery/flow.mmd b/tests/models/tla/native-frontend-recovery/flow.mmd new file mode 100644 index 000000000..2542715ec --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/flow.mmd @@ -0,0 +1,15 @@ +flowchart TD + Down["plain down / down --recover"] --> Live{"Live stop reachable?"} + Live -->|yes| Stop["Owner-mediated stop"] + Live -->|no request sent| Ready{"Saved frontend Ready?"} + Ready -->|yes| Proof["Existing Ready recovery proof"] + Ready -->|no| Settled{"Exact settled start?"} + Settled -->|no| Refuse["Refuse; retain evidence"] + Settled -->|yes| Admission["Dead selected or fresh held admission"] + Admission --> Intent["Commit v3 intent"] + Intent --> Cleanup["Proven native cleanup"] + Proof --> Cleanup + Cleanup --> Removed["Removed and null observations"] + Removed --> Retire["Retire exact frontend files"] + Retire --> Release["Record release; retire admission"] + Release --> Complete["Confirm absence; complete"] diff --git a/tests/models/tla/native-frontend-recovery/flow.svg b/tests/models/tla/native-frontend-recovery/flow.svg new file mode 100644 index 000000000..bac7908e4 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/flow.svg @@ -0,0 +1,66 @@ +yesno request sentyesnonoyesplain down / down--recoverLive stop reachable?Owner-mediated stopSaved frontend Ready?Existing Ready recoveryproofExact settled start?Refuse; retain evidenceDead selected or freshheld admissionCommit v3 intentProven native cleanupRemoved and nullobservationsRetire exact frontend filesRecord release; retireadmissionConfirm absence;complete \ No newline at end of file diff --git a/tests/models/tla/native-frontend-recovery/pre-ready-negative.cfg b/tests/models/tla/native-frontend-recovery/pre-ready-negative.cfg new file mode 100644 index 000000000..be8b05cd3 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/pre-ready-negative.cfg @@ -0,0 +1,4 @@ +CONSTANTS EnforceSettled = FALSE EnforceRelease = TRUE EnforceIntent = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK OnlySettledCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/pre-ready-positive.cfg b/tests/models/tla/native-frontend-recovery/pre-ready-positive.cfg new file mode 100644 index 000000000..eb28c06c3 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/pre-ready-positive.cfg @@ -0,0 +1,4 @@ +CONSTANTS EnforceSettled = TRUE EnforceRelease = TRUE EnforceIntent = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK OnlySettledCleanup ReleaseRecorded NoReplay RetirementHasProof +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/pre-ready-release.cfg b/tests/models/tla/native-frontend-recovery/pre-ready-release.cfg new file mode 100644 index 000000000..6af24fee8 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/pre-ready-release.cfg @@ -0,0 +1,4 @@ +CONSTANTS EnforceSettled = TRUE EnforceRelease = FALSE EnforceIntent = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK ReleaseRecorded +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/pre-ready-replay.cfg b/tests/models/tla/native-frontend-recovery/pre-ready-replay.cfg new file mode 100644 index 000000000..5b5962409 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/pre-ready-replay.cfg @@ -0,0 +1,4 @@ +CONSTANTS EnforceSettled = TRUE EnforceRelease = TRUE EnforceIntent = FALSE +SPECIFICATION Spec +INVARIANTS TypeOK NoReplay +CHECK_DEADLOCK FALSE diff --git a/tests/native-authored-command.test.ts b/tests/native-authored-command.test.ts index dc2465e9e..9bfbd1352 100644 --- a/tests/native-authored-command.test.ts +++ b/tests/native-authored-command.test.ts @@ -9,6 +9,8 @@ import { type serveNativeAuthoredProject, } from "../src/backends/native-authored-project-start.ts"; import { NativeAuthoredProjectNotStartedError } from "../src/backends/native-authored-runtime-binary.ts"; +import { NativeAuthoredStopUnavailableError } from "../src/backends/native-authored-stop-errors.ts"; +import { NativeAuthoredReservedStartError } from "../src/backends/native-authored-unreserved-start.ts"; import { CliUsageError } from "../src/cli/command.ts"; import { HackCliError } from "../src/lib/cli-result.ts"; import { @@ -725,6 +727,83 @@ macTest( } ); +macTest( + "plain down and explicit recovery share proven pre-Ready cleanup without startup", + async () => { + for (const recover of [false, true]) { + const selected = await fixture(); + const calls: string[] = []; + const code = await tryNativeAuthoredCommand({ + ...selected, + options: { ...selected.options, operation: "down", recover }, + stop: async () => { + calls.push("stop"); + throw new NativeAuthoredStopUnavailableError(); + }, + unreservedSelected: async () => true, + retireUnreserved: async () => { + calls.push("unreserved"); + throw new NativeAuthoredReservedStartError(); + }, + nothingToRecover: async () => false, + recover: async () => { + calls.push("recover"); + return {} as Awaited>; + }, + serve: async () => { + throw new Error("Must not replay startup"); + }, + }); + expect(code).toBe(0); + expect(calls).toEqual( + recover ? ["unreserved", "recover"] : ["stop", "unreserved", "recover"] + ); + } + } +); + +macTest( + "down preserves cancellation, uncertain dispatch and busy unreserved outcomes without recovery", + async () => { + for (const failure of ["SIGINT", "SIGTERM", "uncertain", "busy"] as const) { + const selected = await fixture(); + let recoveries = 0; + const result = tryNativeAuthoredCommand({ + ...selected, + options: { + ...selected.options, + operation: "down", + recover: failure === "busy", + }, + stop: async () => { + if (failure === "SIGINT" || failure === "SIGTERM") { + process.emit(failure); + throw new NativeAuthoredStopUnavailableError(); + } + throw new Error("stop dispatched; outcome uncertain"); + }, + unreservedSelected: async () => true, + retireUnreserved: async () => { + throw new Error("admission busy"); + }, + nothingToRecover: async () => false, + recover: async () => { + recoveries++; + throw new Error("must not recover"); + }, + }); + if (failure === "SIGINT" || failure === "SIGTERM") { + expect(await result).toBe(failure === "SIGINT" ? 130 : 143); + } else { + await expect(result).rejects.toThrow( + failure === "busy" ? "recovery is incomplete" : "stop is unconfirmed" + ); + } + expect(recoveries).toBe(0); + } + } +); + macTest( "foreground restart proves the owner-mediated stop before delegating a fresh startup with the exact selection", async () => { diff --git a/tests/native-authored-hook-stop.test.ts b/tests/native-authored-hook-stop.test.ts index c6878ef74..066273555 100644 --- a/tests/native-authored-hook-stop.test.ts +++ b/tests/native-authored-hook-stop.test.ts @@ -4,6 +4,7 @@ import { requestNativeHookStop, serveNativeHookStop, } from "../src/backends/native-authored-hook-stop.ts"; +import { NativeAuthoredStopUnavailableError } from "../src/backends/native-authored-stop-errors.ts"; const run = "a".repeat(32); test("authenticated concurrent stop requests share one owner operation", async () => { @@ -72,9 +73,13 @@ test("client timeout retains a single running operation and a late client observ }, }); try { - await expect( - requestNativeHookStop({ ...owner, run, timeoutMs: 30 }) - ).rejects.toThrow(); + const outcome = await requestNativeHookStop({ + ...owner, + run, + timeoutMs: 30, + }).catch((error: unknown) => error); + expect(outcome).toBeInstanceOf(Error); + expect(outcome).not.toBeInstanceOf(NativeAuthoredStopUnavailableError); expect(calls).toBe(1); finish(true); await requestNativeHookStop({ ...owner, run, timeoutMs: 1000 }); @@ -84,6 +89,18 @@ test("client timeout retains a single running operation and a late client observ await owner.close(true); } }); + +test("a refused connection is distinguished from a dispatched stop", async () => { + const owner = await serveNativeHookStop({ + run, + assertFresh: async () => undefined, + stop: async () => true, + }); + await owner.close(true); + await expect( + requestNativeHookStop({ ...owner, run, timeoutMs: 1000 }) + ).rejects.toBeInstanceOf(NativeAuthoredStopUnavailableError); +}); test("changed private owner refuses before a valid client can stop", async () => { let calls = 0; const owner = await serveNativeHookStop({ diff --git a/tests/native-authored-host-processes.test.ts b/tests/native-authored-host-processes.test.ts index 028dd992e..5c1d55e92 100644 --- a/tests/native-authored-host-processes.test.ts +++ b/tests/native-authored-host-processes.test.ts @@ -215,6 +215,65 @@ ownedCase( }); } ); +ownedCase( + "pre-Ready observation refuses live host records without stopping and accepts settled records without retiring", + async (f) => { + await withNativeAuthoredProjectAdmission(f.scope, async (admission) => { + const owner = await createNativeAuthoredHostProcesses({ + scope: f.scope, + run, + semanticHash: semantic, + projectName: "fixture", + lifecycle: f.lifecycle, + report: () => f.report, + resolveValues: () => Promise.resolve({ TOKEN: canary }), + assertFresh: admission.assertHeld, + signal: f.abort.signal, + remaining: f.remaining, + }); + unknown = true; + try { + await owner.start(); + const calls = await readFile(`${f.state}.calls`, "utf8"); + const check = { + scope: f.scope, + run, + assertFresh: admission.assertHeld, + checkOnly: true, + }; + await expect( + retireNativeAuthoredHostProcesses(check) + ).rejects.toThrow(); + const later = (await readFile(`${f.state}.calls`, "utf8")).slice( + calls.length + ); + expect(later).not.toContain("send-keys"); + expect(later).not.toContain("kill-session"); + await owner.stop(); + await retireNativeAuthoredHostProcesses(check); + const path = join( + f.projectDir, + ".internal/native-authored-runs", + `${run}.host-process-owner.json` + ); + expect(await Bun.file(path).exists()).toBe(true); + await retireNativeAuthoredHostProcesses({ ...check, checkOnly: false }); + expect(await Bun.file(path).exists()).toBe(false); + // Recovery removed its publication; close still releases the original + // fixture owner's held directory handles while refusing changed proof. + await expect(owner.close()).rejects.toThrow(); + unknown = false; + } finally { + if (unknown) { + await owner.stop(); + await owner.close(); + unknown = false; + } + } + }); + } +); + ownedCase( "recovery uses the saved exact controller identity and refuses a foreign live token before any stop", async (f) => { diff --git a/tests/native-authored-project-recovery.test.ts b/tests/native-authored-project-recovery.test.ts index 137056fb7..7d9eac1ac 100644 --- a/tests/native-authored-project-recovery.test.ts +++ b/tests/native-authored-project-recovery.test.ts @@ -140,7 +140,11 @@ function removed() { ), }); } -async function fixture(bootSession = false, liveStop = false) { +async function fixture( + bootSession = false, + liveStop = false, + preReady?: "interrupted" | "released" +) { const root = await realpath( await mkdtemp(join(tmpdir(), "native-frontend-recovery-")) ); @@ -163,7 +167,11 @@ async function fixture(bootSession = false, liveStop = false) { ...(liveStop ? ["phase='reserve';"] : []), "const start = await admission.reserve({review:receipt.review});", ...(liveStop ? ["phase='publish';"] : []), - "const ready=await admission.publish({expectedStart:start,record:{version:2,kind:'native-authored-project-run',receipt}});", + ...(preReady + ? [] + : [ + "const ready=await admission.publish({expectedStart:start,record:{version:2,kind:'native-authored-project-run',receipt}});", + ]), ...(liveStop ? [ `const {captureNativeAuthoredProcessIncarnation}=await import(${JSON.stringify(join(import.meta.dir, "../src/backends/native-authored-process-incarnation.ts"))});`, @@ -176,8 +184,12 @@ async function fixture(bootSession = false, liveStop = false) { "void (async()=>{while(!(await Bun.file(scope.projectRoot+'/stop-original').exists()))await Bun.sleep(10);if(native.exitCode===null)native.kill('SIGKILL');await native.exited;await Bun.write(scope.projectRoot+'/native-settled','yes');process.exit(77)})();", ] : []), - "await Bun.write(scope.projectRoot+'/witness','ready');", - "await new Promise(resolve=>setTimeout(resolve,120000)); });", + ...(preReady === "released" + ? ["}); await Bun.write(scope.projectRoot+'/witness','ready');"] + : [ + "await Bun.write(scope.projectRoot+'/witness','ready');", + "await new Promise(resolve=>setTimeout(resolve,120000)); });", + ]), ...(liveStop ? [ "} catch {await Bun.write(scope.projectRoot+'/setup-failure',phase);throw new Error('Original fixture setup refused; values omitted.')} finally {if(native){if(native.exitCode===null)native.kill('SIGKILL');await native.exited;await Bun.write(scope.projectRoot+'/native-settled','yes');}}", @@ -241,6 +253,9 @@ async function fixture(bootSession = false, liveStop = false) { censusPids.add(Number(await Bun.file(join(root, "native-pid")).text())); censusPids.add(child.pid); } + if (preReady === "released") { + expect(await child.exited).toBe(0); + } const dir = join(scope.projectDir, ".internal", "native-authored-runs"); const key = createHash("sha256") .update(JSON.stringify(scope.branch)) @@ -350,6 +365,81 @@ async function fixture(bootSession = false, liveStop = false) { }; } +for (const admission of ["interrupted", "released"] as const) { + test(`pre-Ready recovery with ${admission} admission retains its origin and retires without replay`, async () => { + const current = await fixture(true, false, admission); + if (admission === "interrupted") { + await expect( + recoverNativeAuthoredProject(current.options) + ).rejects.toThrow(); + expect(current.calls).toEqual([]); + await current.kill(); + } + current.selection.receipt = parseNativeAuthoredReceipt({ + ...receipt(), + phase: "failed-retained", + }); + let startupEntered = false; + const result = await recoverNativeAuthoredProject({ + ...current.options, + request: async (opts) => { + await expect( + withNativeAuthoredProjectAdmission(current.scope, async () => { + startupEntered = true; + }) + ).rejects.toThrow(); + return await current.request(opts); + }, + }); + expect(result.receipt.phase).toBe("removed"); + expect(startupEntered).toBe(false); + expect(current.cleanupCalls()).toBe(1); + const intent = JSON.parse(await Bun.file(current.paths.intent).text()); + expect(intent.record.version).toBe(3); + expect(intent.record.ready).toBeNull(); + expect(intent.record.admission_held).toBe(admission === "released"); + expect(intent.record.admission_releasing).toBe(admission === "released"); + expect(intent.record.phase).toBe("complete"); + expect((await readdir(current.paths.dir)).sort()).toEqual( + [".gitignore", basename(current.paths.intent)].sort() + ); + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); + await withNativeAuthoredProjectAdmission(current.scope, async () => { + startupEntered = true; + }); + expect(startupEntered).toBe(true); + }); +} + +test("pre-Ready selection accepts runtime Ready and cleanup origins but refuses preparing and pending effects before cleanup", async () => { + for (const phase of [ + "ready-observed", + "stop-intent", + "stopped", + "removal-intent", + "removed", + "preparing", + ] as const) { + const current = await fixture(true, false, "interrupted"); + await current.kill(); + current.selection.receipt = parseNativeAuthoredReceipt({ + ...(phase === "removed" ? removed() : receipt()), + phase, + }); + if (phase === "preparing") { + await expect( + recoverNativeAuthoredProject(current.options) + ).rejects.toThrow(); + expect(current.cleanupCalls()).toBe(0); + expect(await Bun.file(current.paths.start).exists()).toBe(true); + } else { + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); + } + } +}, 30_000); + test("explicit frontend recovery refuses a live admission before any native request", async () => { const current = await fixture(); await expect(recoverNativeAuthoredProject(current.options)).rejects.toThrow(); @@ -360,6 +450,100 @@ test("explicit frontend recovery refuses a live admission before any native requ expect(await Bun.file(current.paths.intent).exists()).toBe(false); }); +test("fresh pre-Ready admission survives crashes around durable release and retains a failed release exactly", async () => { + for (const stage of ["prepared", "armed", "released", "error"] as const) { + const current = await fixture(true, false, "released"); + current.selection.receipt = parseNativeAuthoredReceipt({ + ...receipt(), + phase: "failed-retained", + }); + const witness = join(current.root, "recovery-crash"); + const effects = join(current.root, "cleanup-calls"); + const program = [ + "import {spyOn} from 'bun:test'; import * as files from 'node:fs/promises';", + `import {recoverNativeAuthoredProject} from ${JSON.stringify(join(import.meta.dir, "../src/backends/native-authored-project-recovery.ts"))};`, + `const paths=${JSON.stringify(current.paths)},stage=${JSON.stringify(stage)},witness=${JSON.stringify(witness)};`, + "const pause=async()=>{await Bun.write(witness,stage);await new Promise(resolve=>setTimeout(resolve,120000));};", + "const rename=files.rename,rmdir=files.rmdir,unlink=files.unlink;", + "spyOn(files,'unlink').mockImplementation(async(path)=>{await unlink(path);if(stage==='prepared'&&path===paths.intent+'.pending')await pause();});", + "spyOn(files,'rename').mockImplementation(async(from,to)=>{await rename(from,to);if(to===paths.intent){const {record}=await Bun.file(to).json();if((stage==='prepared'&&record.phase==='prepared')||(stage==='armed'&&record.admission_releasing)){await pause();}}});", + "spyOn(files,'rmdir').mockImplementation(async(path,...args)=>{await rmdir(path,...args);if(stage==='released'&&path===paths.admission)await pause();});", + `const selection=${JSON.stringify(current.selection)},result=${JSON.stringify(current.result)};`, + `try {await recoverNativeAuthoredProject({scope:${JSON.stringify(current.scope)},runtime:${JSON.stringify(current.options.runtime)},timeoutMs:30000,request:async opts=>{`, + "if(opts.args[2]==='recovery-selection')return selection;", + `if(opts.args[2]==='recover-live-owner'){await files.appendFile(${JSON.stringify(effects)},'cleanup'+String.fromCharCode(10));return result;}`, + "if(stage==='error')throw new Error('inspection unavailable');return {receipt:result.receipt,observations:{web:null}};}});throw new Error('unexpected completion');}catch{if(stage!=='error')throw new Error('unexpected refusal');await Bun.write(witness,stage);process.exitCode=77;}", + ].join("\n"); + const child = Bun.spawn([process.execPath, "--eval", program], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + env: privateCliEnv({ root: current.root }), + }); + fixtures.push({ root: current.root, child }); + await waitForFileContent({ + path: witness, + what: "pre-Ready recovery crash", + budgetMs: 5000, + complete: (text) => (text === stage ? text : undefined), + }); + if (stage === "error") { + expect(await child.exited).toBe(77); + } else { + child.kill("SIGKILL"); + expect(await child.exited).toBe(137); + } + const saved = JSON.parse( + await Bun.file(current.paths.intent).text() + ).record; + expect(saved.version).toBe(3); + expect(saved.admission_held).toBe(true); + if (stage === "released") { + expect(await lstat(current.paths.admission).catch(() => null)).toBeNull(); + let entered = false; + await expect( + withNativeAuthoredProjectAdmission(current.scope, async () => { + entered = true; + }) + ).rejects.toThrow(); + expect(entered).toBe(false); + expect(await nativeAuthoredProjectNothingToRecover(current.scope)).toBe( + false + ); + // An absent lock is insufficient: the exact intent must have authorized + // its release. Preserve the original bytes to resume the legitimate case. + const bytes = await Bun.file(current.paths.intent).text(); + const unarmed = JSON.parse(bytes); + unarmed.record.admission_releasing = false; + await Bun.write(current.paths.intent, JSON.stringify(unarmed)); + await expect( + recoverNativeAuthoredProject(current.options) + ).rejects.toThrow(); + expect(current.cleanupCalls()).toBe(0); + await Bun.write(current.paths.intent, bytes); + } else { + expect((await lstat(current.paths.admission)).ino).toBe( + saved.admission.directory.ino + ); + expect((await lstat(join(current.paths.admission, "owner"))).ino).toBe( + saved.admission.file.ino + ); + } + await recoverNativeAuthoredProject(current.options); + const prior = ( + await Bun.file(effects) + .text() + .catch(() => "") + ).trim() + ? 1 + : 0; + expect(prior + current.cleanupCalls()).toBe(1); + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("complete"); + } +}, 30_000); + test("dead frontend recovery binds the stored run, holds startup, retires only after Removed/null and archives completed history", async () => { const current = await fixture(); await current.kill(); diff --git a/tests/native-authored-recovery-protocol.test.ts b/tests/native-authored-recovery-protocol.test.ts index 3738f5da6..df01c52c6 100644 --- a/tests/native-authored-recovery-protocol.test.ts +++ b/tests/native-authored-recovery-protocol.test.ts @@ -76,6 +76,77 @@ function sessionSelection() { host_boot_uuid: "12345678-abcd-abcd-abcd-123456789abc", }; } + +test("pre-Ready recovery binds the start review and accepts only settled effect outcomes", () => { + const start = parseNativeAuthoredReceipt(receipt()).review; + for (const phase of [ + "ready-observed", + "failed-retained", + "stop-intent", + "stopped", + "removal-intent", + ] as const) { + const value = selection(); + value.receipt.phase = phase; + expect( + parseNativeAuthoredRecoverySelection({ value, start }).receipt.phase + ).toBe(phase); + if (phase !== "ready-observed") { + expect(() => + parseNativeAuthoredRecoverySelection({ + value, + admitted: parseNativeAuthoredReceipt(receipt()), + }) + ).toThrow(); + } + for (const pending of [ + "create-intent", + "start-intent", + "uncertain", + "stop-intent", + "remove-intent", + ]) { + const changed = structuredClone(value); + changed.receipt.resources["container:web"].phase = pending; + expect(() => + parseNativeAuthoredRecoverySelection({ value: changed, start }) + ).toThrow(); + } + } + const value = selection(); + const neverCreated = { + ...value, + receipt: { + ...value.receipt, + phase: "failed-retained", + resources: { + ...value.receipt.resources, + "container:web": { + ...value.receipt.resources["container:web"], + id: null, + phase: "reserved", + }, + }, + }, + }; + expect( + parseNativeAuthoredRecoverySelection({ value: neverCreated, start }).receipt + .resources["container:web"]?.id + ).toBeNull(); + expect(() => + parseNativeAuthoredRecoverySelection({ + value: { + ...neverCreated, + receipt: { ...neverCreated.receipt, phase: "preparing" }, + }, + start, + }) + ).toThrow(); + const foreign = { ...start, review_id: "0".repeat(64) }; + expect(() => + parseNativeAuthoredRecoverySelection({ value, start: foreign }) + ).toThrow(); +}); function sourceReceipt() { const project = "/private/native-source-fixture"; return { From 075c1e6f675d1019c8bb48227b1a8884d6ecd697 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:46:04 +0000 Subject: [PATCH 3/5] fix(native): preserve recovery routing through storage refusals --- docs/reference/native-authored-commands.md | 4 + src/backends/native-authored-project-run.ts | 15 ++- src/backends/native-authored-stop-errors.ts | 10 ++ .../native-authored-unreserved-start.ts | 49 ++++++-- src/lib/native-authored-command.ts | 4 +- tests/native-authored-command.test.ts | 6 +- .../native-authored-project-recovery.test.ts | 107 +++++++++++++++++- tests/native-authored-project-run.test.ts | 31 +++++ tests/native-exec-selection-cli.test.ts | 6 +- tests/native-project-exec.test.ts | 20 ++-- tests/native-project-start.test.ts | 4 +- tests/native-refusal-cli.test.ts | 19 +++- tests/native-runtime-not-running-cli.test.ts | 4 +- tests/native-up-already-running-cli.test.ts | 8 +- 14 files changed, 244 insertions(+), 43 deletions(-) diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 98484df7d..2d1acf9fd 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -310,6 +310,10 @@ any other refusal, or a successful inspection retains every file. This path does not compile input, acquire values, serve a run or touch guest resources. If it refuses because runtime state was reserved, the command independently attempts the settled recovery proof below; that refusal grants no authority. +If inspection refuses because an interrupted removal no longer has an intact +network, a read-only recovery selection must validate the exact start-bound +settled journal before choosing recovery. Busy, canceled, unknown and malformed +results still cannot choose cleanup from the inspection refusal alone. When Ready was never saved, recovery requires the exact start and private source envelope, repeatedly proves Ready absent, and records a distinct version 3 diff --git a/src/backends/native-authored-project-run.ts b/src/backends/native-authored-project-run.ts index 5b71ba786..cb7d713b5 100644 --- a/src/backends/native-authored-project-run.ts +++ b/src/backends/native-authored-project-run.ts @@ -45,7 +45,10 @@ import { requestNativeAuthoredLiveStop, } from "./native-authored-live-stop.ts"; import type { NativeAuthoredProcessIncarnation } from "./native-authored-process-incarnation.ts"; -import { NativeAuthoredStopUnavailableError } from "./native-authored-stop-errors.ts"; +import { + NativeAuthoredReservedStartError, + NativeAuthoredStopUnavailableError, +} from "./native-authored-stop-errors.ts"; import { NATIVE_GRAPH_UNRESERVED, NativeRuntimeRequestError, @@ -426,7 +429,15 @@ async function storage( }, check ); - } catch { + } catch (error) { + // Preserve only closed, value-free routing hints across the redaction boundary. + // Recovery must still acquire its leases and prove the selected state itself. + if ( + error instanceof NativeAuthoredStopUnavailableError || + error instanceof NativeAuthoredReservedStartError + ) { + throw error; + } return refused(); } finally { await Promise.all(held.map(({ file }) => file.close())); diff --git a/src/backends/native-authored-stop-errors.ts b/src/backends/native-authored-stop-errors.ts index 75004b1d4..58ac63a28 100644 --- a/src/backends/native-authored-stop-errors.ts +++ b/src/backends/native-authored-stop-errors.ts @@ -7,3 +7,13 @@ export class NativeAuthoredStopUnavailableError extends Error { this.name = "NativeAuthoredStopUnavailableError"; } } + +/** Inspection or a settled selection found a journal; cleanup still needs its own proof. */ +export class NativeAuthoredReservedStartError extends Error { + constructor() { + super( + "Native startup has reserved runtime state; stored-generation recovery is required. Values omitted." + ); + this.name = "NativeAuthoredReservedStartError"; + } +} diff --git a/src/backends/native-authored-unreserved-start.ts b/src/backends/native-authored-unreserved-start.ts index 6468ff4d7..528ae127b 100644 --- a/src/backends/native-authored-unreserved-start.ts +++ b/src/backends/native-authored-unreserved-start.ts @@ -4,8 +4,11 @@ import { withNativeAuthoredProjectAdmission, withNativeAuthoredProjectStatus, } from "./native-authored-project-run.ts"; +import { parseNativeAuthoredRecoverySelection } from "./native-authored-recovery-protocol.ts"; +import { NativeAuthoredReservedStartError } from "./native-authored-stop-errors.ts"; import { invokeNativeRuntime, + NativeRuntimeRequestError, type NativeRuntimeSelection, } from "./native-runtime-client.ts"; @@ -15,16 +18,6 @@ function refused(): never { ); } -/** A successful inspection proves a journal exists; cleanup still needs its own proof. */ -export class NativeAuthoredReservedStartError extends Error { - constructor() { - super( - "Native startup has reserved runtime state; stored-generation recovery is required. Values omitted." - ); - this.name = "NativeAuthoredReservedStartError"; - } -} - /** * Read-only selection for explicit recovery: true only when a startup intent is * retained without a Ready mapping or a frontend recovery intent. Any refusal @@ -117,6 +110,42 @@ export async function retireNativeAuthoredUnreservedStart(opts: { } remaining(); await admission.assertHeld(); + if ( + refusal instanceof NativeRuntimeRequestError && + refusal.nativeCode === "native_graph_admission" && + refusal.stopSignal === undefined + ) { + // Inspection expects an intact network even during interrupted removal. + // A closed, start-bound journal selection can route that attempt to the + // independent cleanup owner. An inspection refusal alone never can. + const selected = await request({ + runtime, + cwd: scope.projectRoot, + args: [ + "graph", + "native", + "recovery-selection", + "--run-id", + run, + "--json", + ], + timeoutMs: remaining(), + signal, + boundNativeAuthoredRecoveryDrain: "selection", + }) + .then((value) => + parseNativeAuthoredRecoverySelection({ + value, + start: start.record.review, + }) + ) + .catch(() => null); + remaining(); + await admission.assertHeld(); + if (selected) { + throw new NativeAuthoredReservedStartError(); + } + } await admission.retireUnreserved({ expectedStart: start, refusal }); return { run }; }); diff --git a/src/lib/native-authored-command.ts b/src/lib/native-authored-command.ts index 6e3925f47..2c7515aeb 100644 --- a/src/lib/native-authored-command.ts +++ b/src/lib/native-authored-command.ts @@ -29,9 +29,11 @@ import { serveNativeAuthoredProject, } from "../backends/native-authored-project-start.ts"; import { NativeAuthoredProjectNotStartedError } from "../backends/native-authored-runtime-binary.ts"; -import { NativeAuthoredStopUnavailableError } from "../backends/native-authored-stop-errors.ts"; import { NativeAuthoredReservedStartError, + NativeAuthoredStopUnavailableError, +} from "../backends/native-authored-stop-errors.ts"; +import { nativeAuthoredUnreservedStartSelected, retireNativeAuthoredUnreservedStart, } from "../backends/native-authored-unreserved-start.ts"; diff --git a/tests/native-authored-command.test.ts b/tests/native-authored-command.test.ts index 9bfbd1352..bd41de4a0 100644 --- a/tests/native-authored-command.test.ts +++ b/tests/native-authored-command.test.ts @@ -9,8 +9,10 @@ import { type serveNativeAuthoredProject, } from "../src/backends/native-authored-project-start.ts"; import { NativeAuthoredProjectNotStartedError } from "../src/backends/native-authored-runtime-binary.ts"; -import { NativeAuthoredStopUnavailableError } from "../src/backends/native-authored-stop-errors.ts"; -import { NativeAuthoredReservedStartError } from "../src/backends/native-authored-unreserved-start.ts"; +import { + NativeAuthoredReservedStartError, + NativeAuthoredStopUnavailableError, +} from "../src/backends/native-authored-stop-errors.ts"; import { CliUsageError } from "../src/cli/command.ts"; import { HackCliError } from "../src/lib/cli-result.ts"; import { diff --git a/tests/native-authored-project-recovery.test.ts b/tests/native-authored-project-recovery.test.ts index 7d9eac1ac..4c5730e06 100644 --- a/tests/native-authored-project-recovery.test.ts +++ b/tests/native-authored-project-recovery.test.ts @@ -23,8 +23,13 @@ import { nativeAuthoredProjectNothingToRecover, withNativeAuthoredProjectAdmission, } from "../src/backends/native-authored-project-run.ts"; +import { NativeAuthoredReservedStartError } from "../src/backends/native-authored-stop-errors.ts"; +import { retireNativeAuthoredUnreservedStart } from "../src/backends/native-authored-unreserved-start.ts"; import { createNativeProcessCensusDiagnostics } from "../src/backends/native-process-census-diagnostics.ts"; -import type { invokeNativeRuntime } from "../src/backends/native-runtime-client.ts"; +import { + type invokeNativeRuntime, + NativeRuntimeRequestError, +} from "../src/backends/native-runtime-client.ts"; import { tryNativeAuthoredCommand } from "../src/lib/native-authored-command.ts"; import { installFixtureProcessCensus } from "./helpers/fixture-process-census.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -440,6 +445,51 @@ test("pre-Ready selection accepts runtime Ready and cleanup origins but refuses } }, 30_000); +test("an interrupted network removal routes from refused inspection only through an exact settled journal selection", async () => { + for (const eligible of [true, false]) { + const current = await fixture(true, false, "released"); + current.selection.receipt = parseNativeAuthoredReceipt({ + ...receipt(), + phase: eligible ? "removal-intent" : "preparing", + resources: Object.fromEntries( + Object.entries(receipt().resources).map(([name, resource]) => [ + name, + { + ...resource, + phase: resource.kind === "network" ? "remove-intent" : "removed", + }, + ]) + ), + }); + const calls: string[] = []; + const failure = await retireNativeAuthoredUnreservedStart({ + ...current.options, + request: async (opts) => { + calls.push(opts.args[2] ?? ""); + if (opts.args[2] === "inspect") { + throw new NativeRuntimeRequestError({ + message: "synthetic missing network", + nativeCode: "native_graph_admission", + }); + } + expect(opts.args[2]).toBe("recovery-selection"); + return current.selection; + }, + }).catch((error: unknown) => error); + expect(calls).toEqual(["inspect", "recovery-selection"]); + expect(await Bun.file(current.paths.start).exists()).toBe(true); + expect(current.cleanupCalls()).toBe(0); + if (eligible) { + expect(failure).toBeInstanceOf(NativeAuthoredReservedStartError); + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); + } else { + expect(failure).toBeInstanceOf(Error); + expect(failure).not.toBeInstanceOf(NativeAuthoredReservedStartError); + } + } +}); + test("explicit frontend recovery refuses a live admission before any native request", async () => { const current = await fixture(); await expect(recoverNativeAuthoredProject(current.options)).rejects.toThrow(); @@ -450,6 +500,61 @@ test("explicit frontend recovery refuses a live admission before any native requ expect(await Bun.file(current.paths.intent).exists()).toBe(false); }); +test("unreserved probing preserves files on malformed, failed, cancelled or unrelated inspection refusals", async () => { + for (const scenario of [ + "malformed", + "failed", + "cancelled", + "busy", + ] as const) { + const current = await fixture(true, false, "released"); + const controller = new AbortController(); + const calls: string[] = []; + const failure = await retireNativeAuthoredUnreservedStart({ + ...current.options, + signal: controller.signal, + request: async (opts) => { + calls.push(opts.args[2] ?? ""); + if (opts.args[2] === "inspect") { + throw new NativeRuntimeRequestError({ + message: "synthetic inspection refusal", + nativeCode: + scenario === "busy" + ? "native_runtime_busy" + : "native_graph_admission", + }); + } + if (scenario === "cancelled") { + controller.abort(130); + } + if (scenario !== "malformed") { + throw new Error("synthetic probe failure"); + } + return { invalid: true }; + }, + }).catch((error: unknown) => error); + expect(calls).toEqual( + scenario === "busy" ? ["inspect"] : ["inspect", "recovery-selection"] + ); + expect(failure).toBeInstanceOf(Error); + expect(failure).not.toBeInstanceOf(NativeAuthoredReservedStartError); + expect(await Bun.file(current.paths.start).exists()).toBe(true); + expect(await Bun.file(current.paths.intent).exists()).toBe(false); + expect(current.cleanupCalls()).toBe(0); + } +}); + +test("successful inspection preserves the reserved-start routing signal through real admission storage", async () => { + const current = await fixture(true, false, "released"); + const failure = await retireNativeAuthoredUnreservedStart({ + ...current.options, + request: async () => current.selection.receipt, + }).catch((error: unknown) => error); + expect(failure).toBeInstanceOf(NativeAuthoredReservedStartError); + expect(await Bun.file(current.paths.start).exists()).toBe(true); + expect(current.cleanupCalls()).toBe(0); +}); + test("fresh pre-Ready admission survives crashes around durable release and retains a failed release exactly", async () => { for (const stage of ["prepared", "armed", "released", "error"] as const) { const current = await fixture(true, false, "released"); diff --git a/tests/native-authored-project-run.test.ts b/tests/native-authored-project-run.test.ts index 48de289b2..dde15a1ec 100644 --- a/tests/native-authored-project-run.test.ts +++ b/tests/native-authored-project-run.test.ts @@ -26,7 +26,9 @@ import { prepareNativeAuthoredProjectRunStorage as prepare, removeNativeAuthoredProjectRun as remove, saveNativeAuthoredProjectRun as save, + stopNativeAuthoredProject, } from "../src/backends/native-authored-project-run.ts"; +import { NativeAuthoredStopUnavailableError } from "../src/backends/native-authored-stop-errors.ts"; import { loadNativeProjectRun as loadCompose, saveNativeProjectRun as saveCompose, @@ -137,6 +139,35 @@ async function file(opts: Awaited>) { return join(root, name); } +test("missing Ready reaches the stop caller as an unavailable owner through real storage", async () => { + const scope = await fixture(); + await prepare(scope); + const failure = await stopNativeAuthoredProject({ + scope, + timeoutMs: 5000, + }).catch((error: unknown) => error); + expect(failure).toBeInstanceOf(NativeAuthoredStopUnavailableError); +}); + +test("storage still redacts arbitrary errors and lookalike routing errors", async () => { + const scope = await fixture(); + for (const name of [ + "Error", + "NativeAuthoredStopUnavailableError", + "NativeAuthoredReservedStartError", + ]) { + const original = new Error("private fixture detail"); + original.name = name; + const failure = await admit(scope, async () => { + throw original; + }).catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(failure).not.toBe(original); + expect(String(failure)).not.toContain("private fixture detail"); + expect(failure).not.toBeInstanceOf(NativeAuthoredStopUnavailableError); + } +}); + test("absent native artifact reads have no writes and storage remains private and excluded", async () => { const opts = await fixture(); expect(await load(opts)).toBeNull(); diff --git a/tests/native-exec-selection-cli.test.ts b/tests/native-exec-selection-cli.test.ts index 9df2c0136..4c56bf397 100644 --- a/tests/native-exec-selection-cli.test.ts +++ b/tests/native-exec-selection-cli.test.ts @@ -105,7 +105,7 @@ async function fixture(opts: { readonly composeServices: readonly string[] }) { printf '%s\\n' "$*" >> "$HOME/native-calls" case "$4" in refresh-dependencies) printf '%s' '${refreshed}' ;; - inspect) printf '%s' '${snapshot}' ;; + status) printf '%s' '${snapshot}' ;; exec-selection) printf '%s' '${selection}' ;; *) printf '%s' '{"code":"unexpected_request"}' >&2; exit 1 ;; esac @@ -186,7 +186,7 @@ test("fresh native exec reports a service removed since startup as a changed sel // Refused before any exec request reached the runtime. expect(await nativeCalls(f.root)).toEqual([ "refresh-dependencies", - "inspect", + "status", "exec-selection", ]); }, 30_000); @@ -207,7 +207,7 @@ test("fresh native exec reports unresolvable values apart from a changed selecti expect(result.output).not.toMatch(STACK_FRAME); expect(await nativeCalls(f.root)).toEqual([ "refresh-dependencies", - "inspect", + "status", "exec-selection", ]); }, 30_000); diff --git a/tests/native-project-exec.test.ts b/tests/native-project-exec.test.ts index 6e85630d2..33f1e6157 100644 --- a/tests/native-project-exec.test.ts +++ b/tests/native-project-exec.test.ts @@ -78,7 +78,7 @@ test("native exec preserves literal argv, binary output and nonzero completion", if (request.args[1] === "refresh-dependencies") { return refreshed(); } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(); } expect(request.args.slice(-4)).toEqual(["--", ...argv]); @@ -127,7 +127,7 @@ test("dead foreground owner refuses one-off exec before command effects", async if (request.args[1] === "refresh-dependencies") { return refreshed(); } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { const value = snapshot(); return { ...value, @@ -246,7 +246,7 @@ test("malformed exec output refuses without replay", async () => { if (request.args[1] === "refresh-dependencies") { return refreshed(); } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(); } executions++; @@ -308,7 +308,7 @@ test("fresh exec sends environment only through private stdin and clears its buf expect(request.privateInput).toBeUndefined(); return { ...refreshed(), changed_slots: [0, 3] }; } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(); } if (request.args[1] === "exec-selection") { @@ -333,10 +333,10 @@ test("fresh exec sends environment only through private stdin and clears its buf expect(executions).toBe(1); expect(calls).toEqual([ "refresh-dependencies", - "inspect", + "status", "exec-selection", "exec", - "inspect", + "status", ]); expect(payload?.every((byte) => byte === 0)).toBe(true); }); @@ -350,7 +350,7 @@ test("empty managed environment uses ordinary exec without a private launcher", if (request.args[1] === "refresh-dependencies") { return refreshed(); } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(); } if (request.args[1] === "exec-selection") { @@ -381,7 +381,7 @@ test("fresh exec clears private input on transport failure and never retries", a if (request.args[1] === "refresh-dependencies") { return refreshed(); } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(); } if (request.args[1] === "exec-selection") { @@ -409,7 +409,7 @@ test("changed exec selection refuses before private environment delivery", async if (request.args[1] === "refresh-dependencies") { return refreshed(); } - if (request.args[1] === "inspect") { + if (request.args[1] === "status") { return snapshot(); } if (request.args[1] === "exec-selection") { @@ -461,7 +461,7 @@ test("exec names an inactive service before sending any command", async () => { argv: ["true"], invoke: async (request) => { calls.push(request.args[1]!); - return request.args[1] === "inspect" ? snapshot() : refreshed(); + return request.args[1] === "status" ? snapshot() : refreshed(); }, }) ).rejects.toThrow( diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index d9a853cdb..5f35a1520 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -639,8 +639,8 @@ function failedStartupFixture( }; }; opts.dependencies.invoke = async (request) => { - if (request.args[1] === "inspect") { - events.push("graph inspect"); + if (request.args[1] === "inspect" || request.args[1] === "status") { + events.push(`graph ${request.args[1]}`); return { journal_incomplete: receipt.journalIncomplete === true, receipt: { diff --git a/tests/native-refusal-cli.test.ts b/tests/native-refusal-cli.test.ts index 1cf160beb..08b0351f2 100644 --- a/tests/native-refusal-cli.test.ts +++ b/tests/native-refusal-cli.test.ts @@ -330,7 +330,7 @@ test( expect(detached.output).toContain(NATIVE_PROVIDER_PIN_MESSAGE); expect(detached.output).not.toContain("synthetic-private"); await savedRun(f); - await f.fail("graph inspect", pin); + await f.fail("graph status", pin); const ps = await cli({ fixture: f, args: ["ps", "--json"] }); expect(ps.code).toBe(0); expect(JSON.parse(ps.stdout)).toMatchObject({ @@ -464,13 +464,13 @@ test( async () => { const f = await fixture(); await savedRun(f); - await f.reply("graph inspect", "not-json"); + await f.reply("graph status", "not-json"); const result = await cli({ fixture: f, args: ["ps"] }); expectOneClosedLine( result, "Native runtime returned an invalid control response." ); - expect(await f.calls()).toEqual(["graph inspect"]); + expect(await f.calls()).toEqual(["graph status"]); }, SPAWN_TIMEOUT_MS ); @@ -713,11 +713,13 @@ test.each([ async (code) => { const f = await fixture(); await savedRun(f); - await f.reply("graph inspect", { + const observed = { ...stoppedGraph, receipt: { ...stoppedGraph.receipt, phase: "ready-observed" }, observations: { "container:web": { state: "running" } }, - }); + }; + await f.reply("graph inspect", observed); + await f.reply("graph status", observed); await ( await beginNativeProjectFinalization({ scope: f.scope, run }) ).complete(); @@ -736,7 +738,8 @@ test.each([ expect(refused.output).toContain("Run hack ps --json"); const next = await cli({ fixture: f, args: ["ps", "--json"] }); expect(next.code).toBe(0); - expect((await f.calls()).at(-1)).toBe("graph inspect"); + expect(JSON.parse(next.stdout).data.status).toBe("observed"); + expect((await f.calls()).at(-1)).toBe("graph status"); }, SPAWN_TIMEOUT_MS ); @@ -801,7 +804,11 @@ test( async () => { const f = await fixture(); await savedRun(f); + await ( + await beginNativeProjectFinalization({ scope: f.scope, run }) + ).complete(); await f.reply("graph inspect", stoppedGraph); + await f.reply("graph status", stoppedGraph); await f.reply("graph foreground-owner", { version: 1, run: run.run, diff --git a/tests/native-runtime-not-running-cli.test.ts b/tests/native-runtime-not-running-cli.test.ts index 8d83d6fac..8c4bb6e3a 100644 --- a/tests/native-runtime-not-running-cli.test.ts +++ b/tests/native-runtime-not-running-cli.test.ts @@ -147,9 +147,9 @@ test("native ps reports a stopped runtime as a status, not a failure", async () expect(output).toContain("NATIVE SERVICE"); expect(output).not.toContain("Native runtime request failed"); expect(output).not.toMatch(STACK_FRAME); - // The read reached the runtime exactly once (graph inspect for the mapping). + // The read reached the runtime exactly once (graph status for the mapping). expect(await nativeCalls(f.root)).toEqual([ - `--candidate-root ${join(f.root, "native-home")} graph inspect --run-id ${run.run} --json`, + `--candidate-root ${join(f.root, "native-home")} graph status --run-id ${run.run} --json`, ]); }); diff --git a/tests/native-up-already-running-cli.test.ts b/tests/native-up-already-running-cli.test.ts index 55703b077..d0884cda8 100644 --- a/tests/native-up-already-running-cli.test.ts +++ b/tests/native-up-already-running-cli.test.ts @@ -102,7 +102,7 @@ exit 1 { mode: 0o700 } ); await writeFile( - join(stub, "graph-inspect.out"), + join(stub, "graph-status.out"), JSON.stringify({ journal_incomplete: false, receipt: { @@ -283,7 +283,7 @@ test( }); // Only read-only observations: nothing was started, stopped or reviewed. for (const call of await f.calls()) { - expect(["graph inspect", "graph owner-status"]).toContain(call); + expect(["graph status", "graph owner-status"]).toContain(call); } }, SPAWN_TIMEOUT_MS @@ -316,7 +316,7 @@ test( error: { code: "E_STARTUP_INCOMPLETE" }, }); for (const call of await f.calls()) { - expect(["graph inspect", "graph owner-status"]).toContain(call); + expect(["graph status", "graph owner-status"]).toContain(call); } }, SPAWN_TIMEOUT_MS @@ -479,7 +479,7 @@ test( expect(result.output).not.toContain('"alreadyRunning": true'); } for (const call of await f.calls()) { - expect(["graph inspect", "graph owner-status"]).toContain(call); + expect(["graph status", "graph owner-status"]).toContain(call); } } }, From c5d779fbaae6e3776436a4d876ea6e7e5b8f1548 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:55:16 +0000 Subject: [PATCH 4/5] fix(native): admit lock-free authored observation through bounded transport --- src/backends/native-runtime-client.ts | 2 +- tests/env-migration-tracked-consent.test.ts | 3 ++- tests/native-authored-project-observe.test.ts | 15 +++++++++++++++ tests/native-runtime-client.test.ts | 13 +++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/backends/native-runtime-client.ts b/src/backends/native-runtime-client.ts index d60b04832..65ae23d1a 100644 --- a/src/backends/native-runtime-client.ts +++ b/src/backends/native-runtime-client.ts @@ -628,7 +628,7 @@ function validNativeStatusDrainSelection(args: readonly string[]): boolean { args[3] === "--run-id" && NATIVE_RUN.test(args[4] ?? "") && args[5] === "--action" && - args[6] === "status" && + (args[6] === "status" || args[6] === "observe") && args[7] === "--json" ); } diff --git a/tests/env-migration-tracked-consent.test.ts b/tests/env-migration-tracked-consent.test.ts index 02a341226..f790e696c 100644 --- a/tests/env-migration-tracked-consent.test.ts +++ b/tests/env-migration-tracked-consent.test.ts @@ -215,7 +215,8 @@ for (const backend of ["compose", "native"] as const) { expect(run.output).toContain( `Moved ${KEY} to .hack/hack.env.local.yaml (values omitted).` ); - expect(run.output).toContain("tracked by git and can now be removed"); + expect(run.output).toContain("tracked by git"); + expect(run.output).toContain("and can now be removed"); expect(await status(project)).toEqual([ " D .hack/hack.env.json", " M .hack/docker-compose.yml", diff --git a/tests/native-authored-project-observe.test.ts b/tests/native-authored-project-observe.test.ts index 3b1926991..06bd3f869 100644 --- a/tests/native-authored-project-observe.test.ts +++ b/tests/native-authored-project-observe.test.ts @@ -151,6 +151,21 @@ async function fixture( return { scope, runtime, receipt, reply, mapping, attempts }; } +test.each([ + undefined, + "runtime_not_running", + "native_graph_foreground", +])("authored observation crosses the real bounded runtime transport (%s)", (nativeCode) => + owned(async () => { + const f = await fixture(true, null, nativeCode); + const result = await nativeAuthoredProjectPs(f); + expect(result.status).toBe( + nativeCode === undefined ? "observed" : "runtime_not_running" + ); + expect(result.items).toHaveLength(nativeCode === undefined ? 1 : 0); + expect(await Bun.file(f.attempts).text()).toBe("status\n"); + })); + test("clean absence and retained start are distinct, readonly and invoke nothing", () => owned(async () => { const f = await fixture(false); diff --git a/tests/native-runtime-client.test.ts b/tests/native-runtime-client.test.ts index 61ed069ce..92e6d09ff 100644 --- a/tests/native-runtime-client.test.ts +++ b/tests/native-runtime-client.test.ts @@ -579,6 +579,19 @@ const LATE_WRITER_SELECTIONS = { ], flags: { boundNativeStatusDrain: true }, }, + observe: { + args: [ + "graph", + "native", + "control", + "--run-id", + RUN_ID, + "--action", + "observe", + "--json", + ], + flags: { boundNativeStatusDrain: true }, + }, read: { args: ["graph", "native", "inspect", "--run-id", RUN_ID, "--json"], flags: { boundNativeAuthoredReadDrain: true }, From a39deec1683cc59648fc6f381940e1f228d6ff5f Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:58:20 +0000 Subject: [PATCH 5/5] test(native): assert recovery cancellation exit status --- tests/native-authored-command.test.ts | 77 ++++++++++++++------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/tests/native-authored-command.test.ts b/tests/native-authored-command.test.ts index bd41de4a0..8da1bb096 100644 --- a/tests/native-authored-command.test.ts +++ b/tests/native-authored-command.test.ts @@ -21,6 +21,7 @@ import { } from "../src/lib/native-authored-command.ts"; import type { NativeComposeCommandOptions } from "../src/lib/native-compose-command.ts"; import { refuseUnsupportedNativeUpOptions } from "../src/lib/native-up-refusals.ts"; +import { withOwnedProcessListeners } from "./helpers/owned-process-listeners.ts"; const roots: string[] = []; afterEach(async () => { @@ -360,42 +361,46 @@ macTest( } ); -macTest( - "recovery cancellation awaits the selected owner and restores command signal listeners", - async () => { - const selected = await fixture(); - const before = [ - process.listenerCount("SIGINT"), - process.listenerCount("SIGTERM"), - ]; - const entered = Promise.withResolvers(); - let aborted = false; - const pending = tryNativeAuthoredCommand({ - ...selected, - options: { ...selected.options, operation: "down", recover: true }, - nothingToRecover: () => Promise.resolve(false), - recover: (value) => - new Promise((_, reject) => { - value.signal?.addEventListener( - "abort", - () => { - aborted = true; - reject(new Error("Retained selected recovery.")); - }, - { once: true } - ); - entered.resolve(); - }), - }).catch((error: unknown) => error); - await entered.promise; - process.emit("SIGINT"); - expect(await pending).toHaveProperty("code", "E_LIFECYCLE_FAILED"); - expect(aborted).toBe(true); - expect([ - process.listenerCount("SIGINT"), - process.listenerCount("SIGTERM"), - ]).toEqual(before); - } +macTest.each([ + { signal: "SIGINT", code: 130 }, + { signal: "SIGTERM", code: 143 }, +] as const)( + "recovery $signal cancellation preserves exit $code and restores command signal listeners", + async ({ signal, code }) => + withOwnedProcessListeners(signal, async () => { + const selected = await fixture(); + const before = [ + process.listenerCount("SIGINT"), + process.listenerCount("SIGTERM"), + ]; + const entered = Promise.withResolvers(); + let aborted = false; + const pending = tryNativeAuthoredCommand({ + ...selected, + options: { ...selected.options, operation: "down", recover: true }, + nothingToRecover: () => Promise.resolve(false), + recover: (value) => + new Promise((_, reject) => { + value.signal?.addEventListener( + "abort", + () => { + aborted = true; + reject(new Error("Retained selected recovery.")); + }, + { once: true } + ); + entered.resolve(); + }), + }).catch((error: unknown) => error); + await entered.promise; + process.emit(signal); + expect(await pending).toBe(code); + expect(aborted).toBe(true); + expect([ + process.listenerCount("SIGINT"), + process.listenerCount("SIGTERM"), + ]).toEqual(before); + }) ); macTest( "foreground dispatch captures exact selection and delegates to the lifetime owner",